fixes global

This commit is contained in:
user committed 2025-08-08 09:14:18 +03:00
1 parent 13dc4f39c8
commit cad0f6aebe
64 files changed
+10379 -254

No files matched your search

+14 -4
View File
@@ -37,10 +37,20 @@ class Wrapped_CBotChat(T, PlayerTemplates):
@property
def bot_id(self):
return {
TELEGRAM_API_KEY: 0,
CLIENT_TELEGRAM_API_KEY: 1
}[self._bot_key]
"""
Map known tokens to stable bot IDs.
If tokens are empty/None (Telegram disabled), fall back to hash-based mapping to avoid KeyError.
"""
mapping = {}
if TELEGRAM_API_KEY:
mapping[TELEGRAM_API_KEY] = 0
if CLIENT_TELEGRAM_API_KEY:
mapping[CLIENT_TELEGRAM_API_KEY] = 1
# Try direct mapping first
if self._bot_key in mapping:
return mapping[self._bot_key]
# Fallback: deterministic bucket (keeps old behavior of 0/1 classes)
return 0 if (str(self._chat_id) + str(self._bot_key)).__hash__() % 2 == 0 else 1
async def return_result(self, result, message_type='common', message_meta={}, content_id=None, **kwargs):
if self.db_session:
+42
View File
@@ -0,0 +1,42 @@
from __future__ import annotations
from typing import Dict, Any, Optional, List, Literal
from pydantic import BaseModel, Field
class NodeHealthResponse(BaseModel):
status: Literal["ok", "degraded", "down"] = "ok"
node_id: str
public_key: str
uptime_seconds: Optional[int] = None
cpu_usage: Optional[float] = None
memory_usage_mb: Optional[float] = None
disk_free_mb: Optional[float] = None
last_sync_ts: Optional[int] = None
details: Dict[str, Any] = Field(default_factory=dict)
class ContentStatsItem(BaseModel):
content_id: str
total_chunks: int
stored_chunks: int
missing_chunks: int
size_bytes: Optional[int] = None
verified: Optional[bool] = None
class NodeContentStatsResponse(BaseModel):
total_contents: int
total_chunks: int
stored_chunks: int
missing_chunks: int
contents: List[ContentStatsItem] = Field(default_factory=list)
class NodeStatsReport(BaseModel):
action: Literal["stats_report"] = "stats_report"
reporter_node_id: str
reporter_public_key: str
timestamp: int
metrics: Dict[str, Any] = Field(default_factory=dict)
signature: Optional[str] = None # подпись может быть и в заголовке
+74
View File
@@ -0,0 +1,74 @@
from __future__ import annotations
from typing import List, Optional, Dict, Any, Literal
from pydantic import BaseModel, Field, validator
class SignedRequestHeaders(BaseModel):
"""Заголовки межузлового запроса с подписью Ed25519"""
x_node_communication: Literal["true"] = Field(alias="X-Node-Communication")
x_node_id: str = Field(alias="X-Node-ID")
x_node_public_key: str = Field(alias="X-Node-Public-Key")
x_node_signature: str = Field(alias="X-Node-Signature")
class Config:
populate_by_name = True
class ChunkRef(BaseModel):
chunk_id: str
content_id: str
chunk_index: int
chunk_hash: str
encrypted_data: str
signature: Optional[str] = None
created_at: Optional[str] = None
class ContentRequest(BaseModel):
action: Literal["content_sync"]
sync_type: Literal["content_request", "new_content", "content_list"]
content_info: Dict[str, Any] = Field(default_factory=dict)
timestamp: Optional[int] = None
@validator("content_info")
def validate_content_info(cls, v, values):
st = values.get("sync_type")
if st == "content_request":
# ожидаем content_id и indexes
if "content_id" not in v or "indexes" not in v:
raise ValueError("content_request requires content_info.content_id and content_info.indexes")
if not isinstance(v.get("indexes"), list):
raise ValueError("content_info.indexes must be a list")
elif st == "new_content":
if "content_id" not in v or "total_chunks" not in v:
raise ValueError("new_content requires content_info.content_id and content_info.total_chunks")
return v
class ContentProvideResponse(BaseModel):
success: bool = True
chunks: List[ChunkRef] = Field(default_factory=list)
errors: List[Dict[str, Any]] = Field(default_factory=list)
class ContentStatusResponse(BaseModel):
content_id: str
total_chunks: int
have_indexes: List[int] = Field(default_factory=list)
missing_indexes: List[int] = Field(default_factory=list)
verified: Optional[bool] = None
message: Optional[str] = None
class ContentVerifyRequest(BaseModel):
content_id: str
chunks: List[ChunkRef] = Field(default_factory=list)
verify_signatures: bool = True
class GenericSignedResponse(BaseModel):
success: bool
data: Dict[str, Any] = Field(default_factory=dict)
node_id: Optional[str] = None
timestamp: Optional[str] = None
+54
View File
@@ -0,0 +1,54 @@
from __future__ import annotations
import base64
import hashlib
import logging
from dataclasses import dataclass, field, asdict
from datetime import datetime
from typing import Optional
logger = logging.getLogger(__name__)
@dataclass
class ContentChunk:
"""
Модель чанка зашифрованного контента.
Все бинарные поля представлены в base64-строках для JSON-совместимости.
- chunk_hash: HEX(SHA-256(raw_encrypted_chunk_bytes)) — для дедупликации
- signature: base64-encoded Ed25519 подпись структуры чанка (детали в ChunkManager)
"""
chunk_id: str
content_id: str
chunk_index: int
chunk_hash: str # hex sha256(raw encrypted data)
encrypted_data: str # base64
signature: Optional[str] = None
created_at: str = field(default_factory=lambda: datetime.utcnow().isoformat())
def to_dict(self) -> dict:
return asdict(self)
@classmethod
def from_dict(cls, data: dict) -> "ContentChunk":
required = ["chunk_id", "content_id", "chunk_index", "chunk_hash", "encrypted_data"]
for f in required:
if f not in data:
raise ValueError(f"Missing required field in ContentChunk: {f}")
return cls(
chunk_id=data["chunk_id"],
content_id=data["content_id"],
chunk_index=int(data["chunk_index"]),
chunk_hash=data["chunk_hash"],
encrypted_data=data["encrypted_data"],
signature=data.get("signature"),
created_at=data.get("created_at") or datetime.utcnow().isoformat(),
)
def encrypted_bytes(self) -> bytes:
return base64.b64decode(self.encrypted_data)
@staticmethod
def compute_sha256_hex(buf: bytes) -> str:
return hashlib.sha256(buf).hexdigest()
@@ -0,0 +1,95 @@
"""
Модель данных EncryptedContent для хранения результата шифрования контента.
Полезно для сериализации, логирования и передачи между подсистемами uploader-bot.
"""
from __future__ import annotations
import base64
import json
import logging
from dataclasses import dataclass, field, asdict
from datetime import datetime
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
@dataclass
class EncryptedContent:
"""
Универсальная переносимая модель зашифрованного контента.
Все бинарные поля хранятся в Base64 (строки), чтобы быть JSON-совместимыми.
"""
content_id: str
ciphertext_b64: str
nonce_b64: str
tag_b64: str
# Подпись и открытый ключ подписанта (Ed25519). Могут отсутствовать.
signature: Optional[str] = None
signer_pubkey: Optional[str] = None
# Пользовательские/системные метаданные (должны совпадать при верификации)
metadata: Dict[str, Any] = field(default_factory=dict)
# Служебная метка времени создания структуры
created_at: str = field(default_factory=lambda: datetime.utcnow().isoformat())
def to_dict(self) -> Dict[str, Any]:
"""
Сериализация в словарь (JSON-совместимый).
"""
data = asdict(self)
# Ничего дополнительно не преобразуем — все поля уже JSON-friendly
return data
def to_json(self) -> str:
"""
Сериализация в JSON-строку.
"""
payload = self.to_dict()
try:
return json.dumps(payload, ensure_ascii=False, sort_keys=True)
except Exception as e:
logger.error(f"EncryptedContent.to_json serialization error: {e}")
raise
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "EncryptedContent":
"""
Десериализация из словаря.
"""
required = ["content_id", "ciphertext_b64", "nonce_b64", "tag_b64"]
for f in required:
if f not in data:
raise ValueError(f"Missing required field in EncryptedContent: {f}")
return cls(
content_id=data["content_id"],
ciphertext_b64=data["ciphertext_b64"],
nonce_b64=data["nonce_b64"],
tag_b64=data["tag_b64"],
signature=data.get("signature"),
signer_pubkey=data.get("signer_pubkey"),
metadata=data.get("metadata", {}) or {},
created_at=data.get("created_at") or datetime.utcnow().isoformat(),
)
@classmethod
def from_crypto_result(cls, crypto_result: Dict[str, Any]) -> "EncryptedContent":
"""
Удобный конструктор из результата ContentCipher.encrypt_content()
"""
return cls.from_dict(crypto_result)
# Вспомогательные методы для работы с бинарными данными (если необходимо)
def ciphertext_bytes(self) -> bytes:
return base64.b64decode(self.ciphertext_b64)
def nonce_bytes(self) -> bytes:
return base64.b64decode(self.nonce_b64)
def tag_bytes(self) -> bytes:
return base64.b64decode(self.tag_b64)
@@ -0,0 +1,104 @@
from __future__ import annotations
import enum
import time
from dataclasses import dataclass, field, asdict
from typing import Dict, Any, List, Optional, Literal, Union
class ConversionPriority(enum.IntEnum):
LOW = 10
NORMAL = 50
HIGH = 90
CRITICAL = 100
class ConversionStatus(str, enum.Enum):
QUEUED = "queued"
RUNNING = "running"
SUCCESS = "success"
FAILED = "failed"
CANCELED = "canceled"
@dataclass
class ContentMetadata:
"""
Метаданные контента для NFT и каталогизации.
"""
title: str
description: Optional[str] = None
author: Optional[str] = None
collection: Optional[str] = None
tags: List[str] = field(default_factory=list)
cover_image_b64: Optional[str] = None
# Доп. поля для Web2/Web3 совместимости
language: Optional[str] = None
explicit: Optional[bool] = None
attributes: Dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@dataclass
class ConversionTask:
"""
Описывает задачу на конвертацию для converter-module.
"""
task_id: str
input_path: str
input_ext: str
quality: Literal["high", "low"]
# Доп опции конвертера
trim: Optional[str] = None # формат "start-end" в секундах, пример "0.5-35"
custom: List[str] = field(default_factory=list)
# Интеграция с децентрализованной платформой
priority: ConversionPriority = ConversionPriority.NORMAL
attempts: int = 0
max_retries: int = 3
# NFT/контент метаданные
metadata: ContentMetadata = field(default_factory=lambda: ContentMetadata(title="Untitled"))
# Трассировка/время
created_at: int = field(default_factory=lambda: int(time.time()))
updated_at: int = field(default_factory=lambda: int(time.time()))
def to_dict(self) -> Dict[str, Any]:
d = asdict(self)
d["priority"] = int(self.priority)
return d
@dataclass
class ConversionResult:
"""
Результат конвертации.
"""
task_id: str
status: ConversionStatus
# Путь к выходному файлу конвертера внутри converter-module контейнера/процесса
converter_output_path: Optional[str] = None
# Снимок stdout/stderr или лог-файла конвертера (если доступно)
logs_path: Optional[str] = None
# Интеграция после конвертации
# content_id после шифрования, ключ для расшифровки хранится отдельно безопасно
content_id: Optional[str] = None
# Итоговые чанки (их хеши и base64-данные)
chunks: Optional[List[Dict[str, Any]]] = None
# Метаданные для NFT
nft_metadata: Optional[Dict[str, Any]] = None
# Ошибка (если FAILED)
error: Optional[str] = None
finished_at: Optional[int] = None
def to_dict(self) -> Dict[str, Any]:
d = asdict(self)
d["status"] = str(self.status.value)
return d
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from datetime import datetime
from typing import Optional
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class NFTLicense:
"""
Модель NFT лицензии на доступ к контенту.
Важно:
- license_id: уникальный идентификатор записи лицензии в нашей системе (может совпадать с nft_address или быть внутренним UUID)
- content_id: идентификатор контента (из ContentCipher, sha256 от шифроданных/метаданных)
- owner_address: адрес кошелька TON владельца NFT (user)
- nft_address: адрес NFT-токена лицензии (TON)
- created_at: когда лицензия была создана/закуплена (по данным блокчейна/системы)
- expires_at: необязательное поле срока действия (если лицензия не бессрочная)
"""
license_id: str
content_id: str
owner_address: str
nft_address: str
created_at: datetime = field(default_factory=lambda: datetime.utcnow())
expires_at: Optional[datetime] = None
def is_active(self, now: Optional[datetime] = None) -> bool:
now = now or datetime.utcnow()
if self.expires_at is None:
return True
return now < self.expires_at
def to_dict(self) -> dict:
return {
"license_id": self.license_id,
"content_id": self.content_id,
"owner_address": self.owner_address,
"nft_address": self.nft_address,
"created_at": self.created_at.isoformat(),
"expires_at": self.expires_at.isoformat() if self.expires_at else None,
}
@staticmethod
def from_dict(data: dict) -> "NFTLicense":
try:
return NFTLicense(
license_id=data["license_id"],
content_id=data["content_id"],
owner_address=data["owner_address"],
nft_address=data["nft_address"],
created_at=datetime.fromisoformat(data["created_at"]) if data.get("created_at") else datetime.utcnow(),
expires_at=datetime.fromisoformat(data["expires_at"]) if data.get("expires_at") else None,
)
except Exception as e:
logger.error("Failed to parse NFTLicense from dict: %s", e)
raise
+151
View File
@@ -0,0 +1,151 @@
from __future__ import annotations
import time
import hashlib
import json
from dataclasses import dataclass, field, asdict
from typing import Dict, Any, Optional, List
def _now_ts() -> int:
return int(time.time())
def _gen_nonce(prefix: str = "stats") -> str:
base = f"{prefix}:{_now_ts()}:{time.time_ns()}"
return hashlib.sha256(base.encode("utf-8")).hexdigest()[:16]
@dataclass
class SystemMetrics:
cpu_percent: Optional[float] = None
cpu_load_avg_1m: Optional[float] = None
cpu_load_avg_5m: Optional[float] = None
cpu_load_avg_15m: Optional[float] = None
mem_total_mb: Optional[float] = None
mem_used_mb: Optional[float] = None
mem_available_mb: Optional[float] = None
mem_percent: Optional[float] = None
disk_total_mb: Optional[float] = None
disk_used_mb: Optional[float] = None
disk_free_mb: Optional[float] = None
disk_percent: Optional[float] = None
io_read_mb_s: Optional[float] = None
io_write_mb_s: Optional[float] = None
net_sent_kb_s: Optional[float] = None
net_recv_kb_s: Optional[float] = None
uptime_seconds: Optional[int] = None
timestamp: int = field(default_factory=_now_ts)
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@staticmethod
def from_dict(data: Dict[str, Any]) -> "SystemMetrics":
return SystemMetrics(**data)
@dataclass
class AppMetrics:
total_conversions: int = 0
total_requests: int = 0
total_errors: int = 0
slow_ops_count: int = 0
avg_response_ms: Optional[float] = None
p95_response_ms: Optional[float] = None
p99_response_ms: Optional[float] = None
details: Dict[str, Any] = field(default_factory=dict)
timestamp: int = field(default_factory=_now_ts)
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@staticmethod
def from_dict(data: Dict[str, Any]) -> "AppMetrics":
return AppMetrics(**data)
@dataclass
class NodeStats:
node_id: str
public_key: str
system: SystemMetrics
app: AppMetrics
known_content_items: Optional[int] = None
available_content_items: Optional[int] = None
protocol_version: str = "stats-gossip-v1"
timestamp: int = field(default_factory=_now_ts)
nonce: str = field(default_factory=_gen_nonce)
signature: Optional[str] = None # ed25519
def to_dict(self, include_signature: bool = True) -> Dict[str, Any]:
data = {
"node_id": self.node_id,
"public_key": self.public_key,
"system": self.system.to_dict(),
"app": self.app.to_dict(),
"known_content_items": self.known_content_items,
"available_content_items": self.available_content_items,
"protocol_version": self.protocol_version,
"timestamp": self.timestamp,
"nonce": self.nonce,
}
if include_signature:
data["signature"] = self.signature
return data
@staticmethod
def canonical_payload(data: Dict[str, Any]) -> Dict[str, Any]:
# Для подписи удаляем signature и сортируем
payload = dict(data)
payload.pop("signature", None)
return payload
@staticmethod
def to_signable_json(data: Dict[str, Any]) -> str:
payload = NodeStats.canonical_payload(data)
return json.dumps(payload, sort_keys=True, ensure_ascii=False)
@staticmethod
def from_dict(data: Dict[str, Any]) -> "NodeStats":
return NodeStats(
node_id=data["node_id"],
public_key=data["public_key"],
system=SystemMetrics.from_dict(data["system"]),
app=AppMetrics.from_dict(data["app"]),
known_content_items=data.get("known_content_items"),
available_content_items=data.get("available_content_items"),
protocol_version=data.get("protocol_version", "stats-gossip-v1"),
timestamp=data.get("timestamp", _now_ts()),
nonce=data.get("nonce", _gen_nonce()),
signature=data.get("signature"),
)
@dataclass
class NetworkStats:
# Сводная статистика по сети
node_count: int
active_nodes: int
avg_uptime_seconds: Optional[float] = None
avg_cpu_percent: Optional[float] = None
avg_mem_percent: Optional[float] = None
avg_latency_ms: Optional[float] = None
total_available_content: Optional[int] = None
health_score: Optional[float] = None # 0..100
timestamp: int = field(default_factory=_now_ts)
nodes: List[Dict[str, Any]] = field(default_factory=list) # список упрощенных NodeStats резюме
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@staticmethod
def from_dict(data: Dict[str, Any]) -> "NetworkStats":
return NetworkStats(**data)
@@ -0,0 +1,75 @@
from __future__ import annotations
import json
from dataclasses import dataclass, asdict, field
from datetime import datetime
from typing import Any, Dict, Optional
def _iso_now() -> str:
return datetime.utcnow().isoformat()
@dataclass
class ValidationResult:
"""
Результат валидации контента/чанков.
"""
ok: bool
reason: Optional[str] = None
details: Dict[str, Any] = field(default_factory=dict)
timestamp: str = field(default_factory=_iso_now)
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
def to_json(self) -> str:
return json.dumps(self.to_dict(), ensure_ascii=False, sort_keys=True)
@dataclass
class ContentSignature:
"""
Информация о подписи контента/объекта.
"""
signature: Optional[str]
public_key_hex: Optional[str]
algorithm: str = "ed25519"
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@dataclass
class TrustScore:
"""
Итоговый скор доверия (0.0 - 1.0)
"""
node_id: str
score: float
updated_at: str = field(default_factory=_iso_now)
reason: Optional[str] = None
def to_dict(self) -> Dict[str, Any]:
d = asdict(self)
# Нормализация диапазона
d["score"] = max(0.0, min(1.0, float(d["score"])))
return d
@dataclass
class NodeTrust:
"""
Состояние доверия к ноде.
"""
node_id: str
score: float = 0.5
blacklisted: bool = False
manual_override: bool = False
note: Optional[str] = None
updated_at: str = field(default_factory=_iso_now)
def to_dict(self) -> Dict[str, Any]:
d = asdict(self)
d["score"] = max(0.0, min(1.0, float(d["score"])))
return d