fixes global

scripts/check_network_connectivity.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""
+Network connectivity and node statistics checker for MY Network v3.x
+
+Purpose:
+- Query local node APIs to collect connectivity and version compatibility stats
+- Optionally test bootstrap peers for reachability and latency
+- Print a concise JSON summary that start.sh can parse and log
+
+Behavior:
+- Graceful degradation: if any endpoint is missing/unavailable, it is skipped
+- Only prints JSON to stdout on success; on partial failures still prints JSON with flags
+- Exit codes:
+  0 - success (even with partial data)
+  1 - unexpected error (e.g., invalid arguments)
+
+Inputs (ENV or CLI args):
+- API_BASE (default: http://localhost:8000)
+- TIMEOUT_SECONDS (default: 5)
+- TEST_PEERS (bool as "true"/"false", default: true) - whether to actively probe peers
+"""
+
+import json
+import os
+import sys
+import time
+from urllib.parse import urljoin
+
+import urllib.request
+import urllib.error
+import ssl
+
+API_BASE = os.environ.get("API_BASE", "http://localhost:8000").rstrip("/")
+TIMEOUT = float(os.environ.get("TIMEOUT_SECONDS", "5"))
+TEST_PEERS = os.environ.get("TEST_PEERS", "true").lower() == "true"
+
+# For HTTPS with self-signed certs (when fronted by nginx in early boot)
+CTX = ssl.create_default_context()
+CTX.check_hostname = False
+CTX.verify_mode = ssl.CERT_NONE
+
+
+def http_get_json(url, timeout=TIMEOUT):
+    req = urllib.request.Request(url, method="GET")
+    try:
+        with urllib.request.urlopen(req, timeout=timeout, context=CTX) as resp:
+            data = resp.read()
+            try:
+                return json.loads(data.decode("utf-8")), None
+            except Exception as e:
+                return None, f"JSON parse error: {e}"
+    except urllib.error.HTTPError as e:
+        try:
+            body = e.read().decode("utf-8")
+        except Exception:
+            body = ""
+        return None, f"HTTP error {e.code}: {body}"
+    except Exception as e:
+        return None, f"Request error: {e}"
+
+
+def measure_latency(url, timeout=TIMEOUT):
+    start = time.time()
+    j, err = http_get_json(url, timeout=timeout)
+    elapsed = time.time() - start
+    return j, err, elapsed
+
+
+def main():
+    summary = {
+        "api_base": API_BASE,
+        "node": {
+            "health": {"ok": False, "error": None},
+            "status": {},
+            "version": None,
+        },
+        "peers": {
+            "count_reported": 0,
+            "list": [],
+            "tested": [],
+            "test_enabled": TEST_PEERS,
+        },
+        "network": {
+            "stats": {},
+            "version_mismatch": [],
+        },
+        "errors": [],
+    }
+
+    # 1) Health
+    health_url = urljoin(API_BASE + "/", "health")
+    j, err = http_get_json(health_url)
+    if j is not None:
+        summary["node"]["health"]["ok"] = True
+        summary["node"]["health"]["data"] = j
+    else:
+        summary["node"]["health"]["ok"] = False
+        summary["node"]["health"]["error"] = err
+        summary["errors"].append({"endpoint": health_url, "error": err})
+
+    # 2) Node status
+    status_url = urljoin(API_BASE + "/", "api/v3/node/status")
+    j, err = http_get_json(status_url)
+    if j is not None:
+        summary["node"]["status"] = j
+        # Try to infer version if present
+        version = None
+        for key in ("version", "node_version", "app_version"):
+            if isinstance(j, dict) and key in j:
+                version = j[key]
+                break
+        summary["node"]["version"] = version
+    elif err:
+        summary["errors"].append({"endpoint": status_url, "error": err})
+
+    # 3) Peers
+    peers_url = urljoin(API_BASE + "/", "api/v3/node/peers")
+    j, err = http_get_json(peers_url)
+    if j is not None:
+        # expected shapes: {"count": N, "peers":[...]} or list
+        if isinstance(j, dict):
+            summary["peers"]["count_reported"] = int(j.get("count", len(j.get("peers", [])) or 0))
+            peers = j.get("peers", [])
+        elif isinstance(j, list):
+            peers = j
+            summary["peers"]["count_reported"] = len(peers)
+        else:
+            peers = []
+        # normalize peers to {address, port, version?, id?}
+        norm = []
+        for p in peers:
+            if isinstance(p, dict):
+                addr = p.get("address") or p.get("host") or p.get("ip") or "unknown"
+                port = p.get("port") or 8000
+                pid = p.get("id") or p.get("node_id") or None
+                pver = p.get("version") or p.get("node_version") or None
+            else:
+                # if peer represented as string "host:port"
+                s = str(p)
+                if ":" in s:
+                    addr, port = s.split(":", 1)
+                    try:
+                        port = int(port)
+                    except Exception:
+                        port = 8000
+                else:
+                    addr, port = s, 8000
+                pid, pver = None, None
+            norm.append({"address": addr, "port": port, "id": pid, "version": pver})
+        summary["peers"]["list"] = norm
+    elif err:
+        summary["errors"].append({"endpoint": peers_url, "error": err})
+
+    # 4) Network stats
+    net_url = urljoin(API_BASE + "/", "api/v3/network/stats")
+    j, err = http_get_json(net_url)
+    if j is not None:
+        summary["network"]["stats"] = j
+        # version mismatch calc
+        local_version = summary["node"]["version"]
+        mism = []
+        peers = summary["peers"]["list"]
+        for p in peers:
+            if p.get("version") and local_version and str(p["version"]) != str(local_version):
+                mism.append({
+                    "peer": f"{p.get('address')}:{p.get('port')}",
+                    "peer_version": p["version"],
+                    "local_version": local_version
+                })
+        summary["network"]["version_mismatch"] = mism
+    elif err:
+        summary["errors"].append({"endpoint": net_url, "error": err})
+
+    # 5) Active probing (latency and reachability) for peers
+    if TEST_PEERS and summary["peers"]["list"]:
+        tested = []
+        for p in summary["peers"]["list"]:
+            addr = p["address"]
+            port = p["port"]
+            url = f"http://{addr}:{port}/health"
+            j, err, elapsed = measure_latency(url, timeout=TIMEOUT)
+            tested.append({
+                "peer": f"{addr}:{port}",
+                "ok": err is None and isinstance(j, (dict, list)),
+                "latency_ms": round(elapsed * 1000, 1),
+                "error": err,
+            })
+        summary["peers"]["tested"] = tested
+
+    print(json.dumps(summary, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Exception as e:
+        # Never spam stacktraces to stdout; start.sh will handle gracefully
+        try:
+            print(json.dumps({"fatal_error": str(e)}))
+        except Exception:
+            pass
+        sys.exit(1)

scripts/debug_chunking.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import argparse
+import base64
+import json
+import os
+import sys
+import time
+from typing import Any, Dict, List, Optional
+
+# Логирование максимально подробное
+import logging
+logging.basicConfig(level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
+log = logging.getLogger("debug_chunking")
+
+try:
+    from app.core.content.chunk_manager import ChunkManager
+    from app.core.crypto import ContentCipher
+except Exception as e:
+    print(f"[FATAL] Cannot import app modules: {e}", file=sys.stderr)
+    sys.exit(2)
+
+
+def hexdump(b: bytes, length: int = 64) -> str:
+    if b is None:
+        return "<None>"
+    x = b[:length]
+    return x.hex() + ("..." if len(b) > length else "")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Диагностика чанкинга/сборки контента")
+    ap.add_argument("file", help="Путь к файлу для проверки")
+    ap.add_argument("--aad", default="", help="Associated data (строка)")
+    ap.add_argument("--content-id", default=None, help="Явный content_id (если не указан — будет сгенерирован)")
+    ap.add_argument("--chunk-size", type=int, default=None, help="Переопределить размер чанка")
+    args = ap.parse_args()
+
+    cipher = ContentCipher()
+    cm = ChunkManager(cipher=cipher)
+    if args.chunk_size:
+        cm.CHUNK_SIZE = int(args.chunk_size)  # type: ignore[attr-defined]
+
+    with open(args.file, "rb") as f:
+        data = f.read()
+
+    content_key = cipher.generate_content_key()
+    content_id = args.content_id or ("dbg-" + os.urandom(8).hex())
+    aad = args.aad.encode("utf-8") if args.aad else None
+
+    log.info("Input file: %s size=%d", args.file, len(data))
+    log.info("Chunk size: %d", cm.CHUNK_SIZE)
+    log.debug("Content key: %s", hexdump(content_key))
+
+    t0 = time.perf_counter()
+    chunks = cm.split_content(content_id, data, content_key=content_key, metadata={"debug": True}, associated_data=aad)
+    t1 = time.perf_counter()
+    log.info("Split completed in %.3fs, chunks=%d", (t1 - t0), len(chunks))
+
+    for ch in chunks:
+        ok, err = cm.verify_chunk_integrity(ch, verify_signature=True)
+        if not ok:
+            log.error("Chunk integrity failed: idx=%s id=%s err=%s", ch.chunk_index, ch.chunk_id, err)
+        else:
+            log.debug("Chunk OK: idx=%s id=%s hash=%s enc_len=%d", ch.chunk_index, ch.chunk_id, ch.chunk_hash, len(ch.encrypted_bytes()))
+
+    t2 = time.perf_counter()
+    restored = cm.reassemble_content(chunks, content_key=content_key, associated_data=aad, expected_content_id=content_id)
+    t3 = time.perf_counter()
+    log.info("Reassemble completed in %.3fs", (t3 - t2))
+
+    if restored == data:
+        log.info("Roundtrip OK: restored equals original")
+    else:
+        log.error("Roundtrip FAILED: data mismatch (orig=%d, restored=%d)", len(data), len(restored))
+        return 3
+
+    split_thr = len(data) / max(1e-9, (t1 - t0))
+    reas_thr = len(data) / max(1e-9, (t3 - t2))
+    log.info("Throughput: split=%.2f B/s, reassemble=%.2f B/s", split_thr, reas_thr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/debug_sync.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+import argparse
+import asyncio
+import json
+import logging
+import os
+import sys
+from typing import Any, Dict, List, Optional
+
+logging.basicConfig(level=logging.DEBUG, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
+log = logging.getLogger("debug_sync")
+
+try:
+    from app.core.content.sync_manager import ContentSyncManager
+    from app.core.models.content.chunk import ContentChunk
+except Exception as e:
+    print(f"[FATAL] Cannot import app modules: {e}", file=sys.stderr)
+    sys.exit(2)
+
+
+def load_chunks_from_json(path: str) -> List[ContentChunk]:
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    chunks_raw = data.get("chunks") if isinstance(data, dict) else data
+    if not isinstance(chunks_raw, list):
+        raise ValueError("Invalid chunks JSON: expected list or {'chunks': [...]} object")
+    out: List[ContentChunk] = []
+    for d in chunks_raw:
+        out.append(ContentChunk.from_dict(d))
+    return out
+
+
+async def run_request_chunks(target_url: str, content_id: str, indexes: List[int], batch_size: int) -> Dict[str, Any]:
+    mgr = ContentSyncManager()
+    log.info("Requesting chunks from %s for content_id=%s indexes=%s", target_url, content_id, indexes)
+    res = await mgr.request_chunks(target_url, content_id, indexes, batch_size=batch_size)
+    log.info("Request done. requested=%s received=%s errors=%s", res.get("requested"), res.get("received"), len(res.get("errors", [])))
+    return res
+
+
+async def run_provide_chunks(input_json: str, content_id: str, indexes: List[int], batch_limit: int) -> Dict[str, Any]:
+    """
+    Локальная проверка выдачи чанков: читаем заранее подготовленные чанки из JSON,
+    прогоняем их через provide_chunks (включая локальную проверку целостности).
+    """
+    all_chunks = load_chunks_from_json(input_json)
+    by_idx: Dict[int, ContentChunk] = {c.chunk_index: c for c in all_chunks}
+
+    def storage_reader(cid: str, idx: int) -> Optional[ContentChunk]:
+        if cid != content_id:
+            return None
+        return by_idx.get(idx)
+
+    mgr = ContentSyncManager()
+    log.info("Providing chunks for content_id=%s indexes=%s (batch_limit=%d)", content_id, indexes, batch_limit)
+    res = await mgr.provide_chunks(content_id, indexes, storage_reader=storage_reader, batch_limit=batch_limit)
+    log.info("Provide done. ok=%d errors=%d", len(res.get("chunks", [])), len(res.get("errors", [])))
+    return res
+
+
+async def run_sync_content(nodes: List[str], content_id: str, have_indexes: List[int], total_chunks: int) -> Dict[str, Any]:
+    mgr = ContentSyncManager()
+    log.info("Sync content start: nodes=%s content_id=%s have=%d total=%d", nodes, content_id, len(have_indexes), total_chunks)
+    res = await mgr.sync_content(nodes, content_id, have_indexes=have_indexes, total_chunks=total_chunks)
+    log.info("Sync result: downloaded=%s", res.get("downloaded"))
+    return res
+
+
+def parse_int_list(s: str) -> List[int]:
+    if not s:
+        return []
+    parts = [p.strip() for p in s.split(",") if p.strip()]
+    out: List[int] = []
+    for p in parts:
+        if "-" in p:
+            a, b = p.split("-", 1)
+            out.extend(list(range(int(a), int(b) + 1)))
+        else:
+            out.append(int(p))
+    return out
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Диагностика синхронизации контента между нодами")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    p_req = sub.add_parser("request", help="Запросить чанки у удаленной ноды")
+    p_req.add_argument("--target", required=True, help="Базовый URL удаленной ноды (например http://localhost:8000)")
+    p_req.add_argument("--content-id", required=True, help="Идентификатор контента")
+    p_req.add_argument("--indexes", required=True, help="Список индексов (например '0,1,2,5-10')")
+    p_req.add_argument("--batch-size", type=int, default=32, help="Размер батча (по умолчанию 32)")
+
+    p_prov = sub.add_parser("provide", help="Проверить локальную выдачу чанков из JSON")
+    p_prov.add_argument("--input-json", required=True, help="Путь к JSON с чанками (list[chunk] или {'chunks': [...]})")
+    p_prov.add_argument("--content-id", required=True, help="Идентификатор контента")
+    p_prov.add_argument("--indexes", required=True, help="Список индексов (например '0,1,2,5-10')")
+    p_prov.add_argument("--batch-limit", type=int, default=128, help="Ограничение размеров ответа")
+
+    p_sync = sub.add_parser("sync", help="Полная процедура синхронизации по нескольким нодам")
+    p_sync.add_argument("--nodes", required=True, help="Список узлов через запятую")
+    p_sync.add_argument("--content-id", required=True)
+    p_sync.add_argument("--have-indexes", default="", help="Индексы, которые уже есть локально")
+    p_sync.add_argument("--total-chunks", type=int, required=True, help="Общее количество чанков")
+
+    args = ap.parse_args()
+
+    if args.cmd == "request":
+        indexes = parse_int_list(args.indexes)
+        res = asyncio.run(run_request_chunks(args.target, args.content_id, indexes, batch_size=args.batch_size))
+        print(json.dumps(res, ensure_ascii=False, indent=2))
+        return 0
+
+    if args.cmd == "provide":
+        indexes = parse_int_list(args.indexes)
+        res = asyncio.run(run_provide_chunks(args.input_json, args.content_id, indexes, batch_limit=args.batch_limit))
+        print(json.dumps(res, ensure_ascii=False, indent=2))
+        return 0
+
+    if args.cmd == "sync":
+        nodes = [n.strip() for n in args.nodes.split(",") if n.strip()]
+        have = parse_int_list(args.have_indexes)
+        res = asyncio.run(run_sync_content(nodes, args.content_id, have_indexes=have, total_chunks=args.total_chunks))
+        print(json.dumps(res, ensure_ascii=False, indent=2))
+        return 0
+
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_dev_env.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+# Generate strong dev .env for macOS Docker Compose run
+# Location: uploader-bot/scripts/generate_dev_env.sh
+
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+ENV_FILE="$ROOT_DIR/uploader-bot/.env"
+
+# Functions
+rand_hex() { openssl rand -hex "$1"; }
+ensure_dir() { mkdir -p "$1"; }
+abs_path() { python3 - << 'PY'
+import os,sys
+print(os.path.abspath(sys.argv[1]))
+PY
+}
+
+# Defaults
+POSTGRES_DB_DEFAULT="mynetwork"
+POSTGRES_USER_DEFAULT="myuser"
+POSTGRES_PASSWORD_DEFAULT="$(rand_hex 16)"
+DB_URL_DEFAULT="postgresql+asyncpg://${POSTGRES_USER_DEFAULT}:${POSTGRES_PASSWORD_DEFAULT}@postgres:5432/${POSTGRES_DB_DEFAULT}"
+
+REDIS_URL_DEFAULT="redis://redis:6379/0"
+
+NODE_ID_DEFAULT="local-node-$(rand_hex 4)"
+NODE_TYPE_DEFAULT="bootstrap"
+NODE_VERSION_DEFAULT="3.0.0"
+NETWORK_MODE_DEFAULT="bootstrap"
+ALLOW_INCOMING_DEFAULT="true"
+
+SECRET_KEY_DEFAULT="$(rand_hex 32)"
+JWT_SECRET_KEY_DEFAULT="$(rand_hex 32)"
+ENCRYPTION_KEY_DEFAULT="$(rand_hex 32)"
+
+STORAGE_REL="./uploader-bot/storage"
+LOGS_REL="./uploader-bot/logs"
+KEYS_REL="./uploader-bot/config/keys"
+
+API_HOST_DEFAULT="0.0.0.0"
+API_PORT_DEFAULT="8000"
+UVICORN_HOST_DEFAULT="0.0.0.0"
+UVICORN_PORT_DEFAULT="8000"
+DOCKER_SOCK_DEFAULT="/var/run/docker.sock"
+
+NODE_PRIV_PATH="/app/keys/node_private_key"
+NODE_PUB_PATH="/app/keys/node_public_key"
+
+BOOTSTRAP_CONFIG_DEFAULT="default"
+LOG_LEVEL_DEFAULT="INFO"
+MAX_PEERS_DEFAULT="50"
+SYNC_INTERVAL_DEFAULT="300"
+CONVERT_PAR_DEFAULT="2"
+CONVERT_TIMEOUT_DEFAULT="300"
+
+# Prepare folders
+ensure_dir "$ROOT_DIR/uploader-bot/storage"
+ensure_dir "$ROOT_DIR/uploader-bot/logs"
+ensure_dir "$ROOT_DIR/uploader-bot/config/keys"
+
+# Generate node keys if missing
+PRIV_KEY_HOST="$ROOT_DIR/uploader-bot/config/keys/node_private_key"
+PUB_KEY_HOST="$ROOT_DIR/uploader-bot/config/keys/node_public_key"
+
+if [ ! -f "$PRIV_KEY_HOST" ] || [ ! -f "$PUB_KEY_HOST" ]; then
+  echo "[INFO] Generating ed25519 node keypair..."
+  openssl genpkey -algorithm ed25519 -out "$PRIV_KEY_HOST"
+  openssl pkey -in "$PRIV_KEY_HOST" -pubout -out "$PUB_KEY_HOST"
+  chmod 600 "$PRIV_KEY_HOST" && chmod 644 "$PUB_KEY_HOST"
+fi
+
+# Try to compute NODE_PUBLIC_KEY_HEX (last 32 bytes of DER pubkey)
+NODE_PUBLIC_KEY_HEX=""
+if command -v xxd >/dev/null 2>&1; then
+  NODE_PUBLIC_KEY_HEX="$(openssl pkey -in "$PRIV_KEY_HOST" -pubout -outform DER | tail -c 32 | xxd -p -c 32 || true)"
+fi
+
+# Compose content
+cat > "$ENV_FILE" <<EOF
+# Generated by scripts/generate_dev_env.sh on $(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+# Database (PostgreSQL)
+POSTGRES_DB=${POSTGRES_DB_DEFAULT}
+POSTGRES_USER=${POSTGRES_USER_DEFAULT}
+POSTGRES_PASSWORD=${POSTGRES_PASSWORD_DEFAULT}
+DATABASE_URL=${DB_URL_DEFAULT}
+
+# Redis
+REDIS_URL=${REDIS_URL_DEFAULT}
+
+# Node/network
+NODE_ID=${NODE_ID_DEFAULT}
+NODE_TYPE=${NODE_TYPE_DEFAULT}
+NODE_VERSION=${NODE_VERSION_DEFAULT}
+NETWORK_MODE=${NETWORK_MODE_DEFAULT}
+ALLOW_INCOMING_CONNECTIONS=${ALLOW_INCOMING_DEFAULT}
+
+# Security
+SECRET_KEY=${SECRET_KEY_DEFAULT}
+JWT_SECRET_KEY=${JWT_SECRET_KEY_DEFAULT}
+ENCRYPTION_KEY=${ENCRYPTION_KEY_DEFAULT}
+
+# API / runtime
+API_HOST=${API_HOST_DEFAULT}
+API_PORT=${API_PORT_DEFAULT}
+UVICORN_HOST=${UVICORN_HOST_DEFAULT}
+UVICORN_PORT=${UVICORN_PORT_DEFAULT}
+DOCKER_SOCK_PATH=${DOCKER_SOCK_DEFAULT}
+
+# Node key paths inside container
+NODE_PRIVATE_KEY_PATH=${NODE_PRIV_PATH}
+NODE_PUBLIC_KEY_PATH=${NODE_PUB_PATH}
+NODE_PUBLIC_KEY_HEX=${NODE_PUBLIC_KEY_HEX}
+
+# Storage/logs (host paths are mounted by compose)
+STORAGE_PATH=${STORAGE_REL}
+
+# Bootstrap
+BOOTSTRAP_CONFIG=${BOOTSTRAP_CONFIG_DEFAULT}
+
+# Telegram (optional; leave empty to disable)
+TELEGRAM_API_KEY=
+CLIENT_TELEGRAM_API_KEY=
+
+# Logging / network params
+LOG_LEVEL=${LOG_LEVEL_DEFAULT}
+MAX_PEER_CONNECTIONS=${MAX_PEERS_DEFAULT}
+SYNC_INTERVAL=${SYNC_INTERVAL_DEFAULT}
+CONVERT_MAX_PARALLEL=${CONVERT_PAR_DEFAULT}
+CONVERT_TIMEOUT=${CONVERT_TIMEOUT_DEFAULT}
+EOF
+
+echo "[OK] .env written to: $ENV_FILE"
+echo
+echo "Summary:"
+echo "  POSTGRES_DB=${POSTGRES_DB_DEFAULT}"
+echo "  POSTGRES_USER=${POSTGRES_USER_DEFAULT}"
+echo "  POSTGRES_PASSWORD=<generated>"
+echo "  DATABASE_URL=${DB_URL_DEFAULT}"
+echo "  REDIS_URL=${REDIS_URL_DEFAULT}"
+echo "  NODE_ID=${NODE_ID_DEFAULT}"
+echo "  Keys:"
+echo "    Private: $PRIV_KEY_HOST"
+echo "    Public : $PUB_KEY_HOST"
+echo "  NODE_PUBLIC_KEY_HEX=${NODE_PUBLIC_KEY_HEX:-<not computed>}"
+echo
+echo "Next steps:"
+echo "  1) Open uploader-bot/.env and set TELEGRAM_API_KEY / CLIENT_TELEGRAM_API_KEY if needed."
+echo "  2) Run: docker compose -f uploader-bot/deployment/docker-compose.macos.yml up -d --build"
+echo "  3) Check: curl http://localhost:8000/health"

scripts/generate_node_keys.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""
+Generate Ed25519 keypair for node with stable printable outputs.
+Primary purpose: provide a portable alternative to OpenSSL-based generation.
+If 'cryptography' is unavailable, gracefully degrade with exit code 0 but no output, so caller can fallback.
+
+Outputs JSON to stdout on success:
+{
+  "private_key_pem": "...",
+  "public_key_pem": "...",
+  "public_key_hex": "....",
+  "node_id": "node-...",
+}
+
+Exit codes:
+ 0 - success OR graceful degrade (no output) when cryptography not available
+ 1 - unexpected error
+
+Note: We intentionally avoid printing anything besides the JSON on success.
+"""
+
+import sys
+import json
+
+try:
+    from cryptography.hazmat.primitives.asymmetric import ed25519
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.backends import default_backend
+except Exception:
+    # cryptography not available - graceful degradation: no output, caller will fallback
+    sys.exit(0)
+
+
+def main():
+    try:
+        # Generate Ed25519 keypair
+        private_key = ed25519.Ed25519PrivateKey.generate()
+        public_key = private_key.public_key()
+
+        private_pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        ).decode("utf-8")
+
+        public_pem = public_key.public_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        ).decode("utf-8")
+
+        # Extract raw 32-byte public key
+        public_der = public_key.public_bytes(
+            encoding=serialization.Encoding.DER,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        )
+        # The last 32 bytes are the raw key for Ed25519
+        public_key_hex = public_der[-32:].hex()
+
+        node_id = f"node-{public_key_hex[:16]}"
+
+        print(json.dumps({
+            "private_key_pem": private_pem,
+            "public_key_pem": public_pem,
+            "public_key_hex": public_key_hex,
+            "node_id": node_id,
+        }))
+        return 0
+    except Exception:
+        # Do not spam stdout; caller will fallback to OpenSSL path
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/health_check.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+import argparse
+import asyncio
+import json
+import logging
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, Tuple
+
+import aiohttp
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
+log = logging.getLogger("health_check")
+
+
+@dataclass
+class HealthReport:
+    base_url: str
+    alive: bool
+    ready: bool
+    health_status: int
+    info_status: int
+    metrics_status: int
+    details: Dict[str, Any]
+
+
+async def fetch_json(session: aiohttp.ClientSession, url: str) -> Tuple[int, Dict[str, Any]]:
+    try:
+        async with session.get(url) as resp:
+            status = resp.status
+            try:
+                data = await resp.json(content_type=None)
+            except Exception:
+                text = await resp.text()
+                data = {"raw": text}
+            return status, data
+    except Exception as e:
+        return 0, {"error": str(e)}
+
+
+async def probe(base_url: str, timeout: int = 8) -> HealthReport:
+    timeout_cfg = aiohttp.ClientTimeout(total=timeout)
+    async with aiohttp.ClientSession(timeout=timeout_cfg) as session:
+        details: Dict[str, Any] = {}
+
+        async def _g(path: str):
+            url = f"{base_url.rstrip('/')}{path}"
+            s, d = await fetch_json(session, url)
+            details[path] = {"status": s, "data": d}
+            return s, d
+
+        health_s, _ = await _g("/api/system/health")
+        info_s, _ = await _g("/api/system/info")
+        metrics_s, _ = await _g("/api/system/metrics")
+        live_s, _ = await _g("/api/system/live")
+        ready_s, _ = await _g("/api/system/ready")
+
+        alive = live_s in (200, 204)
+        ready = ready_s in (200, 204)
+
+        return HealthReport(
+            base_url=base_url,
+            alive=alive,
+            ready=ready,
+            health_status=health_s,
+            info_status=info_s,
+            metrics_status=metrics_s,
+            details=details,
+        )
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Проверка здоровья системы (health/info/metrics/ready/live)")
+    ap.add_argument("base_url", help="Базовый URL FastAPI, например http://localhost:8000")
+    ap.add_argument("-t", "--timeout", type=int, default=8)
+    ap.add_argument("--json", action="store_true", help="Вывод в JSON")
+    args = ap.parse_args()
+
+    report = asyncio.run(probe(args.base_url, timeout=args.timeout))
+
+    if args.json:
+        print(json.dumps(report.__dict__, ensure_ascii=False, indent=2))
+    else:
+        print(f"Health check for {report.base_url}")
+        print(f"- alive={report.alive} ready={report.ready}")
+        print(f"- /api/system/health: {report.health_status}")
+        print(f"- /api/system/info: {report.info_status}")
+        print(f"- /api/system/metrics: {report.metrics_status}")
+
+    # Код возврата: 0 если живой и готов, иначе 2
+    return 0 if (report.alive and report.ready) else 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_tests.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+import subprocess
+from typing import List
+
+
+def build_pytest_cmd(args: argparse.Namespace) -> List[str]:
+    cmd = [sys.executable, "-m", "pytest"]
+
+    # Пакеты тестов по умолчанию
+    test_paths = [
+        "uploader-bot/tests",
+    ]
+    cmd += test_paths
+
+    # Маркеры
+    if args.mark:
+        cmd += ["-m", args.mark]
+
+    # Параллельность (pytest-xdist)
+    if args.parallel:
+        cmd += ["-n", str(args.parallel)]
+
+    # Verbose
+    if args.verbose:
+        cmd += ["-vv"]
+
+    # Coverage
+    if args.coverage:
+        # Требуется pytest-cov
+        cov_targets = args.cov_targets or ["app"]
+        cmd += ["--cov", *cov_targets, "--cov-report", "term-missing"]
+        if args.cov_xml:
+            cmd += ["--cov-report", "xml:coverage.xml"]
+        if args.cov_html:
+            cmd += ["--cov-report", "html:coverage_html"]
+
+    # Запрет пропуска warning как ошибок
+    if args.strict:
+        cmd += ["-W", "error"]
+
+    return cmd
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Unified test runner for uploader-bot")
+    parser.add_argument("--mark", help="pytest -m expression (e.g. 'performance or e2e')", default=None)
+    parser.add_argument("--parallel", "-n", type=int, help="Number of workers for pytest-xdist", default=None)
+    parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
+    parser.add_argument("--coverage", action="store_true", help="Enable coverage via pytest-cov")
+    parser.add_argument("--cov-targets", nargs="*", help="Modules for coverage (default: app)", default=None)
+    parser.add_argument("--cov-xml", action="store_true", help="Emit coverage.xml")
+    parser.add_argument("--cov-html", action="store_true", help="Emit HTML coverage report to coverage_html/")
+    parser.add_argument("--strict", action="store_true", help="Treat warnings as errors")
+    parser.add_argument("--maxfail", type=int, default=1, help="Stop after N failures")
+    parser.add_argument("--no-capture", action="store_true", help="Disable output capture (-s)")
+    args = parser.parse_args()
+
+    cmd = build_pytest_cmd(args)
+    # Дополнительные флаги
+    cmd += ["--maxfail", str(args.maxfail)]
+    if args.no_capture:
+        cmd += ["-s"]
+
+    print("[run_tests] Command:", " ".join(cmd), flush=True)
+    env = os.environ.copy()
+    try:
+        return subprocess.call(cmd, env=env)
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/test_network.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+import argparse
+import asyncio
+import json
+from dataclasses import dataclass
+from typing import Any, Dict, List, Tuple, Optional
+
+import aiohttp
+
+
+@dataclass
+class NodeCheckResult:
+    base_url: str
+    ok: bool
+    status: str
+    details: Dict[str, Any]
+
+
+async def fetch_json(session: aiohttp.ClientSession, method: str, url: str, **kwargs) -> Tuple[int, Dict[str, Any]]:
+    try:
+        async with session.request(method, url, **kwargs) as resp:
+            status = resp.status
+            try:
+                data = await resp.json(content_type=None)
+            except Exception:
+                text = await resp.text()
+                data = {"raw": text}
+            return status, data
+    except Exception as e:
+        return 0, {"error": str(e)}
+
+
+async def check_node(session: aiohttp.ClientSession, base_url: str, timeout: int = 8) -> NodeCheckResult:
+    """
+    Диагностика удаленной ноды:
+    - /api/system/health
+    - /api/node/network/status
+    - /api/node/network/ping
+    - /api/node/v3/network/stats (если доступно)
+    - /api/node/content/sync (контракт проверки методом OPTIONS)
+    """
+    details: Dict[str, Any] = {}
+    ok = True
+    status_summary: List[str] = []
+
+    async def _probe(path: str, method: str = "GET", payload: Optional[Dict[str, Any]] = None):
+        url = f"{base_url.rstrip('/')}{path}"
+        kwargs: Dict[str, Any] = {}
+        if payload is not None:
+            kwargs["json"] = payload
+        s, d = await fetch_json(session, method, url, **kwargs)
+        details[path] = {"status": s, "data": d}
+        return s, d
+
+    # Health
+    s, _ = await _probe("/api/system/health")
+    status_summary.append(f"health={s}")
+    ok = ok and (s in (200, 503))  # 503 допустим как сигнал деградации
+
+    # Network status
+    s, _ = await _probe("/api/node/network/status")
+    status_summary.append(f"net.status={s}")
+    ok = ok and (s in (200, 401, 404, 405))
+
+    # Ping
+    s, _ = await _probe("/api/node/network/ping")
+    status_summary.append(f"net.ping={s}")
+    ok = ok and (s in (200, 401, 404, 405))
+
+    # v3 stats (если есть)
+    s, _ = await _probe("/api/node/v3/network/stats")
+    status_summary.append(f"v3.stats={s}")
+    ok = ok and (s in (200, 401, 404, 405))
+
+    # content sync contract presence via OPTIONS
+    s, _ = await _probe("/api/node/content/sync", method="OPTIONS")
+    status_summary.append(f"content.sync.options={s}")
+    ok = ok and (s in (200, 204, 405))
+
+    return NodeCheckResult(
+        base_url=base_url,
+        ok=ok,
+        status=";".join(status_summary),
+        details=details,
+    )
+
+
+async def run(nodes: List[str], parallel: int = 8, timeout: int = 8) -> List[NodeCheckResult]:
+    connector = aiohttp.TCPConnector(limit=parallel, ttl_dns_cache=60)
+    timeout_cfg = aiohttp.ClientTimeout(total=timeout)
+    async with aiohttp.ClientSession(connector=connector, timeout=timeout_cfg) as session:
+        tasks = [check_node(session, n, timeout=timeout) for n in nodes]
+        return await asyncio.gather(*tasks)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Проверка доступности и базового контракта сети нод")
+    parser.add_argument("nodes", nargs="+", help="Базовые URL нод, например http://localhost:8000")
+    parser.add_argument("-n", "--parallel", type=int, default=8, help="Параллельность запросов")
+    parser.add_argument("-t", "--timeout", type=int, default=8, help="Таймаут запроса в секундах")
+    parser.add_argument("--json", action="store_true", help="Вывести результат в JSON")
+    args = parser.parse_args()
+
+    results = asyncio.run(run(args.nodes, parallel=args.parallel, timeout=args.timeout))
+
+    if args.json:
+        print(json.dumps([r.__dict__ for r in results], ensure_ascii=False, indent=2))
+    else:
+        print("Network diagnostics:")
+        for r in results:
+            mark = "OK" if r.ok else "FAIL"
+            print(f"- {r.base_url}: {mark} | {r.status}")
+
+    return 0 if all(r.ok for r in results) else 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())