fixes global

app/core/stats/gossip_manager.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Dict, Any, List, Optional, Tuple, Set
+
+from app.core.crypto import get_ed25519_manager
+from app.core.network.node_client import NodeClient
+from app.core.models.stats.metrics_models import NodeStats
+
+logger = logging.getLogger(__name__)
+
+
+class GossipSecurityError(Exception):
+    pass
+
+
+class GossipManager:
+    """
+    Gossip протокол для обмена статистикой между нодами.
+    - Подпись ed25519 всех исходящих сообщений
+    - Валидация подписи входящих сообщений
+    - Антиспам: проверка timestamp (±300с), дедуп по nonce, rate limiting
+    """
+
+    def __init__(self, rate_limit_per_minute: int = 240) -> None:
+        self._seen_nonces: Set[str] = set()
+        self._nonce_ttl: Dict[str, float] = {}
+        self._rate_counters: Dict[str, Tuple[int, float]] = {}  # node_id -> (count, window_start)
+        self._rate_limit = rate_limit_per_minute
+        self._lock = asyncio.Lock()
+
+    async def _prune(self) -> None:
+        now = time.time()
+        # очистка старых nonces
+        stale = [n for n, ts in self._nonce_ttl.items() if now - ts > 600]
+        for n in stale:
+            self._nonce_ttl.pop(n, None)
+            self._seen_nonces.discard(n)
+
+        # очистка rate окон
+        for node_id, (cnt, wnd) in list(self._rate_counters.items()):
+            if now - wnd > 60:
+                self._rate_counters.pop(node_id, None)
+
+    async def _register_nonce(self, nonce: str) -> bool:
+        await self._prune()
+        if nonce in self._seen_nonces:
+            return False
+        self._seen_nonces.add(nonce)
+        self._nonce_ttl[nonce] = time.time()
+        return True
+
+    async def _check_rate(self, node_id: str) -> bool:
+        now = time.time()
+        cnt, wnd = self._rate_counters.get(node_id, (0, now))
+        if now - wnd > 60:
+            cnt, wnd = 0, now
+        cnt += 1
+        self._rate_counters[node_id] = (cnt, wnd)
+        return cnt <= self._rate_limit
+
+    async def broadcast_stats(self, peers: List[str], stats: NodeStats) -> Dict[str, Dict[str, Any]]:
+        """
+        Подписывает и отправляет статистику на список пиров.
+        Возвращает словарь результатов по нодам.
+        """
+        results: Dict[str, Dict[str, Any]] = {}
+        crypto = get_ed25519_manager()
+        signed_payload = stats.to_dict(include_signature=False)
+        # canonical signing
+        signature = crypto.sign_message(NodeStats.canonical_payload(signed_payload))
+        signed_payload["signature"] = signature
+
+        async with NodeClient() as client:
+            tasks: List[Tuple[str, asyncio.Task]] = []
+            for url in peers:
+                # POST /api/node/stats/report — уже реализованный маршрут приемника
+                task = asyncio.create_task(self._post_signed_report(client, url, signed_payload))
+                tasks.append((url, task))
+
+            for url, t in tasks:
+                try:
+                    results[url] = await t
+                except Exception as e:
+                    logger.exception("broadcast_stats error to %s: %s", url, e)
+                    results[url] = {"success": False, "error": str(e)}
+
+        return results
+
+    async def _post_signed_report(self, client: NodeClient, target_url: str, payload: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Использует NodeClient для отправки подписанного запроса на /api/node/stats/report.
+        """
+        from urllib.parse import urljoin  # локальный импорт чтобы не тянуть наверх
+        endpoint = urljoin(target_url, "/api/node/stats/report")
+
+        # NodeClient формирует заголовки/подпись через _create_signed_request,
+        # но мы уже подписали тело, поэтому вложим его как data.metrics.
+        # Обернем в совместимый формат NodeStatsReport.
+        body = {
+            "action": "stats_report",
+            "reporter_node_id": payload["node_id"],
+            "reporter_public_key": payload["public_key"],
+            "timestamp": payload["timestamp"],
+            "metrics": payload,  # целиком вложим NodeStats как metrics
+            "signature": payload.get("signature"),
+        }
+
+        req = await client._create_signed_request("stats_report", body, target_url)  # noqa: protected access by design
+        try:
+            async with client.session.post(endpoint, **req) as resp:
+                data = await resp.json()
+                return {"success": resp.status == 200, "status": resp.status, "data": data}
+        except Exception as e:
+            logger.warning("Failed to send stats to %s: %s", target_url, e)
+            return {"success": False, "error": str(e)}
+
+    async def receive_stats(self, incoming: Dict[str, Any]) -> NodeStats:
+        """
+        Прием и валидация входящей статистики от другой ноды.
+        Возвращает десериализованный NodeStats при успехе, иначе бросает GossipSecurityError.
+        expected format: NodeStats dict (с signature)
+        """
+        crypto = get_ed25519_manager()
+        try:
+            # базовые проверки
+            for key in ("node_id", "public_key", "timestamp", "nonce", "system", "app"):
+                if key not in incoming:
+                    raise GossipSecurityError(f"Missing field: {key}")
+
+            # timestamp window
+            now = int(time.time())
+            if abs(now - int(incoming["timestamp"])) > 300:
+                raise GossipSecurityError("Timestamp out of window")
+
+            # nonce dedup
+            async with self._lock:
+                if not await self._register_nonce(str(incoming["nonce"])):
+                    raise GossipSecurityError("Duplicate nonce")
+
+            # rate limit per source
+            async with self._lock:
+                if not await self._check_rate(str(incoming["node_id"])):
+                    raise GossipSecurityError("Rate limit exceeded")
+
+            # verify signature
+            signature = incoming.get("signature")
+            if not signature:
+                raise GossipSecurityError("Missing signature")
+            if not crypto.verify_signature(NodeStats.canonical_payload(incoming), signature, incoming["public_key"]):
+                raise GossipSecurityError("Invalid signature")
+
+            return NodeStats.from_dict(incoming)
+        except GossipSecurityError:
+            raise
+        except Exception as e:
+            logger.exception("receive_stats validation error: %s", e)
+            raise GossipSecurityError(str(e))
+
+    async def sync_with_peers(self, peers: List[str], get_local_stats_cb) -> Dict[str, Dict[str, Any]]:
+        """
+        Выполняет сбор локальной статистики через callback и рассылает ее всем пирам.
+        get_local_stats_cb: async () -> NodeStats
+        """
+        try:
+            local_stats: NodeStats = await get_local_stats_cb()
+        except Exception as e:
+            logger.exception("sync_with_peers: failed to get local stats: %s", e)
+            return {"error": {"success": False, "error": "local_stats_failure", "detail": str(e)}}
+
+        return await self.broadcast_stats(peers, local_stats)

app/core/stats/metrics_collector.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import time
+from typing import Optional, Tuple
+
+from app.core.models.stats.metrics_models import SystemMetrics, AppMetrics
+
+logger = logging.getLogger(__name__)
+
+
+def _try_import_psutil():
+    try:
+        import psutil  # type: ignore
+        return psutil
+    except Exception as e:
+        logger.warning("psutil not available, system metrics will be limited: %s", e)
+        return None
+
+
+class MetricsCollector:
+    """
+    Сборщик внутренних метрик:
+    - System: CPU, RAM, Disk, IO, Network
+    - App: conversions, requests, errors, slow ops, latency
+    Хранит только последнюю сессию счетчиков (агрегация истории выполняется в StatsAggregator).
+    """
+
+    def __init__(self) -> None:
+        self._psutil = _try_import_psutil()
+
+        # App counters
+        self._total_conversions = 0
+        self._total_requests = 0
+        self._total_errors = 0
+        self._slow_ops_count = 0
+
+        # Latency rolling values (экспоненциальная сглаженная средняя для p95/p99 — упрощённо)
+        self._avg_response_ms: Optional[float] = None
+        self._p95_response_ms: Optional[float] = None
+        self._p99_response_ms: Optional[float] = None
+
+        # Previous snapshots for rate calculations
+        self._last_disk_io: Optional[Tuple[int, int, float]] = None  # (read_bytes, write_bytes, ts)
+        self._last_net_io: Optional[Tuple[int, int, float]] = None   # (bytes_sent, bytes_recv, ts)
+
+        # Uptime
+        try:
+            self._start_ts = int(os.getenv("NODE_START_TS", str(int(time.time()))))
+        except Exception:
+            self._start_ts = int(time.time())
+
+        # Async lock to protect counters
+        self._lock = asyncio.Lock()
+
+    async def collect_system_metrics(self) -> SystemMetrics:
+        ps = self._psutil
+        now = time.time()
+
+        cpu_percent = None
+        load1 = load5 = load15 = None
+        mem_total = mem_used = mem_available = mem_percent = None
+        disk_total = disk_used = disk_free = disk_percent = None
+        io_read_mb_s = io_write_mb_s = None
+        net_sent_kb_s = net_recv_kb_s = None
+
+        try:
+            if ps:
+                # CPU
+                cpu_percent = float(ps.cpu_percent(interval=None))
+                try:
+                    load1, load5, load15 = ps.getloadavg() if hasattr(ps, "getloadavg") else os.getloadavg()  # type: ignore
+                except Exception:
+                    load1 = load5 = load15 = None
+
+                # Memory
+                vm = ps.virtual_memory()
+                mem_total = round(vm.total / (1024 * 1024), 2)
+                mem_used = round(vm.used / (1024 * 1024), 2)
+                mem_available = round(vm.available / (1024 * 1024), 2)
+                mem_percent = float(vm.percent)
+
+                # Disk
+                du = ps.disk_usage("/")
+                disk_total = round(du.total / (1024 * 1024), 2)
+                disk_used = round(du.used / (1024 * 1024), 2)
+                disk_free = round(du.free / (1024 * 1024), 2)
+                disk_percent = float(du.percent)
+
+                # IO rates
+                try:
+                    dio = ps.disk_io_counters()
+                    if dio and self._last_disk_io:
+                        last_read, last_write, last_ts = self._last_disk_io
+                        dt = max(now - last_ts, 1e-6)
+                        io_read_mb_s = round((max(dio.read_bytes - last_read, 0) / (1024 * 1024)) / dt, 3)
+                        io_write_mb_s = round((max(dio.write_bytes - last_write, 0) / (1024 * 1024)) / dt, 3)
+                    self._last_disk_io = (dio.read_bytes, dio.write_bytes, now) if dio else self._last_disk_io
+                except Exception:
+                    io_read_mb_s = io_write_mb_s = None
+
+                # NET rates
+                try:
+                    nio = ps.net_io_counters()
+                    if nio and self._last_net_io:
+                        last_sent, last_recv, last_ts = self._last_net_io
+                        dt = max(now - last_ts, 1e-6)
+                        net_sent_kb_s = round((max(nio.bytes_sent - last_sent, 0) / 1024) / dt, 3)
+                        net_recv_kb_s = round((max(nio.bytes_recv - last_recv, 0) / 1024) / dt, 3)
+                    self._last_net_io = (nio.bytes_sent, nio.bytes_recv, now) if nio else self._last_net_io
+                except Exception:
+                    net_sent_kb_s = net_recv_kb_s = None
+
+        except Exception as e:
+            logger.exception("collect_system_metrics error: %s", e)
+
+        return SystemMetrics(
+            cpu_percent=cpu_percent,
+            cpu_load_avg_1m=load1,
+            cpu_load_avg_5m=load5,
+            cpu_load_avg_15m=load15,
+            mem_total_mb=mem_total,
+            mem_used_mb=mem_used,
+            mem_available_mb=mem_available,
+            mem_percent=mem_percent,
+            disk_total_mb=disk_total,
+            disk_used_mb=disk_used,
+            disk_free_mb=disk_free,
+            disk_percent=disk_percent,
+            io_read_mb_s=io_read_mb_s,
+            io_write_mb_s=io_write_mb_s,
+            net_sent_kb_s=net_sent_kb_s,
+            net_recv_kb_s=net_recv_kb_s,
+            uptime_seconds=int(time.time()) - self._start_ts,
+        )
+
+    async def collect_app_metrics(self) -> AppMetrics:
+        # Снимок текущих счетчиков; агрегирование распределено в StatsAggregator
+        async with self._lock:
+            return AppMetrics(
+                total_conversions=self._total_conversions,
+                total_requests=self._total_requests,
+                total_errors=self._total_errors,
+                slow_ops_count=self._slow_ops_count,
+                avg_response_ms=self._avg_response_ms,
+                p95_response_ms=self._p95_response_ms,
+                p99_response_ms=self._p99_response_ms,
+                details={},  # можно расширить деталями модулей
+            )
+
+    async def get_current_stats(self) -> Tuple[SystemMetrics, AppMetrics]:
+        sysm = await self.collect_system_metrics()
+        appm = await self.collect_app_metrics()
+        return sysm, appm
+
+    # Hooks to update app metrics
+
+    async def inc_conversions(self, n: int = 1) -> None:
+        async with self._lock:
+            self._total_conversions += n
+
+    async def inc_requests(self, n: int = 1) -> None:
+        async with self._lock:
+            self._total_requests += n
+
+    async def inc_errors(self, n: int = 1) -> None:
+        async with self._lock:
+            self._total_errors += n
+
+    async def inc_slow_ops(self, n: int = 1) -> None:
+        async with self._lock:
+            self._slow_ops_count += n
+
+    async def observe_latency_ms(self, value_ms: float) -> None:
+        """
+        Простая статистика латентности:
+        - EMA для avg
+        - аппроксимация p95/p99 по взвешенному максимуму (упрощённо, без HDR Histogram)
+        """
+        async with self._lock:
+            alpha = 0.1
+            if self._avg_response_ms is None:
+                self._avg_response_ms = value_ms
+            else:
+                self._avg_response_ms = (1 - alpha) * self._avg_response_ms + alpha * value_ms
+
+            # Простая аппроксимация квантили при помощи EMA "максимума"
+            def ema_max(current: Optional[float], x: float, beta: float) -> float:
+                return x if current is None else max((1 - beta) * current, x)
+
+            self._p95_response_ms = ema_max(self._p95_response_ms, value_ms, beta=0.05)
+            self._p99_response_ms = ema_max(self._p99_response_ms, value_ms, beta=0.01)

app/core/stats/stats_aggregator.py

@@ -0,0 +1,198 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import statistics
+import time
+from collections import deque, defaultdict
+from typing import Deque, Dict, Any, Optional, List, Tuple
+
+from app.core.models.stats.metrics_models import SystemMetrics, AppMetrics, NodeStats, NetworkStats
+from app.core.crypto import get_ed25519_manager
+
+logger = logging.getLogger(__name__)
+
+
+class StatsAggregator:
+    """
+    Агрегатор статистики:
+    - хранит историю локальных метрик и входящих метрик от других нод (in-memory, ring buffer)
+    - вычисляет агрегаты и тренды
+    - предоставляет network overview
+    """
+
+    def __init__(self, history_limit: int = 1000) -> None:
+        self._history_limit = history_limit
+
+        # История локальной ноды: deque[(ts, NodeStats)]
+        self._local_history: Deque[Tuple[int, NodeStats]] = deque(maxlen=history_limit)
+
+        # История по нодам сети: node_id -> deque[(ts, NodeStats)]
+        self._peers_history: Dict[str, Deque[Tuple[int, NodeStats]]] = defaultdict(lambda: deque(maxlen=history_limit))
+
+        # Кеш последнего слепка по нодам
+        self._last_by_node: Dict[str, NodeStats] = {}
+
+        # Список известных пиров (URL) - поддержка network overview
+        self._known_peers: List[str] = []
+
+        self._lock = asyncio.Lock()
+
+    async def set_known_peers(self, peers: List[str]) -> None:
+        async with self._lock:
+            self._known_peers = list(sorted(set(peers)))
+
+    async def add_local_snapshot(self, stats: NodeStats) -> None:
+        async with self._lock:
+            ts = stats.timestamp
+            self._local_history.append((ts, stats))
+            self._last_by_node[stats.node_id] = stats
+
+    async def add_peer_snapshot(self, stats: NodeStats) -> None:
+        async with self._lock:
+            ts = stats.timestamp
+            dq = self._peers_history[stats.node_id]
+            dq.append((ts, stats))
+            self._last_by_node[stats.node_id] = stats
+
+    async def get_latest_local(self) -> Optional[NodeStats]:
+        async with self._lock:
+            return self._local_history[-1][1] if self._local_history else None
+
+    async def aggregate_node_stats(self, node_id: Optional[str] = None, last_n: int = 20) -> Dict[str, Any]:
+        """
+        Возвращает агрегаты для указанной ноды (по умолчанию локальная).
+        """
+        async with self._lock:
+            if node_id is None:
+                series = list(self._local_history)[-last_n:]
+            else:
+                series = list(self._peers_history.get(node_id, deque()))[-last_n:]
+
+            if not series:
+                return {"samples": 0}
+
+            # агрегаты по cpu/mem
+            cpu = [s.system.cpu_percent for _, s in series if s.system.cpu_percent is not None]
+            mem = [s.system.mem_percent for _, s in series if s.system.mem_percent is not None]
+
+            res = {
+                "samples": len(series),
+                "time_span_sec": (series[-1][0] - series[0][0]) if len(series) > 1 else 0,
+                "cpu": {
+                    "avg": round(statistics.fmean(cpu), 3) if cpu else None,
+                    "max": round(max(cpu), 3) if cpu else None,
+                    "min": round(min(cpu), 3) if cpu else None,
+                },
+                "mem": {
+                    "avg": round(statistics.fmean(mem), 3) if mem else None,
+                    "max": round(max(mem), 3) if mem else None,
+                    "min": round(min(mem), 3) if mem else None,
+                },
+            }
+            return res
+
+    async def get_network_overview(self) -> NetworkStats:
+        """
+        Сводка по сети с использованием последних значений по всем известным нодам.
+        """
+        async with self._lock:
+            nodes = list(self._last_by_node.values())
+
+        node_count = len(nodes)
+        active_nodes = sum(1 for n in nodes if (int(time.time()) - n.timestamp) <= 300)
+
+        uptimes = [n.system.uptime_seconds for n in nodes if n.system.uptime_seconds is not None]
+        cpus = [n.system.cpu_percent for n in nodes if n.system.cpu_percent is not None]
+        mems = [n.system.mem_percent for n in nodes if n.system.mem_percent is not None]
+
+        avg_uptime = round(statistics.fmean(uptimes), 3) if uptimes else None
+        avg_cpu = round(statistics.fmean(cpus), 3) if cpus else None
+        avg_mem = round(statistics.fmean(mems), 3) if mems else None
+
+        # Простейшая метрика "здоровья" сети: 100 - avg_cpu/avg_mem penalty
+        health_score = None
+        if avg_cpu is not None and avg_mem is not None:
+            penalty = (avg_cpu / 2.0) + (avg_mem / 2.0)  # 0..200
+            health_score = max(0.0, 100.0 - min(100.0, penalty))
+
+        nodes_summary: List[Dict[str, Any]] = []
+        for n in nodes:
+            nodes_summary.append({
+                "node_id": n.node_id,
+                "uptime": n.system.uptime_seconds,
+                "cpu": n.system.cpu_percent,
+                "mem": n.system.mem_percent,
+                "available_content_items": n.available_content_items,
+                "timestamp": n.timestamp,
+            })
+
+        # latency/total_available_content пока не вычисляем здесь, можно обновить из внешних сигналов
+        return NetworkStats(
+            node_count=node_count,
+            active_nodes=active_nodes,
+            avg_uptime_seconds=avg_uptime,
+            avg_cpu_percent=avg_cpu,
+            avg_mem_percent=avg_mem,
+            avg_latency_ms=None,
+            total_available_content=sum((n.available_content_items or 0) for n in nodes) if nodes else None,
+            health_score=health_score,
+            nodes=nodes_summary,
+        )
+
+    async def calculate_trends(self, node_id: Optional[str] = None, window: int = 60) -> Dict[str, Any]:
+        """
+        Грубая оценка тренда по cpu/mem: сравнение первых и последних значений окна.
+        """
+        async with self._lock:
+            series = list(self._local_history if node_id is None else self._peers_history.get(node_id, deque()))
+
+        if not series:
+            return {}
+
+        # берем последние window секунд данных
+        cutoff = int(time.time()) - window
+        window_series = [s for s in series if s[0] >= cutoff]
+        if len(window_series) < 2:
+            return {"samples": len(window_series)}
+
+        first = window_series[0][1]
+        last = window_series[-1][1]
+
+        def delta(a: Optional[float], b: Optional[float]) -> Optional[float]:
+            if a is None or b is None:
+                return None
+            return round(b - a, 3)
+
+        trend = {
+            "samples": len(window_series),
+            "cpu_percent_delta": delta(first.system.cpu_percent, last.system.cpu_percent),
+            "mem_percent_delta": delta(first.system.mem_percent, last.system.mem_percent),
+        }
+        return trend
+
+    async def build_local_signed_stats(self) -> NodeStats:
+        """
+        Собирает последний локальный слепок и подписывает.
+        """
+        async with self._lock:
+            latest = self._local_history[-1][1] if self._local_history else None
+
+        if not latest:
+            raise RuntimeError("No local stats available")
+
+        crypto = get_ed25519_manager()
+        payload = latest.to_dict(include_signature=False)
+        signature = crypto.sign_message(NodeStats.canonical_payload(payload))
+        latest.signature = signature
+        return latest
+
+    # Вспомогательные методы для тестов/диагностики
+
+    async def list_known_peers(self) -> List[str]:
+        async with self._lock:
+            return list(self._known_peers)
+
+    async def last_by_node(self) -> Dict[str, NodeStats]:
+        async with self._lock:
+            return dict(self._last_by_node)