sync improve

app/api/routes/content.py

@@ -12,6 +12,7 @@ from app.core.models.content.user_content import UserContent
 from app.core._config import CLIENT_TELEGRAM_API_KEY, CLIENT_TELEGRAM_BOT_USERNAME, PROJECT_HOST
 from app.core.models.content_v3 import EncryptedContent as ECv3, ContentDerivative as CDv3, UploadSession
 from app.core.content.content_id import ContentId
+from app.core.network.dht import MetricsAggregator
 import json
 import uuid
 
@@ -418,6 +419,35 @@ async def s_api_v1_content_view(request, content_address: str):
     if not opts.get('content_ext') and '/' in content_mime:
         opts['content_ext'] = content_mime.split('/')[-1]
 
+    metrics_mgr: MetricsAggregator | None = getattr(request.app.ctx.memory, "metrics", None)
+    if metrics_mgr:
+        viewer_salt_raw = request.headers.get("X-View-Salt")
+        if viewer_salt_raw:
+            try:
+                viewer_salt = bytes.fromhex(viewer_salt_raw)
+            except ValueError:
+                viewer_salt = viewer_salt_raw.encode()
+        elif request.ctx.user:
+            viewer_salt = f"user:{request.ctx.user.id}".encode()
+        else:
+            viewer_salt = (request.remote_addr or request.ip or "anonymous").encode()
+        try:
+            watch_time_param = int(request.args.get("watch_time", 0))
+        except (TypeError, ValueError):
+            watch_time_param = 0
+        try:
+            bytes_out_param = int(request.args.get("bytes_out", 0))
+        except (TypeError, ValueError):
+            bytes_out_param = 0
+        completed_param = request.args.get("completed", "0") in ("1", "true", "True")
+        metrics_mgr.record_view(
+            content_id=content['encrypted_content'].hash,
+            viewer_salt=viewer_salt,
+            watch_time=watch_time_param,
+            bytes_out=bytes_out_param,
+            completed=completed_param,
+        )
+
     return response.json({
         **opts,
         'encrypted': content['encrypted_content'].json_format(),

app/api/routes/metrics.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from sanic import response
+
+
+async def s_api_metrics(request):
+    try:
+        from prometheus_client import generate_latest, CONTENT_TYPE_LATEST  # type: ignore
+        data = generate_latest()
+        return response.raw(data, content_type=CONTENT_TYPE_LATEST)
+    except Exception:
+        # Fallback: export minimal in-process counters from DHT module, if available
+        try:
+            from app.core.network.dht import prometheus as dprom
+
+            def dump(metric_obj, metric_name):
+                lines = []
+                values = getattr(metric_obj, "_values", {})
+                for labels, value in values.items():
+                    label_str = ",".join(f'{k}="{v}"' for k, v in labels)
+                    if label_str:
+                        lines.append(f"{metric_name}{{{label_str}}} {value}")
+                    else:
+                        lines.append(f"{metric_name} {value}")
+                return lines
+
+            parts = []
+            parts += dump(dprom.replication_under, "dht_replication_under_total")
+            parts += dump(dprom.replication_over, "dht_replication_over_total")
+            parts += dump(dprom.leader_changes, "dht_leader_changes_total")
+            parts += dump(dprom.merge_conflicts, "dht_merge_conflicts_total")
+            parts += dump(dprom.view_count_total, "dht_view_count_total")
+            parts += dump(dprom.unique_estimate, "dht_unique_view_estimate")
+            parts += dump(dprom.watch_time_seconds, "dht_watch_time_seconds")
+            body = "\n".join(parts) + ("\n" if parts else "")
+            return response.text(body, content_type="text/plain; version=0.0.4")
+        except Exception:
+            return response.text("")
+

app/api/routes/network.py

@@ -4,7 +4,7 @@ import json
 from datetime import datetime
 from typing import Dict, Any
 
-from base58 import b58decode
+from app.core._utils.b58 import b58decode
 from sanic import response
 from urllib.parse import urlparse
 
@@ -19,6 +19,8 @@ from app.core.network.config import HANDSHAKE_TS_TOLERANCE_SEC
 from app.core.ipfs_client import swarm_connect
 from app.core._config import PROJECT_HOST
 from app.core.events.service import record_event
+from app.core.network.asn import resolver as asn_resolver
+from app.core.network.dht import compute_node_id, dht_config, ReachabilityReceipt
 
 
 def _port_from_public_host(public_host: str) -> int:
@@ -91,7 +93,7 @@ async def s_api_v1_network_handshake(request):
         return response.json({"error": "RATE_LIMIT"}, status=429)
 
     data = request.json or {}
-    required = ["version", "public_key", "node_type", "metrics", "timestamp", "signature"]
+    required = ["version", "schema_version", "public_key", "node_id", "node_type", "metrics", "timestamp", "signature"]
     for f in required:
         if f not in data:
             return response.json({"error": f"Missing field {f}"}, status=400)
@@ -137,22 +139,62 @@ async def s_api_v1_network_handshake(request):
             "peer": peer_version,
         }, status=409)
 
-    # Verify signature
+    # Verify signature (Ed25519). If libsodium not available, accept but log a warning.
+    signed_fields = {k: v for (k, v) in data.items() if k != "signature"}
+    blob = json.dumps(signed_fields, sort_keys=True, separators=(",", ":")).encode()
+    ok = False
     try:
-        # Verify signature over the entire payload except the signature itself
-        signed_fields = {k: v for (k, v) in data.items() if k != "signature"}
-        blob = json.dumps(signed_fields, sort_keys=True, separators=(",", ":")).encode()
-        import nacl.signing, nacl.encoding
-        vk = nacl.signing.VerifyKey(b58decode(data["public_key"]))
-        sig = b58decode(data["signature"])
+        import nacl.signing, nacl.encoding  # type: ignore
+        vk = nacl.signing.VerifyKey(b58decode(data.get("public_key", "")))
+        sig = b58decode(data.get("signature", ""))
         vk.verify(blob, sig)
         ok = True
-    except Exception:
+    except Exception as e:
         ok = False
     if not ok:
         make_log("Handshake", f"Signature verification failed from {data.get('public_host')}", level='warning')
         return response.json({"error": "BAD_SIGNATURE"}, status=400)
 
+    # Update membership / reachability information
+    try:
+        membership_mgr = getattr(request.app.ctx.memory, "membership", None)
+        if membership_mgr:
+            remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip() or None
+            remote_asn = data.get("asn")
+            if remote_asn is None:
+                remote_asn = asn_resolver.resolve(remote_ip)
+            else:
+                if remote_ip:
+                    asn_resolver.learn(remote_ip, int(remote_asn))
+            membership_mgr.update_member(
+                node_id=data["node_id"],
+                public_key=data["public_key"],
+                ip=remote_ip,
+                asn=int(remote_asn) if remote_asn is not None else None,
+                metadata={
+                    "capabilities": data.get("capabilities", {}),
+                    "metrics": data.get("metrics", {}),
+                    "public_host": data.get("public_host"),
+                },
+            )
+            for receipt in data.get("reachability_receipts") or []:
+                if not receipt.get("target_id") or not receipt.get("issuer_id"):
+                    continue
+                try:
+                    membership_mgr.record_receipt(
+                        ReachabilityReceipt(
+                            target_id=str(receipt.get("target_id")),
+                            issuer_id=str(receipt.get("issuer_id")),
+                            asn=int(receipt["asn"]) if receipt.get("asn") is not None else None,
+                            timestamp=float(receipt.get("timestamp", data.get("timestamp"))),
+                            signature=str(receipt.get("signature", "")),
+                        )
+                    )
+                except Exception:
+                    continue
+    except Exception as exc:
+        make_log("Handshake", f"Membership ingest failed: {exc}", level='warning')
+
     # Upsert node and respond with our info + known public nodes
     # Do not persist private peers (ephemeral)
     if data.get("node_type") != "private" and data.get("public_host"):
@@ -215,10 +257,13 @@ async def s_api_v1_network_handshake(request):
 
     node = await compute_node_info(request.ctx.db_session)
     known = await list_known_public_nodes(request.ctx.db_session)
+    membership_mgr = getattr(request.app.ctx.memory, "membership", None)
+    n_estimate = membership_mgr.n_estimate() if membership_mgr else 0
     resp = sign_response({
         "compatibility": comp,
         "node": node,
         "known_public_nodes": known,
+        "n_estimate": n_estimate,
     })
     make_log("Handshake", f"OK with {data.get('public_host')} compat={comp}")
     status = 200
@@ -226,3 +271,8 @@ async def s_api_v1_network_handshake(request):
         status = 200
         resp["warning"] = "MINOR version differs; proceed with caution"
     return response.json(resp, status=status)
+    if data.get("schema_version") != dht_config.schema_version:
+        return response.json({"error": "UNSUPPORTED_SCHEMA_VERSION"}, status=400)
+    expected_node_id = compute_node_id(b58decode(data["public_key"]))
+    if data.get("node_id") != expected_node_id:
+        return response.json({"error": "NODE_ID_MISMATCH"}, status=400)