relayers new code

app/api/__init__.py

@@ -15,16 +15,28 @@ app.register_middleware(close_db_session, "response")
 
 from app.api.routes._index import s_index, s_favicon
 from app.api.routes._system import s_api_v1_node, s_api_system_version, s_api_system_send_status, s_api_v1_node_friendly
+from app.api.routes.network import (
+    s_api_v1_network_info,
+    s_api_v1_network_nodes,
+    s_api_v1_network_handshake,
+)
 from app.api.routes.auth import s_api_v1_auth_twa, s_api_v1_auth_select_wallet, s_api_v1_auth_me
 from app.api.routes.statics import s_api_tonconnect_manifest, s_api_platform_metadata
 from app.api.routes.node_storage import s_api_v1_storage_post, s_api_v1_storage_get, \
     s_api_v1_storage_decode_cid
 from app.api.routes.progressive_storage import s_api_v1_5_storage_get, s_api_v1_5_storage_post
+from app.api.routes.upload_tus import s_api_v1_upload_tus_hook
 from app.api.routes.account import s_api_v1_account_get
 from app.api.routes._blockchain import s_api_v1_blockchain_send_new_content_message, \
     s_api_v1_blockchain_send_purchase_content_message
 from app.api.routes.content import s_api_v1_content_list, s_api_v1_content_view, s_api_v1_content_friendly_list, s_api_v1_5_content_list
+from app.api.routes.content_index import s_api_v1_content_index, s_api_v1_content_delta
+from app.api.routes.derivatives import s_api_v1_content_derivatives
+from app.api.routes.admin import s_api_v1_admin_node_setrole
 from app.api.routes.tonconnect import s_api_v1_tonconnect_new, s_api_v1_tonconnect_logout
+from app.api.routes.keys import s_api_v1_keys_request
+from app.api.routes.sync import s_api_v1_sync_pin, s_api_v1_sync_status
+from app.api.routes.upload_status import s_api_v1_upload_status
 
 
 app.add_route(s_index, "/", methods=["GET", "OPTIONS"])
@@ -34,6 +46,9 @@ app.add_route(s_api_v1_node, "/api/v1/node", methods=["GET", "OPTIONS"])
 app.add_route(s_api_v1_node_friendly, "/api/v1/nodeFriendly", methods=["GET", "OPTIONS"])
 app.add_route(s_api_system_version, "/api/system.version", methods=["GET", "OPTIONS"])
 app.add_route(s_api_system_send_status, "/api/system.sendStatus", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_network_info, "/api/v1/network.info", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_network_nodes, "/api/v1/network.nodes", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_network_handshake, "/api/v1/network.handshake", methods=["POST", "OPTIONS"])
 
 app.add_route(s_api_tonconnect_manifest, "/api/tonconnect-manifest.json", methods=["GET", "OPTIONS"])
 app.add_route(s_api_platform_metadata, "/api/platform-metadata.json", methods=["GET", "OPTIONS"])
@@ -61,6 +76,19 @@ app.add_route(s_api_v1_content_list, "/api/v1/content.list", methods=["GET", "OP
 app.add_route(s_api_v1_content_view, "/api/v1/content.view/<content_address>", methods=["GET", "OPTIONS"])
 app.add_route(s_api_v1_content_friendly_list, "/api/v1/content.friendlyList", methods=["GET", "OPTIONS"])
 app.add_route(s_api_v1_5_content_list, "/api/v1.5/content.list", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_content_index, "/api/v1/content.index", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_content_delta, "/api/v1/content.delta", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_content_derivatives, "/api/v1/content.derivatives", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_node_setrole, "/api/v1/admin.node.setRole", methods=["POST", "OPTIONS"])
+
+# tusd HTTP hooks
+app.add_route(s_api_v1_upload_tus_hook, "/api/v1/upload.tus-hook", methods=["POST", "OPTIONS"])
+
+# Keys auto-grant
+app.add_route(s_api_v1_keys_request, "/api/v1/keys.request", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_sync_pin, "/api/v1/sync.pin", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_sync_status, "/api/v1/sync.status", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_upload_status, "/api/v1/upload.status/<upload_id>", methods=["GET", "OPTIONS"])
 
 
 @app.exception(BaseException)

app/api/routes/_blockchain.py

@@ -46,7 +46,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
         for field_key, field_value in {
             'title': lambda x: isinstance(x, str),
             'authors': lambda x: isinstance(x, list),
-            'content': lambda x: isinstance(x, str),
+            'content': lambda x: isinstance(x, str),  # may be plaintext CID (legacy) or encrypted IPFS CID (bafy...)
             'image': lambda x: isinstance(x, str),
             'description': lambda x: isinstance(x, str),
             'price': lambda x: (isinstance(x, str) and x.isdigit()),
@@ -57,19 +57,27 @@ async def s_api_v1_blockchain_send_new_content_message(request):
             assert field_key in request.json, f"No {field_key} provided"
             assert field_value(request.json[field_key]), f"Invalid {field_key} provided"
 
-        decrypted_content_cid, err = resolve_content(request.json['content'])
-        assert not err, f"Invalid content CID"
-
-        # Поиск исходного файла загруженного
-        decrypted_content = (await request.ctx.db_session.execute(
-            select(StoredContent).where(StoredContent.hash == decrypted_content_cid.content_hash_b58)
-        )).scalars().first()
-        assert decrypted_content, "No content locally found"
-        assert decrypted_content.type == "local/content_bin", "Invalid content type"
-
-        # Создание фиктивного encrypted_content. Не шифруем для производительности, тк зашифрованная нигде дальше не используется
-        encrypted_content = await create_encrypted_content(request.ctx.db_session, decrypted_content)
-        encrypted_content_cid = encrypted_content.cid
+        # Support legacy: 'content' as decrypted ContentId; and new: 'content' as encrypted IPFS CID
+        encrypted_content_cid = None
+        try:
+            # Legacy path
+            decrypted_content_cid, err = resolve_content(request.json['content'])
+            assert not err
+            decrypted_content = (await request.ctx.db_session.execute(
+                select(StoredContent).where(StoredContent.hash == decrypted_content_cid.content_hash_b58)
+            )).scalars().first()
+            assert decrypted_content and decrypted_content.type == "local/content_bin"
+            encrypted_content = await create_encrypted_content(request.ctx.db_session, decrypted_content)
+            encrypted_content_cid = encrypted_content.cid
+        except BaseException:
+            # New path: treat provided string as encrypted IPFS CID (ENCF v1)
+            encrypted_ipfs_cid = request.json['content']
+            class _EC:  # tiny adapter to mimic .serialize_v2()
+                def __init__(self, s: str):
+                    self._s = s
+                def serialize_v2(self, include_accept_type: bool = False):
+                    return self._s
+            encrypted_content_cid = _EC(encrypted_ipfs_cid)
 
         if request.json['image']:
             image_content_cid, err = resolve_content(request.json['image'])
@@ -94,6 +102,19 @@ async def s_api_v1_blockchain_send_new_content_message(request):
             downloadable=request.json['downloadable'] if 'downloadable' in request.json else False,
         )
 
+        # Try to update ContentIndexItem with cover_url for this encrypted content
+        try:
+            from app.core.models.content_v3 import ContentIndexItem
+            ecid_str = encrypted_content_cid.serialize_v2()
+            row = (await request.ctx.db_session.execute(select(ContentIndexItem).where(ContentIndexItem.encrypted_cid == ecid_str))).scalars().first()
+            if row:
+                payload = row.payload or {}
+                payload['cover_url'] = f"{PROJECT_HOST}/api/v1.5/storage/{image_content_cid.serialize_v2()}" if image_content_cid else None
+                row.payload = payload
+                await request.ctx.db_session.commit()
+        except Exception as _e:
+            make_log("Blockchain", f"index cover update failed: {_e}", level='warning')
+
         royalties_dict = begin_dict(8)
         i = 0
         for royalty_param in request.json['royaltyParams']:

app/api/routes/admin.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import os
+from sanic import response
+from sqlalchemy import select
+
+from app.core.models.my_network import KnownNode
+
+
+def _auth_ok(request) -> bool:
+    token = os.getenv('ADMIN_API_TOKEN')
+    if not token:
+        return False
+    auth = request.headers.get('Authorization', '')
+    return auth.strip() == f"Bearer {token}"
+
+
+async def s_api_v1_admin_node_setrole(request):
+    if not _auth_ok(request):
+        return response.json({"error": "UNAUTHORIZED"}, status=401)
+    data = request.json or {}
+    role = (data.get('role') or '').strip()
+    if role not in ('trusted', 'read-only', 'deny'):
+        return response.json({"error": "BAD_ROLE"}, status=400)
+    pub = (data.get('public_key') or '').strip()
+    host = (data.get('host') or '').strip()
+    if not pub and not host:
+        return response.json({"error": "MISSING_TARGET"}, status=400)
+    session = request.ctx.db_session
+    row = None
+    if pub:
+        row = (await session.execute(select(KnownNode).where(KnownNode.public_key == pub))).scalars().first()
+    if not row and host:
+        row = (await session.execute(select(KnownNode).where(KnownNode.ip == host))).scalars().first()
+    if not row:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    meta = row.meta or {}
+    meta['role'] = role
+    row.meta = meta
+    await session.commit()
+    return response.json({"ok": True, "node": {"ip": row.ip, "public_key": row.public_key, "role": role}})
+

app/api/routes/content.py

@@ -10,6 +10,7 @@ from app.core.models.keys import KnownKey
 from app.core.models import StarsInvoice
 from app.core.models.content.user_content import UserContent
 from app.core._config import CLIENT_TELEGRAM_API_KEY, PROJECT_HOST
+from app.core.models.content_v3 import EncryptedContent as ECv3, ContentDerivative as CDv3
 import json
 import uuid
 
@@ -167,7 +168,7 @@ async def s_api_v1_content_view(request, content_address: str):
     if converted_content:
         user_content_option = 'low_preview'
         if have_access:
-            user_content_option = 'low'  # TODO: подключать high если человек внезапно меломан
+            user_content_option = 'low'
 
         converted_content = (await request.ctx.db_session.execute(select(StoredContent).where(
             StoredContent.hash == converted_content[user_content_option]
@@ -175,6 +176,24 @@ async def s_api_v1_content_view(request, content_address: str):
         if converted_content:
             display_options['content_url'] = converted_content.web_url
             opts['content_ext'] = converted_content.filename.split('.')[-1]
+    else:
+        # v3 fallback: use derivatives table linked via encrypted_cid from onchain meta
+        enc_cid = content['encrypted_content'].meta.get('content_cid') or content['encrypted_content'].meta.get('encrypted_cid')
+        if enc_cid:
+            ec = (await request.ctx.db_session.execute(select(ECv3).where(ECv3.encrypted_cid == enc_cid))).scalars().first()
+            if ec:
+                # choose preview for non-access; low for access
+                desired = ['decrypted_preview'] if not have_access else ['decrypted_low', 'decrypted_high']
+                rows = (await request.ctx.db_session.execute(select(CDv3).where(CDv3.content_id == ec.id, CDv3.status == 'ready'))).scalars().all()
+                chosen = None
+                for kind in desired:
+                    chosen = next((r for r in rows if r.kind == kind), None)
+                    if chosen:
+                        break
+                if chosen and chosen.local_path:
+                    h = chosen.local_path.split('/')[-1]
+                    display_options['content_url'] = f"{PROJECT_HOST}/api/v1.5/storage/{h}"
+                    opts['content_ext'] = (chosen.content_type or '').split('/')[-1] if chosen.content_type else None
 
     content_meta = content['encrypted_content'].json_format()
     from app.core.content.content_id import ContentId

app/api/routes/content_index.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from sanic import response
+from sqlalchemy import select
+from datetime import datetime
+
+from app.core.models.content_v3 import ContentIndexItem
+from app.core.logger import make_log
+
+
+async def s_api_v1_content_index(request):
+    rows = (await request.ctx.db_session.execute(select(ContentIndexItem))).scalars().all()
+    items = [{**r.payload, "encrypted_cid": r.encrypted_cid, "sig": r.sig, "_updated_at": (r.updated_at.isoformat() + 'Z') if r.updated_at else None} for r in rows]
+    # ETag by max updated_at + count
+    max_ts = max((it.get("_updated_at") for it in items if it.get("_updated_at")), default="1970-01-01T00:00:00Z")
+    etag = f'W/"{max_ts}.{len(items)}"'
+    inm = request.headers.get('If-None-Match')
+    if inm and inm == etag:
+        resp = response.empty(status=304)
+        resp.headers['ETag'] = etag
+        return resp
+    for it in items:
+        it.pop("_updated_at", None)
+    make_log("content.index", f"items={len(items)} etag={etag}")
+    resp = response.json({"items": items, "schema": "my-network/index@1"})
+    resp.headers['ETag'] = etag
+    return resp
+
+
+async def s_api_v1_content_delta(request):
+    since = request.args.get('since')
+    if not since:
+        # No since provided → act as full index
+        return await s_api_v1_content_index(request)
+    try:
+        # basic parse
+        _ = datetime.fromisoformat(since.replace('Z', '+00:00'))
+    except Exception:
+        return response.json({"error": "BAD_SINCE"}, status=400)
+
+    rows = (await request.ctx.db_session.execute(select(ContentIndexItem))).scalars().all()
+    out = []
+    max_ts = since
+    for r in rows:
+        upd = (r.updated_at.isoformat() + 'Z') if r.updated_at else None
+        if upd and upd > since:
+            out.append({**r.payload, "encrypted_cid": r.encrypted_cid, "sig": r.sig})
+            if upd > max_ts:
+                max_ts = upd
+    resp = response.json({"items": out, "next_since": max_ts, "schema": "my-network/index@1"})
+    # Weak ETag for delta response
+    resp.headers['ETag'] = f'W/"{max_ts}.{len(out)}"'
+    return resp

app/api/routes/derivatives.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from sanic import response
+from sqlalchemy import select
+
+from app.core.models.content_v3 import EncryptedContent, ContentDerivative
+from app.core._config import PROJECT_HOST
+
+
+async def s_api_v1_content_derivatives(request):
+    cid = request.args.get('cid')
+    if not cid:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+    session = request.ctx.db_session
+    ec = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+    if not ec:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id))).scalars().all()
+    out = []
+    for r in rows:
+        # Derive /api/v1.5/storage/<hash> from local_path if possible
+        path_hash = (r.local_path or '').split('/')[-1]
+        storage_url = f"{PROJECT_HOST}/api/v1.5/storage/{path_hash}" if path_hash else None
+        out.append({
+            'kind': r.kind,
+            'interval': [r.interval_start_ms, r.interval_end_ms] if r.interval_start_ms is not None else None,
+            'content_type': r.content_type,
+            'size_bytes': r.size_bytes,
+            'status': r.status,
+            'url': storage_url,
+        })
+    return response.json({'cid': cid, 'derivatives': out})
+

app/api/routes/keys.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import base64
+import json
+from datetime import datetime
+from typing import Dict, Any
+
+from base58 import b58encode
+from sanic import response
+from sqlalchemy import select
+
+from app.core._secrets import hot_pubkey
+from app.core.logger import make_log
+from app.core.models.content_v3 import EncryptedContent, ContentKey, KeyGrant
+from app.core.network.nodesig import verify_request
+from app.core.network.guard import check_rate_limit
+from app.core.models.my_network import KnownNode
+
+
+def _b64(b: bytes) -> str:
+    return base64.b64encode(b).decode()
+
+
+async def s_api_v1_keys_request(request):
+    # Rate limit per remote IP (reuse handshake limiter)
+    remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip()
+    if not check_rate_limit(request.app.ctx.memory, remote_ip):
+        return response.json({"error": "RATE_LIMIT"}, status=429)
+
+    # Verify NodeSig
+    ok, hdr_node, reason = verify_request(request, request.app.ctx.memory)
+    if not ok:
+        return response.json({"error": reason or "UNAUTHORIZED"}, status=401)
+
+    data: Dict[str, Any] = request.json or {}
+    cid = data.get("encrypted_cid")
+    requester_node = data.get("requestor_node_id")
+    recipient_box_pub_b64 = data.get("recipient_box_pub")
+    if not cid or not requester_node or not recipient_box_pub_b64:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+
+    if requester_node != hdr_node:
+        return response.json({"error": "NODE_ID_MISMATCH"}, status=401)
+
+    session = request.ctx.db_session
+    row = (await session.execute(select(EncryptedContent, ContentKey).join(ContentKey, ContentKey.content_id == EncryptedContent.id).where(EncryptedContent.encrypted_cid == cid))).first()
+    if not row:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    ec: EncryptedContent = row[0]
+    ck: ContentKey = row[1]
+    # Allow only trusted nodes unless explicitly disabled via env
+    TRUSTED_ONLY = (os.getenv('KEY_AUTO_GRANT_TRUSTED_ONLY', '1') == '1')
+    if TRUSTED_ONLY:
+        kn = (await session.execute(select(KnownNode).where(KnownNode.public_key == requester_node))).scalars().first()
+        role = (kn.meta or {}).get('role') if kn else None
+        if role != 'trusted':
+            return response.json({"error": "DENIED_NOT_TRUSTED"}, status=403)
+    if not ck.allow_auto_grant:
+        return response.json({"error": "DENIED"}, status=403)
+
+    # Seal the DEK for recipient using libsodium sealed box
+    try:
+        import nacl.public
+        pk = nacl.public.PublicKey(base64.b64decode(recipient_box_pub_b64))
+        box = nacl.public.SealedBox(pk)
+        sealed = box.encrypt(base64.b64decode(ck.key_ciphertext_b64))
+        sealed_b64 = _b64(sealed)
+    except Exception as e:
+        make_log("keys", f"seal failed: {e}", level="error")
+        return response.json({"error": "SEAL_FAILED"}, status=500)
+
+    issuer = b58encode(hot_pubkey).decode()
+    purpose = (data.get('purpose') or 'full')
+    ttl_sec = int(os.getenv('KEY_GRANT_PREVIEW_TTL_SEC', '0')) if purpose == 'preview' else 0
+    grant_body = {
+        "encrypted_cid": cid,
+        "to_node_id": requester_node,
+        "sealed_key_b64": sealed_b64,
+        "aead_scheme": ec.aead_scheme,
+        "chunk_bytes": ec.chunk_bytes,
+        "constraints": {"ttl_sec": ttl_sec, "scope": purpose},
+        "issued_at": datetime.utcnow().isoformat(),
+        "issuer_node_id": issuer,
+    }
+    try:
+        from app.core._crypto.signer import Signer
+        from app.core._secrets import hot_seed
+        signer = Signer(hot_seed)
+        blob = json.dumps(grant_body, sort_keys=True, separators=(",", ":")).encode()
+        sig = signer.sign(blob)
+    except Exception:
+        sig = ""
+
+    grant = KeyGrant(
+        encrypted_cid=cid,
+        issuer_node_id=issuer,
+        to_node_id=requester_node,
+        sealed_key_b64=sealed_b64,
+        aead_scheme=ec.aead_scheme,
+        chunk_bytes=ec.chunk_bytes,
+        constraints={"ttl_sec": 0, "scope": "full"},
+        sig=sig,
+    )
+    session.add(grant)
+    await session.commit()
+    grant_row = {
+        **grant_body,
+        "sig": sig,
+        "grant_id": grant.id,
+    }
+    return response.json(grant_row)

app/api/routes/network.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from typing import Dict, Any
+
+from base58 import b58decode
+from sanic import response
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.models.my_network import KnownNode
+from app.core.network.constants import CURRENT_PROTOCOL_VERSION, NODE_TYPE_PRIVATE
+from app.core.network.config import NODE_PRIVACY
+from app.core.network.handshake import build_handshake_payload, compute_node_info, sign_response
+from app.core.network.nodes import upsert_known_node, list_known_public_nodes
+from app.core.network.semver import compatibility
+from app.core.network.guard import check_rate_limit, check_timestamp_fresh, check_and_remember_nonce
+from app.core.network.config import HANDSHAKE_TS_TOLERANCE_SEC
+
+
+async def s_api_v1_network_info(request):
+    async with request.app.ctx.memory.transaction("network.info"):
+        node = await compute_node_info(request.ctx.db_session)
+    make_log("Network", "info served")
+    return response.json({"node": node})
+
+
+async def s_api_v1_network_nodes(request):
+    rows = await list_known_public_nodes(request.ctx.db_session)
+    make_log("Network", f"nodes list count={len(rows)}")
+    return response.json({
+        "count": len(rows),
+        "nodes": rows,
+    })
+
+
+async def s_api_v1_network_handshake(request):
+    # Handshake accepted regardless of our privacy; private nodes typically have no external endpoint
+
+    # Rate limit per remote IP
+    remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip()
+    if not check_rate_limit(request.app.ctx.memory, remote_ip):
+        return response.json({"error": "RATE_LIMIT"}, status=429)
+
+    data = request.json or {}
+    required = ["version", "public_key", "node_type", "metrics", "timestamp", "signature"]
+    for f in required:
+        if f not in data:
+            return response.json({"error": f"Missing field {f}"}, status=400)
+    # public_host is required for public nodes only
+    if data.get("node_type") != "private" and not data.get("public_host"):
+        return response.json({"error": "Missing field public_host"}, status=400)
+
+    # Timestamp freshness
+    if not check_timestamp_fresh(data.get("timestamp")):
+        return response.json({"error": "STALE_TIMESTAMP", "tolerance_sec": HANDSHAKE_TS_TOLERANCE_SEC}, status=400)
+
+    # Nonce replay protection (best-effort)
+    if not data.get("nonce") or not check_and_remember_nonce(request.app.ctx.memory, data.get("public_key"), data.get("nonce")):
+        return response.json({"error": "NONCE_REPLAY"}, status=400)
+
+    peer_version = str(data.get("version"))
+    comp = compatibility(peer_version, CURRENT_PROTOCOL_VERSION)
+    if comp == "blocked":
+        # We still store the node but respond with 409
+        try:
+            await upsert_known_node(
+                request.ctx.db_session,
+                host=data.get("public_host"),
+                port=int(str(data.get("public_host") or "").split(":")[-1]) if ":" in str(data.get("public_host") or "") else 80,
+                public_key=str(data.get("public_key")),
+                meta={
+                    "version": peer_version,
+                    "compatibility": comp,
+                    "is_public": data.get("node_type", "public") != "private",
+                    "public_host": data.get("public_host"),
+                    "unsupported_last_checked_at": datetime.utcnow().isoformat(),
+                }
+            )
+        except Exception:
+            pass
+        make_log("Handshake", f"Reject incompatible peer {data.get('public_host')} peer={peer_version} current={CURRENT_PROTOCOL_VERSION}")
+        return response.json({
+            "error": "INCOMPATIBLE_VERSION",
+            "compatibility": comp,
+            "current": CURRENT_PROTOCOL_VERSION,
+            "peer": peer_version,
+        }, status=409)
+
+    # Verify signature
+    try:
+        # Verify signature over the entire payload except the signature itself
+        signed_fields = {k: v for (k, v) in data.items() if k != "signature"}
+        blob = json.dumps(signed_fields, sort_keys=True, separators=(",", ":")).encode()
+        import nacl.signing, nacl.encoding
+        vk = nacl.signing.VerifyKey(b58decode(data["public_key"]))
+        sig = b58decode(data["signature"])
+        vk.verify(blob, sig)
+        ok = True
+    except Exception:
+        ok = False
+    if not ok:
+        make_log("Handshake", f"Signature verification failed from {data.get('public_host')}", level='warning')
+        return response.json({"error": "BAD_SIGNATURE"}, status=400)
+
+    # Upsert node and respond with our info + known public nodes
+    # Do not persist private peers (ephemeral)
+    if data.get("node_type") != "private" and data.get("public_host"):
+        try:
+            await upsert_known_node(
+                request.ctx.db_session,
+                host=data.get("public_host"),
+                port=int(str(data.get("public_host") or "").split(":")[-1]) if ":" in str(data.get("public_host") or "") else 80,
+                public_key=str(data.get("public_key")),
+                meta={
+                    "version": peer_version,
+                    "compatibility": comp,
+                    "is_public": True,
+                    "public_host": data.get("public_host"),
+                    "last_metrics": data.get("metrics", {}),
+                    "capabilities": data.get("capabilities", {}),
+                }
+            )
+        except Exception as e:
+            make_log("Handshake", f"Upsert peer failed: {e}", level='warning')
+
+    # Merge advertised peers from the caller (optional field)
+    for n in data.get("known_public_nodes", []) or []:
+        try:
+            await upsert_known_node(
+                request.ctx.db_session,
+                host=n.get("public_host") or n.get("host"),
+                port=int(n.get("port") or 80),
+                public_key=n.get("public_key") or "",
+                meta={
+                    "version": n.get("version") or "0.0.0",
+                    "compatibility": compatibility(n.get("version") or "0.0.0", CURRENT_PROTOCOL_VERSION),
+                    "is_public": True,
+                    "public_host": n.get("public_host") or n.get("host"),
+                    "capabilities": n.get("capabilities") or {},
+                }
+            )
+        except Exception:
+            pass
+
+    node = await compute_node_info(request.ctx.db_session)
+    known = await list_known_public_nodes(request.ctx.db_session)
+    resp = sign_response({
+        "compatibility": comp,
+        "node": node,
+        "known_public_nodes": known,
+    })
+    make_log("Handshake", f"OK with {data.get('public_host')} compat={comp}")
+    status = 200
+    if comp == "warning":
+        status = 200
+        resp["warning"] = "MINOR version differs; proceed with caution"
+    return response.json(resp, status=status)

app/api/routes/progressive_storage.py

@@ -14,6 +14,7 @@ from app.core.logger import make_log
 from sqlalchemy import select
 from app.core.models.node_storage import StoredContent
 from app.core._config import UPLOADS_DIR
+from app.core.models.content_v3 import ContentDerivative
 from app.core._utils.resolve_content import resolve_content
 
 
@@ -205,6 +206,15 @@ async def s_api_v1_5_storage_get(request, file_hash):
     file_size = os.path.getsize(final_path)
     range_header = request.headers.get("Range")
     
+    # touch derivative last_access_at if exists
+    try:
+        cd = (await request.ctx.db_session.execute(select(ContentDerivative).where(ContentDerivative.local_path.like(f"%/{file_hash}")))).scalars().first()
+        if cd:
+            cd.last_access_at = datetime.utcnow()
+            await request.ctx.db_session.commit()
+    except Exception:
+        pass
+
     if range_header:
         make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Processing Range: {range_header}", level="DEBUG")
         range_spec = range_header.strip().lower()

app/api/routes/sync.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from datetime import datetime
+from sanic import response
+from sqlalchemy import select
+
+from app.core.ipfs_client import pin_add, pin_ls
+from app.core.logger import make_log
+from app.core.models.content_v3 import EncryptedContent, IpfsSync
+from app.core.network.nodesig import verify_request
+from app.core.network.guard import check_rate_limit
+
+
+async def s_api_v1_sync_pin(request):
+    # Rate limit per IP and require NodeSig for POST
+    remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip()
+    if not check_rate_limit(request.app.ctx.memory, remote_ip):
+        return response.json({"error": "RATE_LIMIT"}, status=429)
+
+    ok, node_id, reason = verify_request(request, request.app.ctx.memory)
+    if not ok:
+        return response.json({"error": reason or "UNAUTHORIZED"}, status=401)
+
+    data = request.json or {}
+    cid = data.get("encrypted_cid")
+    if not cid:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+
+    session = request.ctx.db_session
+    row = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+    if not row:
+        # create record with minimal info (unknown meta)
+        row = EncryptedContent(
+            encrypted_cid=cid,
+            title=cid,
+            description="",
+            content_type="application/octet-stream",
+            preview_enabled=False,
+        )
+        session.add(row)
+        await session.flush()
+    sync = (await session.execute(select(IpfsSync).where(IpfsSync.content_id == row.id))).scalars().first()
+    if not sync:
+        sync = IpfsSync(content_id=row.id, pin_state='queued')
+        session.add(sync)
+        await session.flush()
+
+    try:
+        await pin_add(cid, recursive=True)
+        sync.pin_state = 'pinned'
+        sync.pinned_at = datetime.utcnow()
+    except Exception as e:
+        make_log("sync", f"pin failed: {e}", level="error")
+        sync.pin_state = 'failed'
+        sync.pin_error = str(e)
+    await session.commit()
+    return response.json({"ok": True, "state": sync.pin_state})
+
+
+async def s_api_v1_sync_status(request):
+    cid = request.args.get("cid")
+    if not cid:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+    try:
+        info = await pin_ls(cid)
+        state = 'pinned' if info else 'not_pinned'
+    except Exception:
+        state = 'not_pinned'
+        info = {}
+    return response.json({"cid": cid, "state": state, "info": info})

app/api/routes/upload_status.py

@@ -0,0 +1,17 @@
+from sanic import response
+from app.core.models.content_v3 import UploadSession
+
+
+async def s_api_v1_upload_status(request, upload_id: str):
+    session = request.ctx.db_session
+    row = await session.get(UploadSession, upload_id)
+    if not row:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    return response.json({
+        "id": row.id,
+        "state": row.state,
+        "encrypted_cid": row.encrypted_cid,
+        "size_bytes": row.size_bytes,
+        "error": row.error,
+    })
+

app/api/routes/upload_tus.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import base64
+import json
+import os
+from datetime import datetime
+from typing import Dict, Any
+
+from base58 import b58encode
+from sanic import response
+
+from app.core._secrets import hot_pubkey
+from app.core.crypto.aes_gcm_siv_stream import encrypt_file_to_encf
+from app.core.crypto.aesgcm_stream import CHUNK_BYTES
+from app.core.ipfs_client import add_streamed_file
+from app.core.logger import make_log
+from app.core.models.content_v3 import EncryptedContent, ContentKey, IpfsSync, ContentIndexItem, UploadSession
+from app.core.storage import db_session
+
+
+def _b64(s: bytes) -> str:
+    return base64.b64encode(s).decode()
+
+
+async def s_api_v1_upload_tus_hook(request):
+    """
+    tusd HTTP hook endpoint. We mainly handle post-finish to: encrypt -> IPFS add+pin -> record DB.
+    """
+    try:
+        payload: Dict[str, Any] = request.json or {}
+    except Exception:
+        payload = {}
+    event = payload.get("Type") or payload.get("type") or payload.get("Event") or payload.get("event")
+    upload = payload.get("Upload") or payload.get("upload") or {}
+
+    if not event:
+        return response.json({"ok": False, "error": "NO_EVENT"}, status=400)
+
+    if event not in ("post-finish", "postfinish"):
+        # accept but ignore other events
+        return response.json({"ok": True})
+
+    # Extract storage path from tusd payload
+    storage = upload.get("Storage") or {}
+    file_path = storage.get("Path") or storage.get("path")
+    if not file_path:
+        return response.json({"ok": False, "error": "NO_STORAGE_PATH"}, status=400)
+
+    meta = upload.get("MetaData") or {}
+    # Common metadata keys
+    title = meta.get("title") or meta.get("Title") or meta.get("name") or "Untitled"
+    description = meta.get("description") or meta.get("Description") or ""
+    content_type = meta.get("content_type") or meta.get("Content-Type") or "application/octet-stream"
+    preview_enabled = content_type.startswith("audio/") or content_type.startswith("video/")
+    # Optional preview window overrides from tus metadata
+    try:
+        start_ms = int(meta.get("preview_start_ms") or 0)
+        dur_ms = int(meta.get("preview_duration_ms") or 30000)
+    except Exception:
+        start_ms, dur_ms = 0, 30000
+
+    # Record/Update upload session
+    upload_id = upload.get("ID") or upload.get("Id") or upload.get("id")
+    try:
+        size = int(upload.get("Size") or 0)
+    except Exception:
+        size = None
+
+    async with db_session() as session:
+        us = (await session.get(UploadSession, upload_id)) if upload_id else None
+        if not us and upload_id:
+            us = UploadSession(
+                id=upload_id,
+                filename=os.path.basename(file_path),
+                size_bytes=size,
+                state='processing',
+                encrypted_cid=None,
+            )
+            session.add(us)
+            await session.commit()
+
+    # Read & encrypt by streaming (ENCF v1 / AES-SIV)
+    # Generate per-content random DEK and salt
+    dek = os.urandom(32)
+    salt = os.urandom(16)
+    key_fpr = b58encode(hot_pubkey).decode()  # fingerprint as our node id for now
+
+    # Stream encrypt into IPFS add
+    try:
+        with open(file_path, 'rb') as f:
+            result = await add_streamed_file(
+                encrypt_file_to_encf(f, dek, CHUNK_BYTES, salt),
+                filename=os.path.basename(file_path),
+                params={},
+            )
+    except Exception as e:
+        make_log("tus-hook", f"Encrypt+add failed: {e}", level="error")
+        # mark failed
+        async with db_session() as session:
+            if upload_id:
+                us = await session.get(UploadSession, upload_id)
+                if us:
+                    us.state = 'failed'
+                    us.error = str(e)
+                    await session.commit()
+        return response.json({"ok": False, "error": "ENCRYPT_ADD_FAILED"}, status=500)
+
+    encrypted_cid = result.get("Hash")
+    try:
+        enc_size = int(result.get("Size") or 0)
+    except Exception:
+        enc_size = None
+
+    # Persist records
+    async with db_session() as session:
+        ec = EncryptedContent(
+            encrypted_cid=encrypted_cid,
+            title=title,
+            description=description,
+            content_type=content_type,
+            enc_size_bytes=enc_size,
+            plain_size_bytes=os.path.getsize(file_path),
+            preview_enabled=preview_enabled,
+            preview_conf=({"duration_ms": dur_ms, "intervals": [[start_ms, start_ms + dur_ms]]} if preview_enabled else {}),
+            aead_scheme="AES_GCM_SIV",
+            chunk_bytes=CHUNK_BYTES,
+            salt_b64=_b64(salt),
+        )
+        session.add(ec)
+        await session.flush()
+
+        ck = ContentKey(
+            content_id=ec.id,
+            key_ciphertext_b64=_b64(dek),  # NOTE: should be wrapped by local KEK; simplified for PoC
+            key_fingerprint=key_fpr,
+            issuer_node_id=key_fpr,
+            allow_auto_grant=True,
+        )
+        session.add(ck)
+
+        sync = IpfsSync(
+            content_id=ec.id,
+            pin_state='pinned',
+            bytes_total=enc_size,
+            bytes_fetched=enc_size,
+            pinned_at=datetime.utcnow(),
+        )
+        session.add(sync)
+
+        # Publish signed index item
+        item = {
+            "encrypted_cid": encrypted_cid,
+            "title": title,
+            "description": description,
+            "content_type": content_type,
+            "size_bytes": enc_size,
+            "preview_enabled": preview_enabled,
+            "preview_conf": ec.preview_conf,
+            "issuer_node_id": key_fpr,
+            "salt_b64": _b64(salt),
+        }
+        try:
+            from app.core._crypto.signer import Signer
+            from app.core._secrets import hot_seed
+            signer = Signer(hot_seed)
+            blob = json.dumps(item, sort_keys=True, separators=(",", ":")).encode()
+            sig = signer.sign(blob)
+        except Exception:
+            sig = ""
+        session.add(ContentIndexItem(encrypted_cid=encrypted_cid, payload=item, sig=sig))
+
+        await session.commit()
+
+    # Update upload session with result and purge staging to avoid duplicates
+    async with db_session() as session:
+        if upload_id:
+            us = await session.get(UploadSession, upload_id)
+            if us:
+                us.state = 'pinned'
+                us.encrypted_cid = encrypted_cid
+                # prefer using IPFS for downstream conversion; remove staging
+                try:
+                    if file_path and os.path.exists(file_path):
+                        os.remove(file_path)
+                except Exception:
+                    pass
+                us.storage_path = None
+                await session.commit()
+
+    make_log("tus-hook", f"Uploaded+encrypted {file_path} -> {encrypted_cid}")
+    return response.json({"ok": True, "encrypted_cid": encrypted_cid, "upload_id": upload_id})