relayers new code

app/core/background/convert_v3_service.py

@@ -0,0 +1,307 @@
+import asyncio
+import os
+import json
+import shutil
+from datetime import datetime
+from typing import List, Tuple
+
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core._config import UPLOADS_DIR, BACKEND_LOGS_DIR_HOST
+from app.core.models.content_v3 import (
+    EncryptedContent,
+    ContentKey,
+    ContentDerivative,
+    UploadSession,
+)
+from app.core.models.node_storage import StoredContent
+from app.core.ipfs_client import cat_stream
+from app.core.crypto.aesgcm_stream import CHUNK_BYTES
+from app.core.crypto.encf_stream import decrypt_encf_auto
+from app.core.network.key_client import request_key_from_peer
+from app.core.models.my_network import KnownNode
+
+
+CONCURRENCY = int(os.getenv("CONVERT_V3_MAX_CONCURRENCY", "3"))
+
+
+def _ensure_dir(path: str):
+    try:
+        os.makedirs(path, exist_ok=True)
+    except Exception:
+        pass
+
+
+async def _sha256_b58(file_path: str) -> str:
+    import hashlib
+    import base58
+    h = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(2 * 1024 * 1024), b''):
+            h.update(chunk)
+    return base58.b58encode(h.digest()).decode()
+
+
+async def _save_derivative(file_path: str, filename: str) -> Tuple[str, int]:
+    """Move file into UPLOADS_DIR under sha256 b58 name; return (hash_b58, size)."""
+    file_hash = await _sha256_b58(file_path)
+    dst = os.path.join(UPLOADS_DIR, file_hash)
+    try:
+        os.remove(dst)
+    except FileNotFoundError:
+        pass
+    shutil.move(file_path, dst)
+    size = os.path.getsize(dst)
+    return file_hash, size
+
+
+async def _run_media_converter(input_host_path: str, input_ext: str, quality: str, trim_value: str | None, is_audio: bool) -> Tuple[str, dict]:
+    rid = __import__('uuid').uuid4().hex[:8]
+    output_dir_container = f"/tmp/conv_{rid}"
+    output_dir_host = f"/tmp/conv_{rid}"
+    _ensure_dir(output_dir_host)
+    logs_dir_host = BACKEND_LOGS_DIR_HOST
+    _ensure_dir(logs_dir_host)
+
+    cmd = [
+        "docker", "run", "--rm",
+        "-v", f"{input_host_path}:/app/input:ro",
+        "-v", f"{output_dir_host}:/app/output",
+        "-v", f"{logs_dir_host}:/app/logs",
+        "media_converter",
+        "--ext", input_ext,
+        "--quality", quality,
+    ]
+    if trim_value:
+        cmd.extend(["--trim", trim_value])
+    if is_audio:
+        cmd.append("--audio-only")
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        raise RuntimeError(f"media_converter failed: {stderr.decode()}")
+
+    # Find produced media file and optional output.json
+    try:
+        files = os.listdir(output_dir_host)
+    except Exception as e:
+        raise RuntimeError(f"Read output dir error: {e}")
+    media_files = [f for f in files if f != "output.json"]
+    if len(media_files) != 1:
+        raise RuntimeError(f"Expected one media file, found {len(media_files)}: {media_files}")
+    output_media = os.path.join(output_dir_host, media_files[0])
+    ffprobe_meta = {}
+    out_json = os.path.join(output_dir_host, "output.json")
+    if os.path.exists(out_json):
+        try:
+            with open(out_json, 'r') as f:
+                ffprobe_meta = json.load(f)
+        except Exception:
+            ffprobe_meta = {}
+    return output_media, ffprobe_meta
+
+
+async def _convert_content(ec: EncryptedContent, input_host_path: str):
+    content_kind = 'audio' if ec.content_type.startswith('audio/') else ('video' if ec.content_type.startswith('video/') else 'other')
+    if content_kind == 'other':
+        return
+
+    input_ext = (ec.content_type.split('/')[-1] or 'bin')
+    is_audio = content_kind == 'audio'
+    # Required outputs
+    required = ['high', 'low', 'low_preview']
+
+    # Preview interval
+    conf = ec.preview_conf or {}
+    intervals = conf.get('intervals') or [[0, int(conf.get('duration_ms', 30000))]]
+    main_interval = intervals[0]
+    trim_value = None
+    start_s = max(0, int(main_interval[0]) // 1000)
+    dur_s = max(1, int((main_interval[1] - main_interval[0]) // 1000) or 30)
+    trim_value = f"{start_s},{dur_s}"
+
+    qualities = {
+        'high': 'high',
+        'low': 'low',
+        'low_preview': 'low',
+    }
+
+    for opt in required:
+        try:
+            # Mark derivative processing
+            async with db_session() as session:
+                cd = ContentDerivative(
+                    content_id=ec.id,
+                    kind=f"decrypted_{opt if opt != 'low_preview' else 'preview'}",
+                    interval_start_ms=main_interval[0] if opt == 'low_preview' else None,
+                    interval_end_ms=main_interval[1] if opt == 'low_preview' else None,
+                    local_path="",
+                    status='processing',
+                )
+                session.add(cd)
+                await session.commit()
+
+            out_path, ffprobe = await _run_media_converter(
+                input_host_path=input_host_path,
+                input_ext=input_ext,
+                quality=qualities[opt],
+                trim_value=trim_value if opt == 'low_preview' else None,
+                is_audio=is_audio,
+            )
+
+            # Save into store and StoredContent
+            file_hash, size_bytes = await _save_derivative(out_path, os.path.basename(out_path))
+
+            async with db_session() as session:
+                sc = StoredContent(
+                    type="local/content_bin",
+                    hash=file_hash,
+                    user_id=None,
+                    filename=os.path.basename(out_path),
+                    meta={'encrypted_cid': ec.encrypted_cid, 'kind': opt, 'ffprobe_meta': ffprobe},
+                    created=datetime.utcnow(),
+                )
+                session.add(sc)
+                await session.flush()
+
+                # Update derivative record
+                cd = (await session.execute(select(ContentDerivative).where(
+                    ContentDerivative.content_id == ec.id,
+                    ContentDerivative.kind == (f"decrypted_{opt if opt != 'low_preview' else 'preview'}"),
+                    ContentDerivative.status == 'processing'
+                ))).scalars().first()
+                if cd:
+                    cd.local_path = os.path.join(UPLOADS_DIR, file_hash)
+                    cd.size_bytes = size_bytes
+                    cd.content_type = ('audio/mpeg' if is_audio else 'video/mp4') if opt != 'high' else ec.content_type
+                    cd.status = 'ready'
+                await session.commit()
+
+            make_log('convert_v3', f"Converted {ec.encrypted_cid} opt={opt} -> {file_hash}")
+        except Exception as e:
+            make_log('convert_v3', f"Convert error {ec.encrypted_cid} opt={opt}: {e}", level='error')
+            async with db_session() as session:
+                cd = ContentDerivative(
+                    content_id=ec.id,
+                    kind=f"decrypted_{opt if opt != 'low_preview' else 'preview'}",
+                    status='failed',
+                    error=str(e),
+                    local_path="",
+                )
+                session.add(cd)
+                await session.commit()
+
+
+async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, str]]:
+    async with db_session() as session:
+        # Find A/V contents with preview_enabled and no ready low/low_preview derivatives yet
+        ecs = (await session.execute(select(EncryptedContent).where(
+            EncryptedContent.preview_enabled == True
+        ).order_by(EncryptedContent.created_at.desc()))).scalars().all()
+
+        picked: List[Tuple[EncryptedContent, str]] = []
+        for ec in ecs:
+            # Check if derivatives already ready
+            rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id))).scalars().all()
+            kinds_ready = {r.kind for r in rows if r.status == 'ready'}
+            required = {'decrypted_low', 'decrypted_high'} if ec.content_type.startswith('audio/') else {'decrypted_low', 'decrypted_high', 'decrypted_preview'}
+            if required.issubset(kinds_ready):
+                continue
+            # Always decrypt from IPFS using local or remote key
+            storage_path: str | None = None
+            ck = (await session.execute(select(ContentKey).where(ContentKey.content_id == ec.id))).scalars().first()
+            if ck:
+                storage_path = await stage_plain_from_ipfs(ec, ck.key_ciphertext_b64)
+            if not storage_path:
+                peers = (await session.execute(select(KnownNode))).scalars().all()
+                for peer in peers:
+                    base_url = f"http://{peer.ip}:{peer.port}"
+                    dek = await request_key_from_peer(base_url, ec.encrypted_cid)
+                    if not dek:
+                        continue
+                    import base64
+                    dek_b64 = base64.b64encode(dek).decode()
+                    session_ck = ContentKey(
+                        content_id=ec.id,
+                        key_ciphertext_b64=dek_b64,
+                        key_fingerprint=peer.public_key,
+                        issuer_node_id=peer.public_key,
+                        allow_auto_grant=True,
+                    )
+                    session.add(session_ck)
+                    await session.commit()
+                    storage_path = await stage_plain_from_ipfs(ec, dek_b64)
+                    if storage_path:
+                        break
+            if not storage_path or not os.path.exists(storage_path):
+                continue
+            picked.append((ec, storage_path))
+            if len(picked) >= limit:
+                break
+        return picked
+
+
+async def worker_loop():
+    sem = asyncio.Semaphore(CONCURRENCY)
+
+    async def _run_one(ec: EncryptedContent, input_path: str):
+        async with sem:
+            try:
+                await _convert_content(ec, input_path)
+                # After successful conversion, attempt to remove staging file to avoid duplicates
+                try:
+                    if input_path and input_path.startswith("/data/") and os.path.exists(input_path):
+                        os.remove(input_path)
+                except Exception:
+                    pass
+            except Exception as e:
+                make_log('convert_v3', f"job error {ec.encrypted_cid}: {e}", level='error')
+
+    while True:
+        try:
+            batch = await _pick_pending(limit=CONCURRENCY * 2)
+            if not batch:
+                await asyncio.sleep(3)
+                continue
+            tasks = [asyncio.create_task(_run_one(ec, path)) for (ec, path) in batch]
+            await asyncio.gather(*tasks)
+        except Exception as e:
+            make_log('convert_v3', f"loop error: {e}", level='error')
+            await asyncio.sleep(2)
+
+
+async def main_fn(memory):
+    make_log('convert_v3', f"Service started with concurrency={CONCURRENCY}", level='info')
+    await worker_loop()
+
+
+async def stage_plain_from_ipfs(ec: EncryptedContent, dek_b64: str) -> str | None:
+    """Download encrypted ENCF stream from IPFS and decrypt on the fly into a temp file."""
+    import base64, tempfile
+    dek = base64.b64decode(dek_b64)
+    tmp = tempfile.NamedTemporaryFile(prefix=f"dec_{ec.encrypted_cid[:8]}_", delete=False)
+    tmp_path = tmp.name
+    tmp.close()
+    try:
+        async def _aiter():
+            async for ch in cat_stream(ec.encrypted_cid):
+                yield ch
+        await decrypt_encf_auto(_aiter(), dek, tmp_path)
+        return tmp_path
+    except Exception as e:
+        make_log('convert_v3', f"decrypt from ipfs failed: {e}", level='error')
+        try:
+            os.remove(tmp_path)
+        except Exception:
+            pass
+        return None
+
+
+    

app/core/background/derivative_cache_janitor.py

@@ -0,0 +1,87 @@
+import asyncio
+import os
+from datetime import datetime, timedelta
+
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core.models.content_v3 import ContentDerivative
+
+
+MAX_GB = float(os.getenv('DERIVATIVE_CACHE_MAX_GB', '50'))
+TTL_DAYS = int(os.getenv('DERIVATIVE_CACHE_TTL_DAYS', '30'))
+INTERVAL_SEC = int(os.getenv('DERIVATIVE_JANITOR_INTERVAL_SEC', '600'))
+
+
+async def _current_total_size() -> int:
+    async with db_session() as session:
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        return sum(int(r.size_bytes or 0) for r in rows)
+
+
+async def _evict_over_ttl(now: datetime) -> int:
+    removed = 0
+    if TTL_DAYS <= 0:
+        return 0
+    async with db_session() as session:
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        for r in rows:
+            la = r.last_access_at or r.created_at
+            if la and (now - la) > timedelta(days=TTL_DAYS):
+                try:
+                    if r.local_path and os.path.exists(r.local_path):
+                        os.remove(r.local_path)
+                except Exception:
+                    pass
+                r.status = 'pending'
+                r.local_path = None
+                r.size_bytes = None
+                r.last_access_at = None
+                removed += 1
+        await session.commit()
+    return removed
+
+
+async def _evict_to_fit():
+    limit_bytes = int(MAX_GB * (1024 ** 3))
+    total = await _current_total_size()
+    if total <= limit_bytes:
+        return 0
+    to_remove = total - limit_bytes
+    removed = 0
+    async with db_session() as session:
+        # Oldest first by last_access_at
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        rows.sort(key=lambda r: (r.last_access_at or r.created_at or datetime.utcfromtimestamp(0)))
+        for r in rows:
+            if to_remove <= 0:
+                break
+            size = int(r.size_bytes or 0)
+            try:
+                if r.local_path and os.path.exists(r.local_path):
+                    os.remove(r.local_path)
+            except Exception:
+                pass
+            r.status = 'pending'
+            r.local_path = None
+            r.last_access_at = None
+            r.size_bytes = None
+            await session.commit()
+            to_remove -= size
+            removed += 1
+    return removed
+
+
+async def main_fn(memory):
+    make_log('derivative_janitor', f"Started (MAX_GB={MAX_GB}, TTL_DAYS={TTL_DAYS})", level='info')
+    while True:
+        try:
+            now = datetime.utcnow()
+            r1 = await _evict_over_ttl(now)
+            r2 = await _evict_to_fit()
+            if r1 or r2:
+                make_log('derivative_janitor', f"Evicted: ttl={r1}, fit={r2}")
+        except Exception as e:
+            make_log('derivative_janitor', f"Error: {e}", level='error')
+        await asyncio.sleep(INTERVAL_SEC)

app/core/background/index_scout_v3.py

@@ -0,0 +1,177 @@
+import asyncio
+from typing import List
+
+import httpx
+import random
+import shutil
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core.models.my_network import KnownNode
+from app.core.models.content_v3 import EncryptedContent, ContentDerivative
+from app.core.ipfs_client import pin_add, find_providers, swarm_connect
+
+
+INTERVAL_SEC = 60
+PIN_CONCURRENCY = int(os.getenv('SYNC_MAX_CONCURRENT_PINS', '4'))
+DISK_WATERMARK_PCT = int(os.getenv('SYNC_DISK_LOW_WATERMARK_PCT', '90'))
+
+
+async def fetch_index(base_url: str, etag: str | None, since: str | None) -> tuple[List[dict], str | None]:
+    try:
+        headers = {}
+        params = {}
+        if since:
+            params['since'] = since
+        url = f"{base_url.rstrip('/')}/api/v1/content.delta" if since else f"{base_url.rstrip('/')}/api/v1/content.index"
+        if etag:
+            headers['If-None-Match'] = etag
+        async with httpx.AsyncClient(timeout=20) as client:
+            r = await client.get(url, headers=headers, params=params)
+            if r.status_code != 200:
+                if r.status_code == 304:
+                    return [], etag
+                return [], etag
+            j = r.json()
+            new_etag = r.headers.get('ETag') or etag
+            return j.get('items') or [], (j.get('next_since') or new_etag or etag)
+    except Exception:
+        return [], etag
+
+
+async def upsert_content(item: dict):
+    cid = item.get('encrypted_cid')
+    if not cid:
+        return
+    async with db_session() as session:
+        row = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+        if not row:
+            row = EncryptedContent(
+                encrypted_cid=cid,
+                title=item.get('title') or cid,
+                description=item.get('description') or '',
+                content_type=item.get('content_type') or 'application/octet-stream',
+                enc_size_bytes=item.get('size_bytes'),
+                preview_enabled=bool(item.get('preview_enabled')),
+                preview_conf=item.get('preview_conf') or {},
+                salt_b64=item.get('salt_b64'),
+            )
+            session.add(row)
+        else:
+            row.title = item.get('title') or row.title
+            row.description = item.get('description') or row.description
+            row.content_type = item.get('content_type') or row.content_type
+            row.enc_size_bytes = item.get('size_bytes') or row.enc_size_bytes
+            row.preview_enabled = bool(item.get('preview_enabled')) if item.get('preview_enabled') is not None else row.preview_enabled
+            if item.get('preview_conf'):
+                row.preview_conf = item['preview_conf']
+            if item.get('salt_b64'):
+                row.salt_b64 = item['salt_b64']
+        await session.commit()
+
+    # Fetch thumbnail via HTTP if provided and not present locally
+    cover_url = item.get('cover_url')
+    if cover_url:
+        try:
+            async with db_session() as session:
+                ec = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+                have_thumb = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id, ContentDerivative.kind == 'decrypted_thumbnail', ContentDerivative.status == 'ready'))).scalars().first()
+                if not have_thumb:
+                    import httpx, tempfile, os
+                    async with httpx.AsyncClient(timeout=30) as client:
+                        r = await client.get(cover_url)
+                        r.raise_for_status()
+                        tmp = tempfile.NamedTemporaryFile(delete=False)
+                        tmp.write(r.content)
+                        tmp.close()
+                    # Save into store
+                    from app.core.background.convert_v3_service import _save_derivative
+                    h, size = await _save_derivative(tmp.name, os.path.basename(cover_url) or 'thumb.jpg')
+                    cd = ContentDerivative(
+                        content_id=ec.id,
+                        kind='decrypted_thumbnail',
+                        local_path=os.path.join(os.getenv('UPLOADS_DIR', '/app/data'), h),
+                        content_type=r.headers.get('Content-Type') or 'image/jpeg',
+                        size_bytes=size,
+                        status='ready',
+                    )
+                    session.add(cd)
+                    await session.commit()
+        except Exception as e:
+            make_log('index_scout_v3', f"thumbnail fetch failed for {cid}: {e}", level='warning')
+
+
+async def main_fn(memory):
+    make_log('index_scout_v3', 'Service started', level='info')
+    sem = asyncio.Semaphore(PIN_CONCURRENCY)
+    while True:
+        try:
+            async with db_session() as session:
+                nodes = (await session.execute(select(KnownNode))).scalars().all()
+            for n in nodes:
+                base = f"http://{n.ip}:{n.port}"
+                # jitter 0..30s per node to reduce stampede
+                await asyncio.sleep(random.uniform(0, 30))
+                etag = (n.meta or {}).get('index_etag')
+                since = (n.meta or {}).get('index_since')
+                items, marker = await fetch_index(base, etag, since)
+                if not items and marker == etag:
+                    continue
+                # update node markers
+                try:
+                    async with db_session() as session:
+                        row = (await session.execute(select(KnownNode).where(KnownNode.id == n.id))).scalars().first()
+                        if row:
+                            meta = row.meta or {}
+                            meta['index_etag'] = marker
+                            meta['index_since'] = marker if (marker and 'T' in str(marker)) else meta.get('index_since')
+                            row.meta = meta
+                            await session.commit()
+                except Exception:
+                    pass
+
+                if not items:
+                    continue
+                make_log('index_scout_v3', f"Fetched {len(items)} from {base}")
+
+                # Check disk watermark
+                try:
+                    from app.core._config import UPLOADS_DIR
+                    du = shutil.disk_usage(UPLOADS_DIR)
+                    used_pct = int(100 * (1 - du.free / du.total))
+                    if used_pct >= DISK_WATERMARK_PCT:
+                        make_log('index_scout_v3', f"Disk watermark reached ({used_pct}%), skipping pins")
+                        continue
+                except Exception:
+                    pass
+
+                async def _pin_one(cid: str):
+                    async with sem:
+                        try:
+                            # Try to pre-connect to discovered providers
+                            try:
+                                provs = await find_providers(cid, max_results=5)
+                                for p in provs:
+                                    for addr in (p.get('addrs') or [])[:2]:
+                                        try:
+                                            await swarm_connect(addr)
+                                        except Exception:
+                                            pass
+                            except Exception:
+                                pass
+                            await pin_add(cid, recursive=True)
+                        except Exception as e:
+                            make_log('index_scout_v3', f"pin {cid} failed: {e}", level='warning')
+
+                tasks = []
+                for it in items:
+                    await upsert_content(it)
+                    cid = it.get('encrypted_cid')
+                    if cid:
+                        tasks.append(asyncio.create_task(_pin_one(cid)))
+                if tasks:
+                    await asyncio.gather(*tasks)
+        except Exception as e:
+            make_log('index_scout_v3', f"loop error: {e}", level='error')
+        await asyncio.sleep(INTERVAL_SEC)

app/core/crypto/aes_gcm_siv_stream.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import hmac
+import hashlib
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+from gcm_siv import GcmSiv  # requires `gcm_siv` package
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+SCHEME_AES_GCM_SIV = 0x01
+
+
+def _derive_nonce(salt: bytes, idx: int) -> bytes:
+    b = idx.to_bytes(8, 'big')
+    return hmac.new(salt, b, hashlib.sha256).digest()[:12]
+
+
+def build_header(chunk_bytes: int, salt: bytes) -> bytes:
+    assert 0 < chunk_bytes <= (1 << 31)
+    assert 1 <= len(salt) <= 255
+    # MAGIC(4) | ver(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt | reserved(5)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr += bytes([VERSION])
+    hdr += bytes([SCHEME_AES_GCM_SIV])
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr += bytes([len(salt)])
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """
+    Yield ENCF v1 stream using AES-GCM-SIV per chunk with deterministic nonces.
+    Frame: [p_len:4][cipher][tag(16)].
+    """
+    yield build_header(chunk_bytes, salt)
+    idx = 0
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        ct_and_tag = GcmSiv(key).encrypt(nonce, block, associated_data=None)
+        # Split tag
+        tag = ct_and_tag[-16:]
+        ct = ct_and_tag[:-16]
+        yield struct.pack(">I", len(block))
+        yield ct
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """Parse ENCF v1 (AES-GCM-SIV) and write plaintext to out_path."""
+    import aiofiles
+    buf = bytearray()
+
+    async def _fill(n: int):
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # header minimal
+    await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    if version != 1 or scheme != SCHEME_AES_GCM_SIV:
+        raise ValueError("unsupported encf header")
+    chunk_bytes = struct.unpack(">I", bytes(buf[6:10]))[0]
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    salt = bytes(buf[11:11 + salt_len])
+    del buf[:hdr_len]
+
+    async with aiofiles.open(out_path, 'wb') as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            await _fill(4)
+            if len(buf) == 0:
+                break
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            ct = bytes(buf[:p_len])
+            tag = bytes(buf[p_len:p_len+TAG_LEN])
+            del buf[:p_len+TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            pt = GcmSiv(key).decrypt(nonce, ct + tag, associated_data=None)
+            await out.write(pt)
+            idx += 1
+

app/core/crypto/aes_siv_stream.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import os
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+from Crypto.Cipher import SIV
+from Crypto.Cipher import AES
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+
+# Scheme codes
+SCHEME_AES_SIV = 0x02  # RFC5297 AES-SIV (CMAC-based)
+
+
+def build_header(chunk_bytes: int, salt: bytes, scheme: int = SCHEME_AES_SIV) -> bytes:
+    assert 0 < chunk_bytes <= (1 << 31)
+    assert 1 <= len(salt) <= 255
+    # Layout: MAGIC(4) | version(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt(N) | reserved(5 zeros)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr += bytes([VERSION])
+    hdr += bytes([scheme])
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr += bytes([len(salt)])
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def parse_header(buf: bytes) -> tuple[int, int, int, bytes, int]:
+    if len(buf) < 4 + 1 + 1 + 4 + 1:
+        raise ValueError("header too short")
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    chunk_bytes = struct.unpack(">I", buf[6:10])[0]
+    salt_len = buf[10]
+    needed = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    if len(buf) < needed:
+        raise ValueError("incomplete header")
+    salt = buf[11:11 + salt_len]
+    # reserved 5 bytes at the end ignored
+    return version, scheme, chunk_bytes, salt, needed
+
+
+def _ad(salt: bytes, idx: int) -> bytes:
+    return salt + struct.pack(">Q", idx)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """
+    Yield ENCF v1 stream bytes: [header] then for each chunk: [p_len:4][cipher][tag(16)].
+    Uses AES-SIV (RFC5297) with per-chunk associated data salt||index.
+    """
+    yield build_header(chunk_bytes, salt, SCHEME_AES_SIV)
+    idx = 0
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        siv = SIV.new(key=key, ciphermod=AES)  # new object per message
+        siv.update(_ad(salt, idx))
+        ciph, tag = siv.encrypt_and_digest(block)
+        yield struct.pack(">I", len(block))
+        yield ciph
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """
+    Parse ENCF v1 stream from async byte iterator and write plaintext to out_path.
+    """
+    import aiofiles
+    from Crypto.Cipher import SIV as _SIV
+    from Crypto.Cipher import AES as _AES
+
+    buf = bytearray()
+
+    async def _fill(n: int):
+        """Ensure at least n bytes in buffer (or EOF)."""
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # Read and parse header
+    await _fill(4 + 1 + 1 + 4 + 1)  # minimal header
+    # Might still be incomplete if salt_len > 0; keep filling progressively
+    # First, get preliminary to know salt_len
+    if len(buf) < 11:
+        await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    version, scheme, chunk_bytes, salt, consumed = parse_header(bytes(buf))
+    del buf[:consumed]
+    if version != 1:
+        raise ValueError("unsupported ENCF version")
+    if scheme != SCHEME_AES_SIV:
+        raise ValueError("unsupported scheme")
+
+    async with aiofiles.open(out_path, 'wb') as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            # Need at least 4 bytes for p_len
+            await _fill(4)
+            if len(buf) == 0:
+                break  # EOF exactly on boundary
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            # Now need p_len + 16 bytes
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            c = bytes(buf[:p_len])
+            t = bytes(buf[p_len:p_len+TAG_LEN])
+            del buf[:p_len+TAG_LEN]
+            siv = _SIV.new(key=key, ciphermod=_AES)
+            siv.update(_ad(salt, idx))
+            p = siv.decrypt_and_verify(c, t)
+            await out.write(p)
+            idx += 1
+

app/core/crypto/aesgcm_stream.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import os
+import hmac
+import hashlib
+from typing import BinaryIO, Iterator
+
+from Crypto.Cipher import AES
+
+
+CHUNK_BYTES = int(os.getenv("CRYPTO_CHUNK_BYTES", "1048576"))  # 1 MiB
+
+
+def _derive_nonce(salt: bytes, chunk_index: int) -> bytes:
+    """Derive a 12-byte GCM nonce deterministically from per-file salt and chunk index."""
+    idx = chunk_index.to_bytes(8, 'big')
+    digest = hmac.new(salt, idx, hashlib.sha256).digest()
+    return digest[:12]
+
+
+def encrypt_stream_aesgcm(src: BinaryIO, key: bytes, salt: bytes) -> Iterator[bytes]:
+    """
+    Read plaintext from src by CHUNK_BYTES, encrypt each chunk with AES-GCM using a
+    deterministic nonce derived from (salt, index). Yields bytes in framing: [C_i][TAG_i]...
+    Ciphertext length equals plaintext chunk length. Tag is 16 bytes.
+    """
+    assert len(key) in (16, 24, 32)
+    assert len(salt) >= 12
+    idx = 0
+    while True:
+        block = src.read(CHUNK_BYTES)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+        ciphertext, tag = cipher.encrypt_and_digest(block)
+        yield ciphertext
+        yield tag
+        idx += 1
+
+
+def decrypt_stream_aesgcm_iter(byte_iter: Iterator[bytes], key: bytes, salt: bytes) -> Iterator[bytes]:
+    """
+    Decrypt a stream that was produced by encrypt_stream_aesgcm.
+    Frame format: concatenation of [C_i][TAG_i] for each i, where |C_i| = CHUNK_BYTES and |TAG_i|=16.
+    We accept arbitrary chunking from the underlying iterator and reframe accordingly.
+    """
+    assert len(key) in (16, 24, 32)
+    buf = bytearray()
+    idx = 0
+    TAG_LEN = 16
+    def _try_yield():
+        nonlocal idx
+        out = []
+        while len(buf) >= CHUNK_BYTES + TAG_LEN:
+            c = bytes(buf[:CHUNK_BYTES])
+            t = bytes(buf[CHUNK_BYTES:CHUNK_BYTES+TAG_LEN])
+            del buf[:CHUNK_BYTES+TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+            try:
+                p = cipher.decrypt_and_verify(c, t)
+            except Exception as e:
+                raise ValueError(f"Decrypt failed at chunk {idx}: {e}")
+            out.append(p)
+            idx += 1
+        return out
+    for chunk in byte_iter:
+        if not chunk:
+            continue
+        buf.extend(chunk)
+        for p in _try_yield():
+            yield p
+    # At end, buffer must be empty
+    if len(buf) != 0:
+        raise ValueError("Trailing bytes in encrypted stream (incomplete frame)")
+

app/core/crypto/encf_stream.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import AsyncIterator
+
+from .aes_gcm_siv_stream import MAGIC as _MAGIC, VERSION as _VER, SCHEME_AES_GCM_SIV
+from .aes_gcm_siv_stream import decrypt_encf_to_file as _dec_gcmsiv
+from .aes_siv_stream import decrypt_encf_to_file as _dec_siv
+
+
+async def decrypt_encf_auto(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """
+    Detect scheme by peeking header, then delegate to proper decrypter.
+    Re-feeds the peeked bytes back to the chosen decoder.
+    """
+    buf = bytearray()
+
+    async def _fill(n: int):
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                ch = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if ch:
+                buf.extend(ch)
+
+    await _fill(11)
+    if buf[:4] != _MAGIC:
+        raise ValueError("bad magic")
+    scheme = buf[5]
+
+    async def _prepend_iter():
+        nonlocal buf
+        if buf:
+            yield bytes(buf)
+        async for ch in byte_iter:
+            yield ch
+
+    if scheme == SCHEME_AES_GCM_SIV:
+        await _dec_gcmsiv(_prepend_iter(), key, out_path)
+    else:
+        await _dec_siv(_prepend_iter(), key, out_path)
+

app/core/crypto/x25519.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import base64
+from typing import Tuple
+
+from nacl import public, signing, bindings
+
+
+def ed25519_to_x25519(ed_seed: bytes) -> Tuple[public.PrivateKey, public.PublicKey]:
+    """Convert Ed25519 seed (32 bytes) to X25519 key pair using libsodium conversion."""
+    if len(ed_seed) != 32:
+        raise ValueError("ed25519 seed must be 32 bytes")
+    sk_ed = signing.SigningKey(ed_seed)
+    sk_ed_bytes = sk_ed._seed + sk_ed.verify_key._key  # 64-byte expanded sk (seed||pub)
+    sk_x_bytes = bindings.crypto_sign_ed25519_sk_to_curve25519(sk_ed_bytes)
+    pk_x_bytes = bindings.crypto_sign_ed25519_pk_to_curve25519(bytes(sk_ed.verify_key))
+    sk_x = public.PrivateKey(sk_x_bytes)
+    pk_x = public.PublicKey(pk_x_bytes)
+    return sk_x, pk_x
+
+
+def x25519_pub_b64_from_ed_seed(ed_seed: bytes) -> str:
+    _, pk = ed25519_to_x25519(ed_seed)
+    return base64.b64encode(bytes(pk)).decode()
+

app/core/ipfs_client.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import os
+from typing import AsyncIterator, Dict, Any, Iterable, Optional
+
+import httpx
+
+IPFS_API_URL = os.getenv("IPFS_API_URL", "http://ipfs:5001")
+IPFS_GATEWAY_URL = os.getenv("IPFS_GATEWAY_URL", "http://ipfs:8080")
+
+
+async def add_streamed_file(stream_iter: Iterable[bytes], filename: str = "file.bin", params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """
+    Stream-encrypt pipeline can pass a generator of bytes here. We stream to /api/v0/add as multipart.
+    Returns dict with fields from IPFS: { Name, Hash, Size }.
+    """
+    params = params or {}
+    # Ensure deterministic chunking and CIDv1
+    default_params = {
+        "cid-version": 1,
+        "raw-leaves": "true",
+        "chunker": f"size-{int(os.getenv('CRYPTO_CHUNK_BYTES', '1048576'))}",
+        "pin": "true",
+        "wrap-with-directory": "false",
+        "progress": "true",
+    }
+    q = {**default_params, **params}
+
+    async with httpx.AsyncClient(timeout=None) as client:
+        files = {"file": (filename, stream_iter, "application/octet-stream")}
+        r = await client.post(f"{IPFS_API_URL}/api/v0/add", params=q, files=files)
+        r.raise_for_status()
+        # /add may emit NDJSON lines; most often single JSON
+        try:
+            data = r.json()
+        except Exception:
+            # Fallback: last non-empty line
+            last = [ln for ln in r.text.splitlines() if ln.strip()][-1]
+            import json as _json
+            data = _json.loads(last)
+        return data
+
+
+async def pin_add(cid: str, recursive: bool = True) -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=None) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/pin/add", params={"arg": cid, "recursive": str(recursive).lower(), "progress": "true"})
+        r.raise_for_status()
+        return r.json()
+
+
+async def pin_ls(cid: str) -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=30) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/pin/ls", params={"arg": cid})
+        r.raise_for_status()
+        return r.json()
+
+
+async def swarm_connect(multiaddr: str) -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=10) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/swarm/connect", params={"arg": multiaddr})
+        r.raise_for_status()
+        return r.json()
+
+
+async def cat_stream(cid: str):
+    client = httpx.AsyncClient(timeout=None)
+    try:
+        async with client.stream("POST", f"{IPFS_API_URL}/api/v0/cat", params={"arg": cid}) as r:
+            r.raise_for_status()
+            async for chunk in r.aiter_bytes():
+                if chunk:
+                    yield chunk
+    finally:
+        await client.aclose()
+
+
+async def find_providers(cid: str, max_results: int = 8):
+    """Query DHT for providers of a CID and return a list of {peer, addrs[]}.
+    Uses /api/v0/dht/findprovs and parses NDJSON stream.
+    """
+    out = []
+    async with httpx.AsyncClient(timeout=30) as client:
+        async with client.stream("POST", f"{IPFS_API_URL}/api/v0/dht/findprovs", params={"arg": cid}) as r:
+            r.raise_for_status()
+            async for line in r.aiter_lines():
+                if not line:
+                    continue
+                try:
+                    j = httpx.Response(200, text=line).json()
+                except Exception:
+                    import json as _json
+                    try:
+                        j = _json.loads(line)
+                    except Exception:
+                        continue
+                # Entries can include 'Extra' or 'Responses'
+                resps = j.get('Responses') or []
+                for resp in resps:
+                    peer = resp.get('ID') or resp.get('ID', '')
+                    addrs = resp.get('Addrs') or []
+                    if peer:
+                        out.append({"peer": peer, "addrs": addrs})
+                        if len(out) >= max_results:
+                            return out
+    return out

app/core/models/__init__.py

@@ -13,3 +13,11 @@ from app.core.models.asset import Asset
 from app.core.models.my_network import KnownNode, KnownNodeIncident, RemoteContentIndex
 from app.core.models.promo import PromoAction
 from app.core.models.tasks import BlockchainTask
+from app.core.models.content_v3 import (
+    EncryptedContent,
+    ContentKey,
+    IpfsSync,
+    ContentDerivative,
+    ContentIndexItem,
+    KeyGrant,
+)

app/core/models/content_v3.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+from datetime import datetime
+from sqlalchemy import Column, BigInteger, Integer, String, DateTime, JSON, Boolean, ForeignKey
+from sqlalchemy.orm import relationship
+
+from .base import AlchemyBase
+
+
+class EncryptedContent(AlchemyBase):
+    __tablename__ = 'encrypted_contents'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    # CID of encrypted source stored in IPFS (CIDv1 base32)
+    encrypted_cid = Column(String(128), nullable=False, unique=True)
+
+    # Public metadata
+    title = Column(String(512), nullable=False)
+    description = Column(String(4096), nullable=True)
+    content_type = Column(String(64), nullable=False)  # e.g. audio/flac, video/mp4, application/octet-stream
+
+    # Sizes
+    enc_size_bytes = Column(BigInteger, nullable=True)
+    plain_size_bytes = Column(BigInteger, nullable=True)
+
+    # Preview flags and config (all preview params live here, not in derivatives)
+    preview_enabled = Column(Boolean, nullable=False, default=False)
+    preview_conf = Column(JSON, nullable=False, default=dict)
+
+    # Crypto parameters (fixed per network)
+    aead_scheme = Column(String(32), nullable=False, default='AES_GCM_SIV')
+    chunk_bytes = Column(Integer, nullable=False, default=1048576)
+    salt_b64 = Column(String(64), nullable=True)  # per-content salt used for nonce derivation
+
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class ContentKey(AlchemyBase):
+    __tablename__ = 'content_keys'
+
+    content_id = Column(Integer, ForeignKey('encrypted_contents.id'), primary_key=True)
+    key_ciphertext_b64 = Column(String(512), nullable=False)
+    key_fingerprint = Column(String(128), nullable=False)
+    issuer_node_id = Column(String(128), nullable=False)
+    allow_auto_grant = Column(Boolean, nullable=False, default=True)
+    lease_expires_at = Column(DateTime, nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+
+    content = relationship('EncryptedContent', uselist=False, foreign_keys=[content_id])
+
+
+class IpfsSync(AlchemyBase):
+    __tablename__ = 'ipfs_sync'
+
+    content_id = Column(Integer, ForeignKey('encrypted_contents.id'), primary_key=True)
+    pin_state = Column(String(32), nullable=False, default='pinned')  # not_pinned|queued|pinning|pinned|failed
+    pin_error = Column(String(1024), nullable=True)
+    bytes_total = Column(BigInteger, nullable=True)
+    bytes_fetched = Column(BigInteger, nullable=True)
+    providers_cache = Column(JSON, nullable=False, default=list)
+    first_seen_at = Column(DateTime, nullable=True)
+    pinned_at = Column(DateTime, nullable=True)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    content = relationship('EncryptedContent', uselist=False, foreign_keys=[content_id])
+
+
+class ContentDerivative(AlchemyBase):
+    __tablename__ = 'content_derivatives'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    content_id = Column(Integer, ForeignKey('encrypted_contents.id'), nullable=False)
+    kind = Column(String(64), nullable=False)  # decrypted_high|decrypted_low|decrypted_thumbnail|decrypted_preview
+    interval_start_ms = Column(Integer, nullable=True)
+    interval_end_ms = Column(Integer, nullable=True)
+    local_path = Column(String(1024), nullable=False)
+    content_type = Column(String(64), nullable=True)
+    size_bytes = Column(BigInteger, nullable=True)
+    status = Column(String(32), nullable=False, default='pending')  # pending|processing|ready|failed
+    error = Column(String(1024), nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    last_access_at = Column(DateTime, nullable=True)
+
+    content = relationship('EncryptedContent', uselist=False, foreign_keys=[content_id])
+
+
+class ContentIndexItem(AlchemyBase):
+    __tablename__ = 'content_index_items'
+
+    encrypted_cid = Column(String(128), primary_key=True)
+    payload = Column(JSON, nullable=False, default=dict)
+    sig = Column(String(512), nullable=False)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class KeyGrant(AlchemyBase):
+    __tablename__ = 'key_grants'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    encrypted_cid = Column(String(128), nullable=False)
+    issuer_node_id = Column(String(128), nullable=False)
+    to_node_id = Column(String(128), nullable=False)
+    sealed_key_b64 = Column(String(1024), nullable=False)
+    aead_scheme = Column(String(32), nullable=False)
+    chunk_bytes = Column(Integer, nullable=False)
+    constraints = Column(JSON, nullable=False, default=dict)
+    issued_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    sig = Column(String(512), nullable=False)
+
+
+class UploadSession(AlchemyBase):
+    __tablename__ = 'upload_sessions'
+
+    id = Column(String(128), primary_key=True)  # tus Upload.ID
+    filename = Column(String(512), nullable=True)
+    size_bytes = Column(BigInteger, nullable=True)
+    state = Column(String(32), nullable=False, default='uploading')  # uploading|processing|pinned|failed
+    encrypted_cid = Column(String(128), nullable=True)
+    storage_path = Column(String(1024), nullable=True)
+    error = Column(String(1024), nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)

app/core/models/memory.py

@@ -42,6 +42,10 @@ class Memory:
         self._telegram_bot = Bot(TELEGRAM_API_KEY)
         self._client_telegram_bot = Bot(CLIENT_TELEGRAM_API_KEY)
 
+        # Network handshake guards
+        self._handshake_rl = {"minute": 0, "counts": {}}
+        self._handshake_nonces = {}
+
     @asynccontextmanager
     async def transaction(self, desc=""):
         make_log("Memory.transaction", f"Starting transaction; {desc}", level='debug')
@@ -77,4 +81,3 @@ class Memory:
 
             self._execute_queue.append([_fn, args, kwargs])
 
-

app/core/network/__init__.py

@@ -0,0 +1,2 @@
+# Network package for MY nodes
+

app/core/network/config.py

@@ -0,0 +1,38 @@
+import os
+from typing import List
+
+from app.core._config import PROJECT_HOST
+from .constants import NODE_TYPE_PUBLIC, NODE_TYPE_PRIVATE
+
+
+def _csv_list(val: str) -> List[str]:
+    return [x.strip() for x in (val or "").split(",") if x.strip()]
+
+
+# Handshake / network config driven by env
+NODE_PRIVACY = os.getenv("NODE_PRIVACY", NODE_TYPE_PUBLIC).strip().lower()
+if NODE_PRIVACY not in (NODE_TYPE_PUBLIC, NODE_TYPE_PRIVATE):
+    NODE_PRIVACY = NODE_TYPE_PUBLIC
+
+# Public endpoint for network (can be empty for private nodes)
+_env_public_host = os.getenv("PUBLIC_HOST")
+PUBLIC_HOST = _env_public_host if (_env_public_host is not None and _env_public_host.strip() != "") else None
+
+HANDSHAKE_INTERVAL_SEC = int(os.getenv("HANDSHAKE_INTERVAL_SEC", "5"))
+UNSUPPORTED_RECHECK_INTERVAL_SEC = int(os.getenv("UNSUPPORTED_RECHECK_INTERVAL_SEC", str(24 * 3600)))
+
+BOOTSTRAP_SEEDS = _csv_list(os.getenv("BOOTSTRAP_SEEDS", ""))
+BOOTSTRAP_REQUIRED = int(os.getenv("BOOTSTRAP_REQUIRED", "1")) == 1
+BOOTSTRAP_TIMEOUT_SEC = int(os.getenv("BOOTSTRAP_TIMEOUT_SEC", "20"))
+
+# Security knobs
+NETWORK_TLS_VERIFY = int(os.getenv("NETWORK_TLS_VERIFY", "1")) == 1
+HANDSHAKE_TS_TOLERANCE_SEC = int(os.getenv("HANDSHAKE_TS_TOLERANCE_SEC", "300"))
+HANDSHAKE_RATE_LIMIT_PER_MIN = int(os.getenv("HANDSHAKE_RATE_LIMIT_PER_MIN", "60"))
+
+# Capabilities
+NODE_IS_BOOTSTRAP = int(os.getenv("NODE_IS_BOOTSTRAP", "0")) == 1
+MAX_CONTENT_SIZE_MB = int(os.getenv("MAX_CONTENT_SIZE_MB", "512"))
+
+# Privacy allowlist (for NODE_PRIVACY=private)
+PRIVATE_ALLOWLIST = _csv_list(os.getenv("PRIVATE_ALLOWLIST", "/api/system.version"))

app/core/network/constants.py

@@ -0,0 +1,6 @@
+CURRENT_PROTOCOL_VERSION = "3.0.0"
+
+# Node roles/types
+NODE_TYPE_PUBLIC = "public"
+NODE_TYPE_PRIVATE = "private"
+

app/core/network/guard.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import time
+from typing import Dict, Set
+
+from app.core.network.config import HANDSHAKE_RATE_LIMIT_PER_MIN, HANDSHAKE_TS_TOLERANCE_SEC
+
+
+def check_rate_limit(memory, remote_ip: str) -> bool:
+    """Simple per-IP rate limit within current minute window.
+    Returns True if allowed, False if limited.
+    """
+    now = int(time.time())
+    minute = now // 60
+    rl = getattr(memory, "_handshake_rl", None)
+    if rl is None or rl.get("minute") != minute:
+        rl = {"minute": minute, "counts": {}}
+        memory._handshake_rl = rl
+    counts = rl["counts"]
+    cnt = counts.get(remote_ip, 0)
+    if cnt >= HANDSHAKE_RATE_LIMIT_PER_MIN:
+        return False
+    counts[remote_ip] = cnt + 1
+    return True
+
+
+def check_timestamp_fresh(ts: int) -> bool:
+    now = int(time.time())
+    return abs(now - int(ts)) <= HANDSHAKE_TS_TOLERANCE_SEC
+
+
+def check_and_remember_nonce(memory, pubkey_b58: str, nonce: str) -> bool:
+    """Return True if nonce is new; remember nonce with TTL ~ tolerance window.
+    We keep a compact in-memory set per pubkey.
+    """
+    now = int(time.time())
+    store = getattr(memory, "_handshake_nonces", None)
+    if store is None:
+        store = {}
+        memory._handshake_nonces = store
+
+    entry = store.get(pubkey_b58)
+    if entry is None:
+        entry = {"nonces": {}, "updated": now}
+        store[pubkey_b58] = entry
+
+    nonces: Dict[str, int] = entry["nonces"]
+    # prune old nonces
+    to_delete = [k for k, t in nonces.items() if now - int(t) > HANDSHAKE_TS_TOLERANCE_SEC]
+    for k in to_delete:
+        nonces.pop(k, None)
+
+    if nonce in nonces:
+        return False
+    # prevent unbounded growth
+    if len(nonces) > 2048:
+        # drop half oldest
+        for k, _ in sorted(nonces.items(), key=lambda kv: kv[1])[:1024]:
+            nonces.pop(k, None)
+    nonces[nonce] = now
+    entry["updated"] = now
+    return True
+

app/core/network/handshake.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+import os
+import time
+import shutil
+import secrets
+from typing import Dict, Any
+
+from base58 import b58encode
+from sqlalchemy import select
+
+from app.core._secrets import hot_pubkey, hot_seed
+from app.core._crypto.signer import Signer
+from app.core.logger import make_log
+from app.core.models.my_network import KnownNode
+from app.core.models.node_storage import StoredContent
+from app.core.storage import db_session
+from .constants import CURRENT_PROTOCOL_VERSION
+from .nodes import list_known_public_nodes
+from .config import PUBLIC_HOST, NODE_PRIVACY, NODE_IS_BOOTSTRAP, MAX_CONTENT_SIZE_MB
+from app.core._config import ALLOWED_CONTENT_TYPES
+from .constants import NODE_TYPE_PUBLIC
+
+
+START_TS = time.time()
+
+
+async def _metrics(session) -> Dict[str, Any]:
+    # Lightweight metrics for handshake
+    # Count total content (any type)
+    total_contents = (await session.execute(select(StoredContent))).scalars().all()
+    content_count = len(total_contents)
+    # Basic system metrics
+    try:
+        load1, load5, load15 = os.getloadavg()
+    except Exception:
+        load1 = load5 = load15 = 0.0
+    try:
+        from app.core._config import UPLOADS_DIR
+        du = shutil.disk_usage(UPLOADS_DIR)
+        disk_total_gb = round(du.total / (1024 ** 3), 2)
+        disk_free_gb = round(du.free / (1024 ** 3), 2)
+    except Exception:
+        disk_total_gb = disk_free_gb = -1
+    uptime_sec = int(time.time() - START_TS)
+    return {
+        "content_count": content_count,
+        "uptime_sec": uptime_sec,
+        "loadavg": [load1, load5, load15],
+        "disk_total_gb": disk_total_gb,
+        "disk_free_gb": disk_free_gb,
+    }
+
+
+def _sign(obj: Dict[str, Any]) -> str:
+    signer = Signer(hot_seed)
+    blob = json.dumps(obj, sort_keys=True, separators=(",", ":")).encode()
+    return signer.sign(blob)
+
+
+async def build_handshake_payload(session) -> Dict[str, Any]:
+    payload = {
+        "version": CURRENT_PROTOCOL_VERSION,
+        "public_key": b58encode(hot_pubkey).decode(),
+        # public_host is optional for private nodes
+        **({"public_host": PUBLIC_HOST} if PUBLIC_HOST else {}),
+        "node_type": NODE_PRIVACY if NODE_PRIVACY != NODE_TYPE_PUBLIC else NODE_TYPE_PUBLIC,
+        "metrics": await _metrics(session),
+        "capabilities": {
+            "accepts_inbound": NODE_PRIVACY == NODE_TYPE_PUBLIC,
+            "is_bootstrap": NODE_IS_BOOTSTRAP,
+            "supported_types": ALLOWED_CONTENT_TYPES,
+            "max_content_size_mb": MAX_CONTENT_SIZE_MB,
+        },
+        "timestamp": int(datetime.utcnow().timestamp()),
+        "nonce": secrets.token_hex(16),
+    }
+    try:
+        payload["known_public_nodes"] = await list_known_public_nodes(session)
+    except Exception:
+        payload["known_public_nodes"] = []
+    payload["signature"] = _sign(payload)
+    return payload
+
+
+async def compute_node_info(session) -> Dict[str, Any]:
+    node_info = {
+        "id": b58encode(hot_pubkey).decode(),
+        "public_key": b58encode(hot_pubkey).decode(),
+        **({"public_host": PUBLIC_HOST} if PUBLIC_HOST else {}),
+        "version": CURRENT_PROTOCOL_VERSION,
+        "node_type": NODE_PRIVACY,
+        "metrics": await _metrics(session),
+        "capabilities": {
+            "accepts_inbound": NODE_PRIVACY == NODE_TYPE_PUBLIC,
+            "is_bootstrap": NODE_IS_BOOTSTRAP,
+            "supported_types": ALLOWED_CONTENT_TYPES,
+            "max_content_size_mb": MAX_CONTENT_SIZE_MB,
+        },
+    }
+    return node_info
+
+def sign_response(data: Dict[str, Any]) -> Dict[str, Any]:
+    body = {
+        **data,
+        "timestamp": int(datetime.utcnow().timestamp()),
+    }
+    sig = _sign(body)
+    body["server_public_key"] = b58encode(hot_pubkey).decode()
+    body["server_signature"] = sig
+    return body

app/core/network/key_client.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import base64
+from typing import Optional
+
+import httpx
+from base58 import b58encode
+
+from app.core._secrets import hot_seed, hot_pubkey
+from app.core.crypto.x25519 import ed25519_to_x25519
+from app.core.logger import make_log
+from app.core.network.nodesig import sign_headers
+
+
+async def request_key_from_peer(base_url: str, encrypted_cid: str) -> Optional[bytes]:
+    """
+    Request a sealed key from peer and decrypt it using our X25519 private key.
+    Returns plaintext DEK bytes or None on failure.
+    """
+    try:
+        sk_x, pk_x = ed25519_to_x25519(hot_seed)
+        node_id = b58encode(hot_pubkey).decode()
+        body = {
+            "encrypted_cid": encrypted_cid,
+            "requestor_node_id": node_id,
+            "recipient_box_pub": base64.b64encode(bytes(pk_x)).decode(),
+        }
+        path = "/api/v1/keys.request"
+        headers = sign_headers("POST", path, json.dumps(body).encode(), hot_seed, b58encode(hot_pubkey).decode())
+        async with httpx.AsyncClient(timeout=15) as client:
+            r = await client.post(f"{base_url.rstrip('/')}{path}", json=body, headers=headers)
+            if r.status_code != 200:
+                make_log('key_client', f"{base_url} returned {r.status_code}: {r.text}", level='warning')
+                return None
+            j = r.json()
+        sealed_b64 = j.get('sealed_key_b64')
+        if not sealed_b64:
+            return None
+        sealed = base64.b64decode(sealed_b64)
+        from nacl.public import SealedBox
+        sb = SealedBox(sk_x)
+        dek = sb.decrypt(sealed)
+        return dek
+    except Exception as e:
+        make_log('key_client', f"request/decrypt failed: {e}", level='error')
+        return None

app/core/network/nodes.py

@@ -0,0 +1,261 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timedelta
+import json
+from typing import Dict, Any, Optional, List
+
+import httpx
+from base58 import b58encode
+from sqlalchemy import select, update
+
+from app.core.logger import make_log
+from app.core.models.my_network import KnownNode
+from app.core.storage import db_session
+from app.core._secrets import hot_pubkey
+from .config import (
+    HANDSHAKE_INTERVAL_SEC,
+    UNSUPPORTED_RECHECK_INTERVAL_SEC,
+    BOOTSTRAP_SEEDS,
+    BOOTSTRAP_REQUIRED,
+    BOOTSTRAP_TIMEOUT_SEC,
+    NODE_PRIVACY,
+    NETWORK_TLS_VERIFY,
+)
+from .constants import NODE_TYPE_PRIVATE
+from .semver import compatibility
+from .constants import CURRENT_PROTOCOL_VERSION
+
+
+def _now() -> datetime:
+    return datetime.utcnow()
+
+
+async def upsert_known_node(session, host: str, port: int, public_key: str, meta: Dict[str, Any]) -> KnownNode:
+    # Host can be full URL; normalize host/ip and port if available
+    host = (host or "").replace("http://", "").replace("https://", "").strip("/")
+    h_only = host
+    if ":" in host:
+        h_only, port_str = host.rsplit(":", 1)
+        try:
+            port = int(port_str)
+        except Exception:
+            pass
+    # Prefer match by public_key (stable identity)
+    if public_key:
+        result = await session.execute(select(KnownNode).where(KnownNode.public_key == public_key))
+        row = result.scalars().first()
+        if row:
+            row.ip = h_only or row.ip
+            row.port = port or row.port
+            row.public_key = public_key or row.public_key
+            row.meta = {**(row.meta or {}), **(meta or {})}
+            row.last_sync = _now()
+            await session.commit()
+            return row
+    # Fallback by IP/host
+    result = await session.execute(select(KnownNode).where(KnownNode.ip == h_only))
+    row = result.scalars().first()
+    if row:
+        row.port = port or row.port
+        row.public_key = public_key or row.public_key
+        row.meta = {**(row.meta or {}), **(meta or {})}
+        row.last_sync = _now()
+        await session.commit()
+        return row
+    node = KnownNode(
+        ip=h_only,
+        port=port or 80,
+        public_key=public_key,
+        reputation=0,
+        last_sync=_now(),
+        meta=meta or {},
+        located_at=_now(),
+    )
+    session.add(node)
+    await session.commit()
+    return node
+
+
+def _compatibility_for_meta(remote_version: str) -> str:
+    if not remote_version or remote_version == "0.0.0":
+        return "warning"
+    return compatibility(remote_version, CURRENT_PROTOCOL_VERSION)
+
+
+async def list_known_public_nodes(session) -> List[Dict[str, Any]]:
+    rows = (await session.execute(select(KnownNode))).scalars().all()
+    result = []
+    for r in rows:
+        meta = r.meta or {}
+        if not meta.get("is_public", True):
+            continue
+        result.append({
+            "host": r.ip,
+            "port": r.port,
+            "public_key": r.public_key,
+            "version": meta.get("version"),
+            "compatibility": _compatibility_for_meta(meta.get("version", "0.0.0")),
+            "last_seen": (r.last_sync.isoformat() + "Z") if r.last_sync else None,
+            "public_host": meta.get("public_host"),
+            "capabilities": meta.get("capabilities") or {},
+        })
+    return result
+
+
+async def _handshake_with(session, base_url: str) -> Optional[Dict[str, Any]]:
+    url = base_url.rstrip("/") + "/api/v1/network.handshake"
+    from .handshake import build_handshake_payload
+    payload = await build_handshake_payload(session)
+    timeout = httpx.Timeout(5.0, read=10.0)
+    async with httpx.AsyncClient(timeout=timeout, verify=NETWORK_TLS_VERIFY) as client:
+        r = await client.post(url, json=payload)
+        if r.status_code == 403 and NODE_PRIVACY == NODE_TYPE_PRIVATE:
+            # We are private; outbound is allowed, inbound denied by peers is fine
+            pass
+        r.raise_for_status()
+        data = r.json()
+        # Verify server signature if present
+        try:
+            import nacl.signing
+            from base58 import b58decode
+            required = ["server_signature", "server_public_key", "timestamp"]
+            if all(k in data for k in required):
+                signed_fields = {k: data[k] for k in data if k not in ("server_signature", "server_public_key")}
+                blob = json.dumps(signed_fields, sort_keys=True, separators=(",", ":")).encode()
+                vk = nacl.signing.VerifyKey(b58decode(data["server_public_key"]))
+                vk.verify(blob, b58decode(data["server_signature"]))
+        except Exception as e:
+            make_log("Handshake", f"Server signature verification failed for {base_url}: {e}", level='warning')
+        return data
+
+
+async def pick_next_node(session) -> Optional[KnownNode]:
+    rows = (await session.execute(select(KnownNode))).scalars().all()
+    if not rows:
+        return None
+    # Prefer nodes with oldest last_sync
+    rows.sort(key=lambda r: (r.last_sync or datetime.fromtimestamp(0)))
+    now = _now()
+    for r in rows:
+        meta = r.meta or {}
+        compat = _compatibility_for_meta(meta.get("version", "0.0.0"))
+        if compat == "blocked":
+            last = datetime.fromisoformat(meta.get("unsupported_last_checked_at")) if meta.get("unsupported_last_checked_at") else None
+            if last and (now - last) < timedelta(seconds=UNSUPPORTED_RECHECK_INTERVAL_SEC):
+                continue
+        # Backoff after failures
+        if meta.get("last_failure_at"):
+            try:
+                last_fail = datetime.fromisoformat(meta.get("last_failure_at"))
+                fail_count = int(meta.get("fail_count", 1))
+                # Exponential backoff: 30s * 2^fail_count, capped 2h
+                wait = min(7200, 30 * (2 ** max(0, fail_count)))
+                if (now - last_fail) < timedelta(seconds=wait):
+                    continue
+            except Exception:
+                pass
+        return r
+    # If we only have unsupported nodes and all are within cooldown, skip this round
+    return None
+
+
+async def perform_handshake_round():
+    async with db_session(auto_commit=True) as session:
+        # Private nodes still do outbound handshakes; inbound typically unreachable without public endpoint
+        node = await pick_next_node(session)
+        if not node:
+            return
+        base_url = node.meta.get("public_host") or f"http://{node.ip}:{node.port}"
+        try:
+            resp = await _handshake_with(session, base_url)
+            # Merge known nodes received
+            for peer in (resp or {}).get("known_public_nodes", []):
+                try:
+                    await upsert_known_node(
+                        session,
+                        host=peer.get("host") or peer.get("public_host") or "",
+                        port=int(peer.get("port") or 80),
+                        public_key=peer.get("public_key") or "",
+                        meta={
+                            "is_public": True,
+                            "version": peer.get("version") or "0.0.0",
+                            "public_host": peer.get("public_host") or (f"http://{peer.get('host')}:{peer.get('port')}" if peer.get('host') else None),
+                        }
+                    )
+                except Exception as e:
+                    make_log("Handshake", f"Ignore bad peer from {base_url}: {e}", level='warning')
+            # Update last_sync and meta for node
+            node.last_sync = _now()
+            node.meta = {**(node.meta or {}), "last_response": resp, "fail_count": 0}
+            await session.commit()
+            make_log("Handshake", f"Handshake OK with {base_url}")
+        except Exception as e:
+            make_log("Handshake", f"Handshake failed with {base_url}: {e}", level='warning')
+            # Record incident-lite in meta
+            meta = node.meta or {}
+            meta["last_error"] = str(e)
+            meta["last_failure_at"] = _now().isoformat()
+            meta["fail_count"] = int(meta.get("fail_count", 0)) + 1
+            node.meta = meta
+            await session.commit()
+
+
+async def network_handshake_daemon(app):
+    # Stagger start a bit to allow HTTP server to come up
+    await asyncio.sleep(3)
+    make_log("Handshake", f"Daemon started; interval={HANDSHAKE_INTERVAL_SEC}s")
+    while True:
+        try:
+            await perform_handshake_round()
+        except Exception as e:
+            make_log("Handshake", f"Round error: {e}", level='error')
+        await asyncio.sleep(HANDSHAKE_INTERVAL_SEC)
+
+
+async def bootstrap_once_and_exit_if_failed():
+    # Do not try to bootstrap private nodes as inbound is blocked, but outbound required for seeds discovery
+    seeds = BOOTSTRAP_SEEDS or []
+    if not seeds:
+        return  # Nothing to do
+    async with db_session(auto_commit=True) as session:
+        # If we already know nodes, skip bootstrap
+        have_any = (await session.execute(select(KnownNode))).scalars().first()
+        if have_any:
+            return
+    make_log("Bootstrap", f"Starting bootstrap with seeds={seeds}; required={BOOTSTRAP_REQUIRED}")
+
+    deadline = _now() + timedelta(seconds=BOOTSTRAP_TIMEOUT_SEC)
+    ok = False
+    for seed in seeds:
+        try:
+            async with db_session(auto_commit=True) as session:
+                resp = await _handshake_with(session, seed)
+                if resp:
+                    ok = True
+                    # Seed itself gets inserted by handshake handling route; also insert it explicitly
+                    try:
+                        await upsert_known_node(
+                            session,
+                            host=seed,
+                            port=80,
+                            public_key=resp.get("node", {}).get("public_key", ""),
+                            meta={
+                                "is_public": True,
+                                "version": resp.get("node", {}).get("version", "0.0.0"),
+                                "public_host": resp.get("node", {}).get("public_host") or seed,
+                            }
+                        )
+                    except Exception:
+                        pass
+                    break
+        except Exception as e:
+            make_log("Bootstrap", f"Seed failed {seed}: {e}", level='warning')
+        if _now() > deadline:
+            break
+
+    if BOOTSTRAP_REQUIRED and not ok:
+        make_log("Bootstrap", "Failed to reach any bootstrap seeds; exiting", level='error')
+        # Hard exit; Sanic won't stop otherwise
+        import os
+        os._exit(2)

app/core/network/nodesig.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import secrets
+import time
+from typing import Dict, Tuple
+
+from base58 import b58decode, b58encode
+
+from app.core.network.guard import check_timestamp_fresh, check_and_remember_nonce
+
+
+def _body_sha256(body: bytes) -> str:
+    h = hashlib.sha256()
+    h.update(body or b"")
+    return h.hexdigest()
+
+
+def canonical_string(method: str, path: str, body: bytes, ts: int, nonce: str, node_id: str) -> bytes:
+    parts = [
+        method.upper(),
+        path,
+        _body_sha256(body),
+        str(int(ts)),
+        str(nonce),
+        node_id,
+    ]
+    return ("\n".join(parts)).encode()
+
+
+def sign_headers(method: str, path: str, body: bytes, sk_bytes: bytes, pk_b58: str) -> Dict[str, str]:
+    import nacl.signing
+    ts = int(time.time())
+    nonce = secrets.token_hex(16)
+    msg = canonical_string(method, path, body, ts, nonce, pk_b58)
+    sig = nacl.signing.SigningKey(sk_bytes).sign(msg).signature
+    return {
+        "X-Node-Id": pk_b58,
+        "X-Node-Ts": str(ts),
+        "X-Node-Nonce": nonce,
+        "X-Node-Sig": b58encode(sig).decode(),
+    }
+
+
+def verify_request(request, memory) -> Tuple[bool, str, str]:
+    """Verify NodeSig headers of an incoming Sanic request.
+    Returns (ok, node_id, error). ok==True if signature valid, timestamp fresh, nonce unused.
+    """
+    try:
+        node_id = request.headers.get("X-Node-Id", "").strip()
+        ts = int(request.headers.get("X-Node-Ts", "0").strip() or 0)
+        nonce = request.headers.get("X-Node-Nonce", "").strip()
+        sig_b58 = request.headers.get("X-Node-Sig", "").strip()
+        if not node_id or not ts or not nonce or not sig_b58:
+            return False, "", "MISSING_HEADERS"
+        if not check_timestamp_fresh(ts):
+            return False, node_id, "STALE_TS"
+        if not check_and_remember_nonce(memory, node_id, nonce):
+            return False, node_id, "NONCE_REPLAY"
+        import nacl.signing
+        vk = nacl.signing.VerifyKey(b58decode(node_id))
+        sig = b58decode(sig_b58)
+        msg = canonical_string(request.method, request.path, request.body or b"", ts, nonce, node_id)
+        vk.verify(msg, sig)
+        return True, node_id, ""
+    except Exception as e:
+        return False, "", f"BAD_SIGNATURE: {e}"
+

app/core/network/semver.py

@@ -0,0 +1,26 @@
+from typing import Tuple
+
+
+def parse_semver(v: str) -> Tuple[int, int, int]:
+    try:
+        parts = v.split(".")
+        major = int(parts[0])
+        minor = int(parts[1]) if len(parts) > 1 else 0
+        patch = int(parts[2]) if len(parts) > 2 else 0
+        return major, minor, patch
+    except Exception:
+        return 0, 0, 0
+
+
+def compatibility(peer: str, current: str) -> str:
+    """Return one of: compatible, warning, blocked"""
+    pM, pm, pp = parse_semver(peer)
+    cM, cm, cp = parse_semver(current)
+    if pM != cM:
+        return "blocked"
+    # Same major
+    if pm == cm:
+        return "compatible"
+    # Different minor within same major => warning
+    return "warning"
+