relayers new code

app/core/background/convert_v3_service.py

@@ -0,0 +1,307 @@
+import asyncio
+import os
+import json
+import shutil
+from datetime import datetime
+from typing import List, Tuple
+
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core._config import UPLOADS_DIR, BACKEND_LOGS_DIR_HOST
+from app.core.models.content_v3 import (
+    EncryptedContent,
+    ContentKey,
+    ContentDerivative,
+    UploadSession,
+)
+from app.core.models.node_storage import StoredContent
+from app.core.ipfs_client import cat_stream
+from app.core.crypto.aesgcm_stream import CHUNK_BYTES
+from app.core.crypto.encf_stream import decrypt_encf_auto
+from app.core.network.key_client import request_key_from_peer
+from app.core.models.my_network import KnownNode
+
+
+CONCURRENCY = int(os.getenv("CONVERT_V3_MAX_CONCURRENCY", "3"))
+
+
+def _ensure_dir(path: str):
+    try:
+        os.makedirs(path, exist_ok=True)
+    except Exception:
+        pass
+
+
+async def _sha256_b58(file_path: str) -> str:
+    import hashlib
+    import base58
+    h = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(2 * 1024 * 1024), b''):
+            h.update(chunk)
+    return base58.b58encode(h.digest()).decode()
+
+
+async def _save_derivative(file_path: str, filename: str) -> Tuple[str, int]:
+    """Move file into UPLOADS_DIR under sha256 b58 name; return (hash_b58, size)."""
+    file_hash = await _sha256_b58(file_path)
+    dst = os.path.join(UPLOADS_DIR, file_hash)
+    try:
+        os.remove(dst)
+    except FileNotFoundError:
+        pass
+    shutil.move(file_path, dst)
+    size = os.path.getsize(dst)
+    return file_hash, size
+
+
+async def _run_media_converter(input_host_path: str, input_ext: str, quality: str, trim_value: str | None, is_audio: bool) -> Tuple[str, dict]:
+    rid = __import__('uuid').uuid4().hex[:8]
+    output_dir_container = f"/tmp/conv_{rid}"
+    output_dir_host = f"/tmp/conv_{rid}"
+    _ensure_dir(output_dir_host)
+    logs_dir_host = BACKEND_LOGS_DIR_HOST
+    _ensure_dir(logs_dir_host)
+
+    cmd = [
+        "docker", "run", "--rm",
+        "-v", f"{input_host_path}:/app/input:ro",
+        "-v", f"{output_dir_host}:/app/output",
+        "-v", f"{logs_dir_host}:/app/logs",
+        "media_converter",
+        "--ext", input_ext,
+        "--quality", quality,
+    ]
+    if trim_value:
+        cmd.extend(["--trim", trim_value])
+    if is_audio:
+        cmd.append("--audio-only")
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        raise RuntimeError(f"media_converter failed: {stderr.decode()}")
+
+    # Find produced media file and optional output.json
+    try:
+        files = os.listdir(output_dir_host)
+    except Exception as e:
+        raise RuntimeError(f"Read output dir error: {e}")
+    media_files = [f for f in files if f != "output.json"]
+    if len(media_files) != 1:
+        raise RuntimeError(f"Expected one media file, found {len(media_files)}: {media_files}")
+    output_media = os.path.join(output_dir_host, media_files[0])
+    ffprobe_meta = {}
+    out_json = os.path.join(output_dir_host, "output.json")
+    if os.path.exists(out_json):
+        try:
+            with open(out_json, 'r') as f:
+                ffprobe_meta = json.load(f)
+        except Exception:
+            ffprobe_meta = {}
+    return output_media, ffprobe_meta
+
+
+async def _convert_content(ec: EncryptedContent, input_host_path: str):
+    content_kind = 'audio' if ec.content_type.startswith('audio/') else ('video' if ec.content_type.startswith('video/') else 'other')
+    if content_kind == 'other':
+        return
+
+    input_ext = (ec.content_type.split('/')[-1] or 'bin')
+    is_audio = content_kind == 'audio'
+    # Required outputs
+    required = ['high', 'low', 'low_preview']
+
+    # Preview interval
+    conf = ec.preview_conf or {}
+    intervals = conf.get('intervals') or [[0, int(conf.get('duration_ms', 30000))]]
+    main_interval = intervals[0]
+    trim_value = None
+    start_s = max(0, int(main_interval[0]) // 1000)
+    dur_s = max(1, int((main_interval[1] - main_interval[0]) // 1000) or 30)
+    trim_value = f"{start_s},{dur_s}"
+
+    qualities = {
+        'high': 'high',
+        'low': 'low',
+        'low_preview': 'low',
+    }
+
+    for opt in required:
+        try:
+            # Mark derivative processing
+            async with db_session() as session:
+                cd = ContentDerivative(
+                    content_id=ec.id,
+                    kind=f"decrypted_{opt if opt != 'low_preview' else 'preview'}",
+                    interval_start_ms=main_interval[0] if opt == 'low_preview' else None,
+                    interval_end_ms=main_interval[1] if opt == 'low_preview' else None,
+                    local_path="",
+                    status='processing',
+                )
+                session.add(cd)
+                await session.commit()
+
+            out_path, ffprobe = await _run_media_converter(
+                input_host_path=input_host_path,
+                input_ext=input_ext,
+                quality=qualities[opt],
+                trim_value=trim_value if opt == 'low_preview' else None,
+                is_audio=is_audio,
+            )
+
+            # Save into store and StoredContent
+            file_hash, size_bytes = await _save_derivative(out_path, os.path.basename(out_path))
+
+            async with db_session() as session:
+                sc = StoredContent(
+                    type="local/content_bin",
+                    hash=file_hash,
+                    user_id=None,
+                    filename=os.path.basename(out_path),
+                    meta={'encrypted_cid': ec.encrypted_cid, 'kind': opt, 'ffprobe_meta': ffprobe},
+                    created=datetime.utcnow(),
+                )
+                session.add(sc)
+                await session.flush()
+
+                # Update derivative record
+                cd = (await session.execute(select(ContentDerivative).where(
+                    ContentDerivative.content_id == ec.id,
+                    ContentDerivative.kind == (f"decrypted_{opt if opt != 'low_preview' else 'preview'}"),
+                    ContentDerivative.status == 'processing'
+                ))).scalars().first()
+                if cd:
+                    cd.local_path = os.path.join(UPLOADS_DIR, file_hash)
+                    cd.size_bytes = size_bytes
+                    cd.content_type = ('audio/mpeg' if is_audio else 'video/mp4') if opt != 'high' else ec.content_type
+                    cd.status = 'ready'
+                await session.commit()
+
+            make_log('convert_v3', f"Converted {ec.encrypted_cid} opt={opt} -> {file_hash}")
+        except Exception as e:
+            make_log('convert_v3', f"Convert error {ec.encrypted_cid} opt={opt}: {e}", level='error')
+            async with db_session() as session:
+                cd = ContentDerivative(
+                    content_id=ec.id,
+                    kind=f"decrypted_{opt if opt != 'low_preview' else 'preview'}",
+                    status='failed',
+                    error=str(e),
+                    local_path="",
+                )
+                session.add(cd)
+                await session.commit()
+
+
+async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, str]]:
+    async with db_session() as session:
+        # Find A/V contents with preview_enabled and no ready low/low_preview derivatives yet
+        ecs = (await session.execute(select(EncryptedContent).where(
+            EncryptedContent.preview_enabled == True
+        ).order_by(EncryptedContent.created_at.desc()))).scalars().all()
+
+        picked: List[Tuple[EncryptedContent, str]] = []
+        for ec in ecs:
+            # Check if derivatives already ready
+            rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id))).scalars().all()
+            kinds_ready = {r.kind for r in rows if r.status == 'ready'}
+            required = {'decrypted_low', 'decrypted_high'} if ec.content_type.startswith('audio/') else {'decrypted_low', 'decrypted_high', 'decrypted_preview'}
+            if required.issubset(kinds_ready):
+                continue
+            # Always decrypt from IPFS using local or remote key
+            storage_path: str | None = None
+            ck = (await session.execute(select(ContentKey).where(ContentKey.content_id == ec.id))).scalars().first()
+            if ck:
+                storage_path = await stage_plain_from_ipfs(ec, ck.key_ciphertext_b64)
+            if not storage_path:
+                peers = (await session.execute(select(KnownNode))).scalars().all()
+                for peer in peers:
+                    base_url = f"http://{peer.ip}:{peer.port}"
+                    dek = await request_key_from_peer(base_url, ec.encrypted_cid)
+                    if not dek:
+                        continue
+                    import base64
+                    dek_b64 = base64.b64encode(dek).decode()
+                    session_ck = ContentKey(
+                        content_id=ec.id,
+                        key_ciphertext_b64=dek_b64,
+                        key_fingerprint=peer.public_key,
+                        issuer_node_id=peer.public_key,
+                        allow_auto_grant=True,
+                    )
+                    session.add(session_ck)
+                    await session.commit()
+                    storage_path = await stage_plain_from_ipfs(ec, dek_b64)
+                    if storage_path:
+                        break
+            if not storage_path or not os.path.exists(storage_path):
+                continue
+            picked.append((ec, storage_path))
+            if len(picked) >= limit:
+                break
+        return picked
+
+
+async def worker_loop():
+    sem = asyncio.Semaphore(CONCURRENCY)
+
+    async def _run_one(ec: EncryptedContent, input_path: str):
+        async with sem:
+            try:
+                await _convert_content(ec, input_path)
+                # After successful conversion, attempt to remove staging file to avoid duplicates
+                try:
+                    if input_path and input_path.startswith("/data/") and os.path.exists(input_path):
+                        os.remove(input_path)
+                except Exception:
+                    pass
+            except Exception as e:
+                make_log('convert_v3', f"job error {ec.encrypted_cid}: {e}", level='error')
+
+    while True:
+        try:
+            batch = await _pick_pending(limit=CONCURRENCY * 2)
+            if not batch:
+                await asyncio.sleep(3)
+                continue
+            tasks = [asyncio.create_task(_run_one(ec, path)) for (ec, path) in batch]
+            await asyncio.gather(*tasks)
+        except Exception as e:
+            make_log('convert_v3', f"loop error: {e}", level='error')
+            await asyncio.sleep(2)
+
+
+async def main_fn(memory):
+    make_log('convert_v3', f"Service started with concurrency={CONCURRENCY}", level='info')
+    await worker_loop()
+
+
+async def stage_plain_from_ipfs(ec: EncryptedContent, dek_b64: str) -> str | None:
+    """Download encrypted ENCF stream from IPFS and decrypt on the fly into a temp file."""
+    import base64, tempfile
+    dek = base64.b64decode(dek_b64)
+    tmp = tempfile.NamedTemporaryFile(prefix=f"dec_{ec.encrypted_cid[:8]}_", delete=False)
+    tmp_path = tmp.name
+    tmp.close()
+    try:
+        async def _aiter():
+            async for ch in cat_stream(ec.encrypted_cid):
+                yield ch
+        await decrypt_encf_auto(_aiter(), dek, tmp_path)
+        return tmp_path
+    except Exception as e:
+        make_log('convert_v3', f"decrypt from ipfs failed: {e}", level='error')
+        try:
+            os.remove(tmp_path)
+        except Exception:
+            pass
+        return None
+
+
+    

app/core/background/derivative_cache_janitor.py

@@ -0,0 +1,87 @@
+import asyncio
+import os
+from datetime import datetime, timedelta
+
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core.models.content_v3 import ContentDerivative
+
+
+MAX_GB = float(os.getenv('DERIVATIVE_CACHE_MAX_GB', '50'))
+TTL_DAYS = int(os.getenv('DERIVATIVE_CACHE_TTL_DAYS', '30'))
+INTERVAL_SEC = int(os.getenv('DERIVATIVE_JANITOR_INTERVAL_SEC', '600'))
+
+
+async def _current_total_size() -> int:
+    async with db_session() as session:
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        return sum(int(r.size_bytes or 0) for r in rows)
+
+
+async def _evict_over_ttl(now: datetime) -> int:
+    removed = 0
+    if TTL_DAYS <= 0:
+        return 0
+    async with db_session() as session:
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        for r in rows:
+            la = r.last_access_at or r.created_at
+            if la and (now - la) > timedelta(days=TTL_DAYS):
+                try:
+                    if r.local_path and os.path.exists(r.local_path):
+                        os.remove(r.local_path)
+                except Exception:
+                    pass
+                r.status = 'pending'
+                r.local_path = None
+                r.size_bytes = None
+                r.last_access_at = None
+                removed += 1
+        await session.commit()
+    return removed
+
+
+async def _evict_to_fit():
+    limit_bytes = int(MAX_GB * (1024 ** 3))
+    total = await _current_total_size()
+    if total <= limit_bytes:
+        return 0
+    to_remove = total - limit_bytes
+    removed = 0
+    async with db_session() as session:
+        # Oldest first by last_access_at
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        rows.sort(key=lambda r: (r.last_access_at or r.created_at or datetime.utcfromtimestamp(0)))
+        for r in rows:
+            if to_remove <= 0:
+                break
+            size = int(r.size_bytes or 0)
+            try:
+                if r.local_path and os.path.exists(r.local_path):
+                    os.remove(r.local_path)
+            except Exception:
+                pass
+            r.status = 'pending'
+            r.local_path = None
+            r.last_access_at = None
+            r.size_bytes = None
+            await session.commit()
+            to_remove -= size
+            removed += 1
+    return removed
+
+
+async def main_fn(memory):
+    make_log('derivative_janitor', f"Started (MAX_GB={MAX_GB}, TTL_DAYS={TTL_DAYS})", level='info')
+    while True:
+        try:
+            now = datetime.utcnow()
+            r1 = await _evict_over_ttl(now)
+            r2 = await _evict_to_fit()
+            if r1 or r2:
+                make_log('derivative_janitor', f"Evicted: ttl={r1}, fit={r2}")
+        except Exception as e:
+            make_log('derivative_janitor', f"Error: {e}", level='error')
+        await asyncio.sleep(INTERVAL_SEC)

app/core/background/index_scout_v3.py

@@ -0,0 +1,177 @@
+import asyncio
+from typing import List
+
+import httpx
+import random
+import shutil
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core.models.my_network import KnownNode
+from app.core.models.content_v3 import EncryptedContent, ContentDerivative
+from app.core.ipfs_client import pin_add, find_providers, swarm_connect
+
+
+INTERVAL_SEC = 60
+PIN_CONCURRENCY = int(os.getenv('SYNC_MAX_CONCURRENT_PINS', '4'))
+DISK_WATERMARK_PCT = int(os.getenv('SYNC_DISK_LOW_WATERMARK_PCT', '90'))
+
+
+async def fetch_index(base_url: str, etag: str | None, since: str | None) -> tuple[List[dict], str | None]:
+    try:
+        headers = {}
+        params = {}
+        if since:
+            params['since'] = since
+        url = f"{base_url.rstrip('/')}/api/v1/content.delta" if since else f"{base_url.rstrip('/')}/api/v1/content.index"
+        if etag:
+            headers['If-None-Match'] = etag
+        async with httpx.AsyncClient(timeout=20) as client:
+            r = await client.get(url, headers=headers, params=params)
+            if r.status_code != 200:
+                if r.status_code == 304:
+                    return [], etag
+                return [], etag
+            j = r.json()
+            new_etag = r.headers.get('ETag') or etag
+            return j.get('items') or [], (j.get('next_since') or new_etag or etag)
+    except Exception:
+        return [], etag
+
+
+async def upsert_content(item: dict):
+    cid = item.get('encrypted_cid')
+    if not cid:
+        return
+    async with db_session() as session:
+        row = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+        if not row:
+            row = EncryptedContent(
+                encrypted_cid=cid,
+                title=item.get('title') or cid,
+                description=item.get('description') or '',
+                content_type=item.get('content_type') or 'application/octet-stream',
+                enc_size_bytes=item.get('size_bytes'),
+                preview_enabled=bool(item.get('preview_enabled')),
+                preview_conf=item.get('preview_conf') or {},
+                salt_b64=item.get('salt_b64'),
+            )
+            session.add(row)
+        else:
+            row.title = item.get('title') or row.title
+            row.description = item.get('description') or row.description
+            row.content_type = item.get('content_type') or row.content_type
+            row.enc_size_bytes = item.get('size_bytes') or row.enc_size_bytes
+            row.preview_enabled = bool(item.get('preview_enabled')) if item.get('preview_enabled') is not None else row.preview_enabled
+            if item.get('preview_conf'):
+                row.preview_conf = item['preview_conf']
+            if item.get('salt_b64'):
+                row.salt_b64 = item['salt_b64']
+        await session.commit()
+
+    # Fetch thumbnail via HTTP if provided and not present locally
+    cover_url = item.get('cover_url')
+    if cover_url:
+        try:
+            async with db_session() as session:
+                ec = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+                have_thumb = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id, ContentDerivative.kind == 'decrypted_thumbnail', ContentDerivative.status == 'ready'))).scalars().first()
+                if not have_thumb:
+                    import httpx, tempfile, os
+                    async with httpx.AsyncClient(timeout=30) as client:
+                        r = await client.get(cover_url)
+                        r.raise_for_status()
+                        tmp = tempfile.NamedTemporaryFile(delete=False)
+                        tmp.write(r.content)
+                        tmp.close()
+                    # Save into store
+                    from app.core.background.convert_v3_service import _save_derivative
+                    h, size = await _save_derivative(tmp.name, os.path.basename(cover_url) or 'thumb.jpg')
+                    cd = ContentDerivative(
+                        content_id=ec.id,
+                        kind='decrypted_thumbnail',
+                        local_path=os.path.join(os.getenv('UPLOADS_DIR', '/app/data'), h),
+                        content_type=r.headers.get('Content-Type') or 'image/jpeg',
+                        size_bytes=size,
+                        status='ready',
+                    )
+                    session.add(cd)
+                    await session.commit()
+        except Exception as e:
+            make_log('index_scout_v3', f"thumbnail fetch failed for {cid}: {e}", level='warning')
+
+
+async def main_fn(memory):
+    make_log('index_scout_v3', 'Service started', level='info')
+    sem = asyncio.Semaphore(PIN_CONCURRENCY)
+    while True:
+        try:
+            async with db_session() as session:
+                nodes = (await session.execute(select(KnownNode))).scalars().all()
+            for n in nodes:
+                base = f"http://{n.ip}:{n.port}"
+                # jitter 0..30s per node to reduce stampede
+                await asyncio.sleep(random.uniform(0, 30))
+                etag = (n.meta or {}).get('index_etag')
+                since = (n.meta or {}).get('index_since')
+                items, marker = await fetch_index(base, etag, since)
+                if not items and marker == etag:
+                    continue
+                # update node markers
+                try:
+                    async with db_session() as session:
+                        row = (await session.execute(select(KnownNode).where(KnownNode.id == n.id))).scalars().first()
+                        if row:
+                            meta = row.meta or {}
+                            meta['index_etag'] = marker
+                            meta['index_since'] = marker if (marker and 'T' in str(marker)) else meta.get('index_since')
+                            row.meta = meta
+                            await session.commit()
+                except Exception:
+                    pass
+
+                if not items:
+                    continue
+                make_log('index_scout_v3', f"Fetched {len(items)} from {base}")
+
+                # Check disk watermark
+                try:
+                    from app.core._config import UPLOADS_DIR
+                    du = shutil.disk_usage(UPLOADS_DIR)
+                    used_pct = int(100 * (1 - du.free / du.total))
+                    if used_pct >= DISK_WATERMARK_PCT:
+                        make_log('index_scout_v3', f"Disk watermark reached ({used_pct}%), skipping pins")
+                        continue
+                except Exception:
+                    pass
+
+                async def _pin_one(cid: str):
+                    async with sem:
+                        try:
+                            # Try to pre-connect to discovered providers
+                            try:
+                                provs = await find_providers(cid, max_results=5)
+                                for p in provs:
+                                    for addr in (p.get('addrs') or [])[:2]:
+                                        try:
+                                            await swarm_connect(addr)
+                                        except Exception:
+                                            pass
+                            except Exception:
+                                pass
+                            await pin_add(cid, recursive=True)
+                        except Exception as e:
+                            make_log('index_scout_v3', f"pin {cid} failed: {e}", level='warning')
+
+                tasks = []
+                for it in items:
+                    await upsert_content(it)
+                    cid = it.get('encrypted_cid')
+                    if cid:
+                        tasks.append(asyncio.create_task(_pin_one(cid)))
+                if tasks:
+                    await asyncio.gather(*tasks)
+        except Exception as e:
+            make_log('index_scout_v3', f"loop error: {e}", level='error')
+        await asyncio.sleep(INTERVAL_SEC)