improve work

app/core/_utils/resolve_content.py

@@ -2,8 +2,9 @@ from app.core.content.content_id import ContentId
 
 
 def resolve_content(content_id) -> ContentId:  # -> [content, error]
+    if isinstance(content_id, ContentId):
+        return content_id, None
     try:
         return ContentId.deserialize(content_id), None
     except BaseException as e:
         return None, f"{e}"
-

app/core/_utils/share_links.py

@@ -0,0 +1,24 @@
+from typing import Optional
+from urllib.parse import urlencode
+
+STARTAPP_SEPARATOR = '!'
+STARTAPP_LIMIT = 64
+
+
+def build_content_links(content_token: str, ref_id: Optional[str], *, project_host: str, bot_username: str):
+    """Return tuple of (startapp_payload, telegram_url, web_url)."""
+    payload = content_token
+    short_ref = (ref_id or '').strip()[:3]
+    if short_ref:
+        candidate = f"{content_token}{STARTAPP_SEPARATOR}{short_ref}"
+        if len(candidate) <= STARTAPP_LIMIT:
+            payload = candidate
+
+    telegram_url = f"https://t.me/{bot_username}/content?startapp={payload}"
+
+    query = [('content', content_token)]
+    if ref_id:
+        query.append(('ref', ref_id))
+    web_url = f"{project_host}/viewContent?{urlencode(query)}"
+
+    return payload, telegram_url, web_url

app/core/background/convert_service.py

@@ -136,8 +136,7 @@ async def convert_loop(memory):
             ]
             if trim_value:
                 cmd.extend(["--trim", trim_value])
-            if content_kind == "audio":
-                cmd.append("--audio-only")  # audio-only flag
+            # converter auto-detects audio/video, no explicit flag required
 
             process = await asyncio.create_subprocess_exec(
                 *cmd,

app/core/background/convert_v3_service.py

@@ -2,8 +2,11 @@ import asyncio
 import os
 import json
 import shutil
+import tempfile
+from dataclasses import dataclass
 from datetime import datetime
-from typing import List, Tuple, Optional
+from pathlib import Path
+from typing import List, Optional, Tuple
 
 from sqlalchemy import select
 
@@ -22,9 +25,41 @@ from app.core.crypto.encf_stream import decrypt_encf_auto
 from app.core.crypto.keywrap import unwrap_dek, wrap_dek, KeyWrapError
 from app.core.network.key_client import request_key_from_peer
 from app.core.models.my_network import KnownNode
+from app.core._utils.resolve_content import resolve_content
+from app.core.content.content_id import ContentId
 
 
 CONCURRENCY = int(os.getenv("CONVERT_V3_MAX_CONCURRENCY", "3"))
+STAGING_SUBDIR = os.getenv("CONVERT_V3_STAGING_SUBDIR", "convert-staging")
+UPLOADS_PATH = Path(UPLOADS_DIR).resolve()
+_host_uploads_env = os.getenv("BACKEND_DATA_DIR_HOST")
+HOST_UPLOADS_PATH = Path(_host_uploads_env).resolve() if _host_uploads_env else None
+
+
+@dataclass
+class PlainStaging:
+    container_path: str
+    host_path: str
+
+
+def _container_to_host(path: str) -> str:
+    """Map a container path under UPLOADS_DIR to the host path for docker -v."""
+    if not HOST_UPLOADS_PATH:
+        raise RuntimeError("BACKEND_DATA_DIR_HOST is not configured for convert_v3")
+    real_path = Path(path).resolve()
+    try:
+        real_path.relative_to(UPLOADS_PATH)
+    except ValueError:
+        # Not under uploads; best effort fallback to original string
+        return str(real_path)
+    rel = real_path.relative_to(UPLOADS_PATH)
+    return str(HOST_UPLOADS_PATH / rel)
+
+
+MEDIA_CONVERTER_CPU_LIMIT = os.getenv("MEDIA_CONVERTER_CPU_LIMIT")
+MEDIA_CONVERTER_MEM_LIMIT = os.getenv("MEDIA_CONVERTER_MEM_LIMIT")
+MEDIA_CONVERTER_CPUSET = os.getenv("MEDIA_CONVERTER_CPUSET") or os.getenv("CONVERT_CPUSET")
+ERROR_TRUNCATE_LIMIT = 512
 
 
 def _ensure_dir(path: str):
@@ -57,27 +92,49 @@ async def _save_derivative(file_path: str, filename: str) -> Tuple[str, int]:
     return file_hash, size
 
 
-async def _run_media_converter(input_host_path: str, input_ext: str, quality: str, trim_value: Optional[str], is_audio: bool) -> Tuple[str, dict]:
+async def _run_media_converter(staging: PlainStaging, input_ext: str, quality: str, trim_value: Optional[str], is_audio: bool):
+    if not os.path.exists(staging.container_path):
+        raise FileNotFoundError(f"Plain input missing at {staging.container_path}")
+
+    host_input_path = staging.host_path
+    if not host_input_path or not host_input_path.startswith('/'):
+        host_input_path = os.path.abspath(host_input_path)
+
     rid = __import__('uuid').uuid4().hex[:8]
-    output_dir_container = f"/tmp/conv_{rid}"
-    output_dir_host = f"/tmp/conv_{rid}"
-    _ensure_dir(output_dir_host)
-    logs_dir_host = BACKEND_LOGS_DIR_HOST
-    _ensure_dir(logs_dir_host)
+    output_dir_container = UPLOADS_PATH / "convert-output" / f"conv_{rid}"
+    output_dir_host = _container_to_host(output_dir_container)
+    _ensure_dir(str(output_dir_container))
+
+    logs_dir_candidate = os.getenv("BACKEND_LOGS_DIR_HOST", "")
+    logs_dir_host = logs_dir_candidate if logs_dir_candidate else str(HOST_UPLOADS_PATH / "logs" / "converter") if HOST_UPLOADS_PATH else "/tmp/converter-logs"
+    if not logs_dir_host.startswith('/'):
+        logs_dir_host = os.path.join(os.getcwd(), logs_dir_host)
+    try:
+        os.makedirs(logs_dir_host, exist_ok=True)
+    except Exception:
+        fallback_logs = HOST_UPLOADS_PATH / "logs" / "converter" if HOST_UPLOADS_PATH else Path("/tmp/converter-logs")
+        logs_dir_host = str(fallback_logs)
+        os.makedirs(logs_dir_host, exist_ok=True)
 
     cmd = [
         "docker", "run", "--rm",
-        "-v", f"{input_host_path}:/app/input:ro",
+        "-v", f"{host_input_path}:/app/input:ro",
         "-v", f"{output_dir_host}:/app/output",
         "-v", f"{logs_dir_host}:/app/logs",
-        "media_converter",
-        "--ext", input_ext,
-        "--quality", quality,
     ]
+    if MEDIA_CONVERTER_CPU_LIMIT:
+        cmd.extend(["--cpus", str(MEDIA_CONVERTER_CPU_LIMIT)])
+    if MEDIA_CONVERTER_MEM_LIMIT:
+        cmd.extend(["--memory", str(MEDIA_CONVERTER_MEM_LIMIT)])
+    if MEDIA_CONVERTER_CPUSET:
+        cmd.extend(["--cpuset-cpus", MEDIA_CONVERTER_CPUSET])
+
+    cmd.append("media_converter")
+    cmd.extend(["--ext", input_ext, "--quality", quality])
     if trim_value:
         cmd.extend(["--trim", trim_value])
-    if is_audio:
-        cmd.append("--audio-only")
+
+    make_log('convert_v3', f"Run media_converter cmd: {' '.join(cmd)}")
 
     proc = await asyncio.create_subprocess_exec(
         *cmd,
@@ -90,15 +147,15 @@ async def _run_media_converter(input_host_path: str, input_ext: str, quality: st
 
     # Find produced media file and optional output.json
     try:
-        files = os.listdir(output_dir_host)
+        files = os.listdir(output_dir_container)
     except Exception as e:
         raise RuntimeError(f"Read output dir error: {e}")
     media_files = [f for f in files if f != "output.json"]
     if len(media_files) != 1:
         raise RuntimeError(f"Expected one media file, found {len(media_files)}: {media_files}")
-    output_media = os.path.join(output_dir_host, media_files[0])
+    output_media = os.path.join(output_dir_container, media_files[0])
     ffprobe_meta = {}
-    out_json = os.path.join(output_dir_host, "output.json")
+    out_json = os.path.join(output_dir_container, "output.json")
     if os.path.exists(out_json):
         try:
             with open(out_json, 'r') as f:
@@ -108,24 +165,74 @@ async def _run_media_converter(input_host_path: str, input_ext: str, quality: st
     return output_media, ffprobe_meta
 
 
-async def _convert_content(ec: EncryptedContent, input_host_path: str):
-    content_kind = 'audio' if ec.content_type.startswith('audio/') else ('video' if ec.content_type.startswith('video/') else 'other')
-    if content_kind == 'other':
-        return
+async def _update_upload_session(ec: EncryptedContent, all_success: bool, errors: List[str]):
+    async with db_session() as session:
+        upload_row = (await session.execute(
+            select(UploadSession).where(UploadSession.encrypted_cid == ec.encrypted_cid)
+        )).scalars().first()
+        if upload_row:
+            if all_success:
+                upload_row.state = 'converted'
+                upload_row.error = None
+            elif upload_row.state != 'converted':
+                upload_row.state = 'conversion_failed'
+                if errors:
+                    upload_row.error = _short_error(errors[0])
+            await session.commit()
 
+
+async def _convert_content(ec: EncryptedContent, staging: PlainStaging):
+    content_kind = 'audio' if ec.content_type.startswith('audio/') else ('video' if ec.content_type.startswith('video/') else 'other')
     input_ext = (ec.content_type.split('/')[-1] or 'bin')
     is_audio = content_kind == 'audio'
-    # Required outputs
-    required = ['high', 'low', 'low_preview']
+    encrypted_hash_b58 = ContentId.deserialize(ec.encrypted_cid).content_hash_b58
 
-    # Preview interval
+    if content_kind == 'other':
+        errors: List[str] = []
+        all_success = True
+        try:
+            file_hash, size_bytes = await _save_derivative(staging.container_path, staging.container_path)
+            plain_path = os.path.join(UPLOADS_DIR, file_hash)
+            plain_filename = f"{ec.encrypted_cid}.{input_ext}" if input_ext else ec.encrypted_cid
+            async with db_session() as session:
+                existing = (await session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first()
+                if not existing:
+                    sc = StoredContent(
+                        type="local/content_bin",
+                        hash=file_hash,
+                        user_id=None,
+                        filename=plain_filename,
+                        meta={'encrypted_cid': ec.encrypted_cid, 'kind': 'original'},
+                        created=datetime.utcnow(),
+                    )
+                    session.add(sc)
+                    await session.flush()
+                derivative = ContentDerivative(
+                    content_id=ec.id,
+                    kind='decrypted_original',
+                    local_path=plain_path,
+                    content_type=ec.content_type,
+                    size_bytes=size_bytes,
+                    status='ready',
+                )
+                session.add(derivative)
+                await session.commit()
+            make_log('convert_v3', f"Stored original derivative for {ec.encrypted_cid}")
+        except Exception as e:
+            all_success = False
+            errors.append(str(e))
+            make_log('convert_v3', f"Convert error {ec.encrypted_cid} opt=original: {e}", level='error')
+        await _update_upload_session(ec, all_success, errors)
+        return
+
+    # audio/video path
+    required = ['high', 'low', 'low_preview']
     conf = ec.preview_conf or {}
     intervals = conf.get('intervals') or [[0, int(conf.get('duration_ms', 30000))]]
     main_interval = intervals[0]
-    trim_value = None
     start_s = max(0, int(main_interval[0]) // 1000)
     dur_s = max(1, int((main_interval[1] - main_interval[0]) // 1000) or 30)
-    trim_value = f"{start_s},{dur_s}"
+    trim_value = f"{start_s}-{start_s + dur_s}"
 
     qualities = {
         'high': 'high',
@@ -133,81 +240,128 @@ async def _convert_content(ec: EncryptedContent, input_host_path: str):
         'low_preview': 'low',
     }
 
+    all_success = True
+    errors: List[str] = []
+
     for opt in required:
+        derivative_kind = f"decrypted_{opt if opt != 'low_preview' else 'preview'}"
+        derivative_id: Optional[int] = None
         try:
-            # Mark derivative processing
             async with db_session() as session:
                 cd = ContentDerivative(
                     content_id=ec.id,
-                    kind=f"decrypted_{opt if opt != 'low_preview' else 'preview'}",
+                    kind=derivative_kind,
                     interval_start_ms=main_interval[0] if opt == 'low_preview' else None,
                     interval_end_ms=main_interval[1] if opt == 'low_preview' else None,
                     local_path="",
                     status='processing',
                 )
                 session.add(cd)
+                await session.flush()
+                derivative_id = cd.id
                 await session.commit()
 
             out_path, ffprobe = await _run_media_converter(
-                input_host_path=input_host_path,
+                staging=staging,
                 input_ext=input_ext,
                 quality=qualities[opt],
                 trim_value=trim_value if opt == 'low_preview' else None,
                 is_audio=is_audio,
             )
 
-            # Save into store and StoredContent
             file_hash, size_bytes = await _save_derivative(out_path, os.path.basename(out_path))
 
             async with db_session() as session:
-                sc = StoredContent(
-                    type="local/content_bin",
-                    hash=file_hash,
-                    user_id=None,
-                    filename=os.path.basename(out_path),
-                    meta={'encrypted_cid': ec.encrypted_cid, 'kind': opt, 'ffprobe_meta': ffprobe},
-                    created=datetime.utcnow(),
-                )
-                session.add(sc)
-                await session.flush()
+                sc = (await session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first()
+                meta_payload = {'encrypted_cid': ec.encrypted_cid, 'kind': opt, 'ffprobe_meta': ffprobe}
+                if sc:
+                    sc.type = sc.type or "local/content_bin"
+                    sc.filename = os.path.basename(out_path)
+                    sc.meta = meta_payload
+                    sc.updated = datetime.utcnow()
+                else:
+                    sc = StoredContent(
+                        type="local/content_bin",
+                        hash=file_hash,
+                        user_id=None,
+                        filename=os.path.basename(out_path),
+                        meta=meta_payload,
+                        created=datetime.utcnow(),
+                    )
+                    session.add(sc)
+                    await session.flush()
 
-                # Update derivative record
-                cd = (await session.execute(select(ContentDerivative).where(
-                    ContentDerivative.content_id == ec.id,
-                    ContentDerivative.kind == (f"decrypted_{opt if opt != 'low_preview' else 'preview'}"),
-                    ContentDerivative.status == 'processing'
-                ))).scalars().first()
+                encrypted_sc = (await session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash_b58))).scalars().first()
+                if encrypted_sc:
+                    meta = dict(encrypted_sc.meta or {})
+                    converted = dict(meta.get('converted_content') or {})
+                    converted[opt] = file_hash
+                    meta['converted_content'] = converted
+                    encrypted_sc.meta = meta
+                    if opt == 'high':
+                        encrypted_sc.decrypted_content_id = sc.id
+                    encrypted_sc.updated = datetime.utcnow()
+
+                cd = await session.get(ContentDerivative, derivative_id) if derivative_id else None
                 if cd:
                     cd.local_path = os.path.join(UPLOADS_DIR, file_hash)
                     cd.size_bytes = size_bytes
-                    cd.content_type = ('audio/mpeg' if is_audio else 'video/mp4') if opt != 'high' else ec.content_type
+                    if is_audio:
+                        cd.content_type = 'audio/flac' if opt == 'high' else 'audio/mpeg'
+                    else:
+                        cd.content_type = ec.content_type if opt == 'high' else 'video/mp4'
                     cd.status = 'ready'
+                    cd.error = None
                 await session.commit()
 
+            output_parent = Path(out_path).parent
+            shutil.rmtree(output_parent, ignore_errors=True)
             make_log('convert_v3', f"Converted {ec.encrypted_cid} opt={opt} -> {file_hash}")
         except Exception as e:
             make_log('convert_v3', f"Convert error {ec.encrypted_cid} opt={opt}: {e}", level='error')
+            all_success = False
+            errors.append(_short_error(e))
             async with db_session() as session:
-                cd = ContentDerivative(
-                    content_id=ec.id,
-                    kind=f"decrypted_{opt if opt != 'low_preview' else 'preview'}",
-                    status='failed',
-                    error=str(e),
-                    local_path="",
-                )
-                session.add(cd)
+                cd = await session.get(ContentDerivative, derivative_id) if derivative_id else None
+                if cd:
+                    cd.status = 'failed'
+                    cd.error = _short_error(e)
+                else:
+                    session.add(ContentDerivative(
+                        content_id=ec.id,
+                        kind=derivative_kind,
+                        status='failed',
+                        error=_short_error(e),
+                        local_path="",
+                    ))
                 await session.commit()
 
+    await _update_upload_session(ec, all_success, errors)
 
-async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, str]]:
+
+async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, PlainStaging]]:
     async with db_session() as session:
         # Find A/V contents with preview_enabled and no ready low/low_preview derivatives yet
         ecs = (await session.execute(select(EncryptedContent).where(
             EncryptedContent.preview_enabled == True
         ).order_by(EncryptedContent.created_at.desc()))).scalars().all()
 
-        picked: List[Tuple[EncryptedContent, str]] = []
+        picked: List[Tuple[EncryptedContent, PlainStaging]] = []
         for ec in ecs:
+            try:
+                cid_obj, cid_err = resolve_content(ec.encrypted_cid)
+                if cid_err:
+                    make_log('convert_v3', f"Skip {ec.encrypted_cid}: resolve error {cid_err}", level='debug')
+                    continue
+                encrypted_hash_b58 = cid_obj.content_hash_b58
+            except Exception as exc:
+                make_log('convert_v3', f"Skip {ec.encrypted_cid}: resolve exception {exc}", level='warning')
+                continue
+
+            sc = (await session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash_b58))).scalars().first()
+            if not sc or sc.onchain_index is None:
+                continue
+
             # Check if derivatives already ready
             rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id))).scalars().all()
             kinds_ready = {r.kind for r in rows if r.status == 'ready'}
@@ -215,11 +369,11 @@ async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, str]]:
             if required.issubset(kinds_ready):
                 continue
             # Always decrypt from IPFS using local or remote key
-            storage_path: Optional[str] = None
+            staging: Optional[PlainStaging] = None
             ck = (await session.execute(select(ContentKey).where(ContentKey.content_id == ec.id))).scalars().first()
             if ck:
-                storage_path = await stage_plain_from_ipfs(ec, ck.key_ciphertext_b64)
-            if not storage_path:
+                staging = await stage_plain_from_ipfs(ec, ck.key_ciphertext_b64)
+            if not staging:
                 peers = (await session.execute(select(KnownNode))).scalars().all()
                 for peer in peers:
                     base_url = f"http://{peer.ip}:{peer.port}"
@@ -240,12 +394,12 @@ async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, str]]:
                     )
                     session.add(session_ck)
                     await session.commit()
-                    storage_path = await stage_plain_from_ipfs(ec, dek_b64)
-                    if storage_path:
+                    staging = await stage_plain_from_ipfs(ec, dek_b64)
+                    if staging:
                         break
-            if not storage_path or not os.path.exists(storage_path):
+            if not staging or not os.path.exists(staging.container_path):
                 continue
-            picked.append((ec, storage_path))
+            picked.append((ec, staging))
             if len(picked) >= limit:
                 break
         return picked
@@ -254,14 +408,14 @@ async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, str]]:
 async def worker_loop():
     sem = asyncio.Semaphore(CONCURRENCY)
 
-    async def _run_one(ec: EncryptedContent, input_path: str):
+    async def _run_one(ec: EncryptedContent, staging: PlainStaging):
         async with sem:
             try:
-                await _convert_content(ec, input_path)
+                await _convert_content(ec, staging)
                 # After successful conversion, attempt to remove staging file to avoid duplicates
                 try:
-                    if input_path and input_path.startswith("/data/") and os.path.exists(input_path):
-                        os.remove(input_path)
+                    if staging and staging.container_path and os.path.exists(staging.container_path):
+                        os.remove(staging.container_path)
                 except Exception:
                     pass
             except Exception as e:
@@ -273,7 +427,7 @@ async def worker_loop():
             if not batch:
                 await asyncio.sleep(3)
                 continue
-            tasks = [asyncio.create_task(_run_one(ec, path)) for (ec, path) in batch]
+            tasks = [asyncio.create_task(_run_one(ec, staging)) for (ec, staging) in batch]
             await asyncio.gather(*tasks)
         except Exception as e:
             make_log('convert_v3', f"loop error: {e}", level='error')
@@ -285,15 +439,20 @@ async def main_fn(memory):
     await worker_loop()
 
 
-async def stage_plain_from_ipfs(ec: EncryptedContent, dek_wrapped: str) -> Optional[str]:
-    """Download encrypted ENCF stream from IPFS and decrypt on the fly into a temp file."""
-    import tempfile
+async def stage_plain_from_ipfs(ec: EncryptedContent, dek_wrapped: str) -> Optional[PlainStaging]:
+    """Download encrypted ENCF stream from IPFS and decrypt on the fly into shared staging."""
+    os.makedirs(UPLOADS_PATH / STAGING_SUBDIR, exist_ok=True)
     try:
         dek = unwrap_dek(dek_wrapped)
     except KeyWrapError as exc:
         make_log('convert_v3', f"unwrap failed for {ec.encrypted_cid}: {exc}", level='error')
         return None
-    tmp = tempfile.NamedTemporaryFile(prefix=f"dec_{ec.encrypted_cid[:8]}_", delete=False)
+
+    tmp = tempfile.NamedTemporaryFile(
+        prefix=f"dec_{ec.encrypted_cid[:8]}_",
+        dir=UPLOADS_PATH / STAGING_SUBDIR,
+        delete=False,
+    )
     tmp_path = tmp.name
     tmp.close()
     try:
@@ -301,7 +460,8 @@ async def stage_plain_from_ipfs(ec: EncryptedContent, dek_wrapped: str) -> Optio
             async for ch in cat_stream(ec.encrypted_cid):
                 yield ch
         await decrypt_encf_auto(_aiter(), dek, tmp_path)
-        return tmp_path
+        host_path = _container_to_host(tmp_path)
+        return PlainStaging(container_path=tmp_path, host_path=host_path)
     except Exception as e:
         make_log('convert_v3', f"decrypt from ipfs failed: {e}", level='error')
         try:
@@ -312,3 +472,8 @@ async def stage_plain_from_ipfs(ec: EncryptedContent, dek_wrapped: str) -> Optio
 
 
     
+def _short_error(message: str, limit: int = ERROR_TRUNCATE_LIMIT) -> str:
+    if not message:
+        return message
+    message = str(message)
+    return message if len(message) <= limit else message[: limit - 3] + '...'

app/core/background/derivative_cache_janitor.py

@@ -25,14 +25,14 @@ async def _evict_over_ttl(now: datetime) -> int:
     removed = 0
     # Pull TTL from ServiceConfig each time
     async with db_session() as session:
-        ttl_days = await ServiceConfig(session).get('DERIVATIVE_CACHE_TTL_DAYS', ENV_TTL_DAYS)
-    if int(ttl_days) <= 0:
+        ttl_days = int(await ServiceConfig(session).get('DERIVATIVE_CACHE_TTL_DAYS', ENV_TTL_DAYS))
+    if ttl_days <= 0:
         return 0
     async with db_session() as session:
         rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
         for r in rows:
             la = r.last_access_at or r.created_at
-            if la and (now - la) > timedelta(days=TTL_DAYS):
+            if la and (now - la) > timedelta(days=ttl_days):
                 try:
                     if r.local_path and os.path.exists(r.local_path):
                         os.remove(r.local_path)
@@ -80,7 +80,11 @@ async def _evict_to_fit():
 
 
 async def main_fn(memory):
-    make_log('derivative_janitor', f"Started (MAX_GB={MAX_GB}, TTL_DAYS={TTL_DAYS})", level='info')
+    async with db_session() as session:
+        cfg = ServiceConfig(session)
+        runtime_max_gb = float(await cfg.get('DERIVATIVE_CACHE_MAX_GB', ENV_MAX_GB))
+        runtime_ttl_days = int(await cfg.get('DERIVATIVE_CACHE_TTL_DAYS', ENV_TTL_DAYS))
+    make_log('derivative_janitor', f"Started (MAX_GB={runtime_max_gb}, TTL_DAYS={runtime_ttl_days})", level='info')
     while True:
         try:
             now = datetime.utcnow()

app/core/background/index_scout_v3.py

@@ -1,5 +1,6 @@
 import asyncio
-from typing import List
+import os
+from typing import List, Optional
 
 import httpx
 import random
@@ -18,7 +19,7 @@ ENV_PIN_CONCURRENCY = int(os.getenv('SYNC_MAX_CONCURRENT_PINS', '4'))
 ENV_DISK_WATERMARK_PCT = int(os.getenv('SYNC_DISK_LOW_WATERMARK_PCT', '90'))
 
 
-async def fetch_index(base_url: str, etag: str | None, since: str | None) -> tuple[List[dict], str | None]:
+async def fetch_index(base_url: str, etag: Optional[str], since: Optional[str]) -> tuple[List[dict], Optional[str]]:
     try:
         headers = {}
         params = {}

app/core/background/indexer_service.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 from base64 import b64decode
 from datetime import datetime
 
@@ -6,7 +7,7 @@ from base58 import b58encode
 from sqlalchemy import String, and_, desc, cast
 from tonsdk.boc import Cell
 from tonsdk.utils import Address
-from app.core._config import CLIENT_TELEGRAM_BOT_USERNAME
+from app.core._config import CLIENT_TELEGRAM_BOT_USERNAME, PROJECT_HOST
 from app.core._blockchain.ton.platform import platform
 from app.core._blockchain.ton.toncenter import toncenter
 from app.core._utils.send_status import send_status
@@ -18,6 +19,10 @@ from app.core._utils.resolve_content import resolve_content
 from app.core.models.wallet_connection import WalletConnection
 from app.core._keyboards import get_inline_keyboard
 from app.core.models._telegram import Wrapped_CBotChat
+from app.core._utils.share_links import build_content_links
+
+
+MIN_ONCHAIN_INDEX = int(os.getenv("MIN_ONCHAIN_INDEX", "8"))
 from sqlalchemy import select, and_, desc
 from app.core.storage import db_session
 import os
@@ -110,11 +115,15 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
         )).scalars().first()
         last_known_index = last_known_index_.onchain_index if last_known_index_ else 0
         last_known_index = max(last_known_index, 0)
+        if last_known_index < (MIN_ONCHAIN_INDEX - 1):
+            make_log(
+                "Indexer",
+                f"Adjusting last_known_index from {last_known_index} to {MIN_ONCHAIN_INDEX - 1} (MIN_ONCHAIN_INDEX)",
+                level="debug"
+            )
+            last_known_index = MIN_ONCHAIN_INDEX - 1
         make_log("Indexer", f"Last known index: {last_known_index}", level="debug")
-        if last_known_index_:
-            next_item_index = last_known_index + 1
-        else:
-            next_item_index = 0
+        next_item_index = last_known_index + 1
 
         resolve_item_result = await toncenter.run_get_method(platform.address.to_string(1, 1, 1), 'get_nft_address_by_index', [['num', next_item_index]])
         make_log("Indexer", f"Resolve item result: {resolve_item_result}", level="debug")
@@ -141,6 +150,13 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
 
         assert item_get_data_result['stack'][2][0] == 'num', "Item index is not a number"
         item_index = int(item_get_data_result['stack'][2][1], 16)
+        if item_index < MIN_ONCHAIN_INDEX:
+            make_log(
+                "Indexer",
+                f"Skip on-chain item {item_index}: below MIN_ONCHAIN_INDEX={MIN_ONCHAIN_INDEX}",
+                level="info"
+            )
+            return platform_found, seqno
         assert item_index == next_item_index, "Item index mismatch"
 
         item_platform_address = Cell.one_from_boc(b64decode(item_get_data_result['stack'][3][1]['bytes'])).begin_parse().read_msg_addr()
@@ -222,17 +238,32 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
 
                     if user:
                         user_uploader_wrapper = Wrapped_CBotChat(memory._telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)
+                        ref_id = (user.meta or {}).get('ref_id')
+                        if not ref_id:
+                            ref_id = user.ensure_ref_id()
+                            await session.commit()
+
+                        _, startapp_url, web_url = build_content_links(
+                            encrypted_stored_content.cid.serialize_v2(),
+                            ref_id,
+                            project_host=PROJECT_HOST,
+                            bot_username=CLIENT_TELEGRAM_BOT_USERNAME
+                        )
+
+                        message_text = user.translated('p_contentWasIndexed').format(
+                            item_address=item_address.to_string(1, 1, 1),
+                            item_index=item_index,
+                        )
+                        message_text += f"\n\n<a href=\"{web_url}\">🔗 Открыть контент</a>"
+
                         await user_uploader_wrapper.send_message(
-                            user.translated('p_contentWasIndexed').format(
-                                item_address=item_address.to_string(1, 1, 1),
-                                item_index=item_index,
-                            ),
+                            message_text,
                             message_type='notification',
                             reply_markup=get_inline_keyboard([
                                 [{
                                     'text': user.translated('viewTrackAsClient_button'),
-                                    'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}?start=C{encrypted_stored_content.cid.serialize_v2()}"
-                                }],
+                                    'url': startapp_url
+                                }]
                             ])
                         )
 
@@ -263,6 +294,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
                     **encrypted_stored_content.meta,
                     **item_metadata_packed
                 }
+                encrypted_stored_content.content_id = item_content_cid_str
 
                 await session.commit()
                 return platform_found, seqno
@@ -283,6 +315,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
             encrypted=True,
             decrypted_content_id=None,
             key_id=None,
+            content_id=item_content_cid_str,
             updated=datetime.now()
         )
         session.add(onchain_stored_content)

app/core/background/license_service.py

@@ -100,6 +100,7 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
         process_content = (await session.execute(select(UserContent).where(
             and_(
                 UserContent.type.startswith('nft/'),
+                UserContent.type != 'nft/ignored',
                 UserContent.updated < (datetime.now() - timedelta(minutes=60)),
             )
         ).order_by(UserContent.updated.asc()))).scalars().first()

app/core/content/content_id.py

@@ -1,3 +1,6 @@
+from base64 import b32decode
+from typing import Optional, Tuple
+
 from base58 import b58encode, b58decode
 
 from tonsdk.boc import begin_cell
@@ -12,25 +15,50 @@ from app.core._utils.string_binary import string_to_bytes_fixed_size, bytes_to_s
 # cid_v2#_ cid_version:uint8 content_sha256:uint256 *[Param]s = CIDv2;
 
 class ContentId:
+    """Unified abstraction for legacy ContentID and ENCF/IPFS CID strings."""
+
     def __init__(
         self,
-        version: int = None,
-        content_hash: bytes = None,  # only SHA256
-        onchain_index: int = None,
-        accept_type: str = None,
-        encryption_key_sha256: bytes = None,
+        version: Optional[int] = None,
+        content_hash: Optional[bytes] = None,  # only SHA256
+        onchain_index: Optional[int] = None,
+        accept_type: Optional[str] = None,
+        encryption_key_sha256: Optional[bytes] = None,
+        *,
+        raw_value: Optional[str] = None,
+        cid_format: Optional[str] = None,
+        multibase_prefix: Optional[str] = None,
+        multicodec: Optional[int] = None,
+        multihash_code: Optional[int] = 0x12,
+        multihash_length: Optional[int] = 32,
     ):
         self.version = version
         self.content_hash = content_hash
 
-        self.onchain_index = onchain_index or -1
+        self.onchain_index = onchain_index if onchain_index is not None else -1
         self.accept_type = accept_type
         self.encryption_key_sha256 = encryption_key_sha256
         if self.encryption_key_sha256:
             assert len(self.encryption_key_sha256) == 32, "Invalid encryption key length"
 
+        self._raw_value = raw_value
+        if cid_format:
+            self.cid_format = cid_format
+        else:
+            if self.version == 1:
+                self.cid_format = 'content_id_v1'
+            elif self.version == 2:
+                self.cid_format = 'content_id_v2'
+            else:
+                self.cid_format = 'content_id_v2'
+        self.multibase_prefix = multibase_prefix
+        self.multicodec = multicodec
+        self.multihash_code = multihash_code
+        self.multihash_length = multihash_length
+
     @property
     def content_hash_b58(self) -> str:
+        assert self.content_hash, "Content hash is not set"
         return b58encode(self.content_hash).decode()
 
     @property
@@ -38,6 +66,11 @@ class ContentId:
         return self.onchain_index if (not (self.onchain_index is None) and self.onchain_index >= 0) else None
 
     def serialize_v2(self, include_accept_type=False) -> str:
+        if self.cid_format == 'ipfs':
+            if self._raw_value:
+                return self._raw_value
+            return self._serialize_ipfs()
+
         cid_bin = (
             (2).to_bytes(1, 'big')  # cid version
             + self.content_hash
@@ -60,6 +93,8 @@ class ContentId:
         return b58encode(cid_bin).decode()
 
     def serialize_v1(self) -> str:
+        if self.cid_format == 'ipfs':
+            raise ValueError("Cannot serialize IPFS CID as ContentId v1")
         at_bin = string_to_bytes_fixed_size(self.accept_type, 15)
         assert len(self.content_hash) == 32, "Invalid hash length"
         if self.onchain_index < 0:
@@ -133,13 +168,31 @@ class ContentId:
 
     @classmethod
     def deserialize(cls, cid: str):
-        cid_version = int.from_bytes(b58decode(cid)[0:1], 'big')
+        if not cid:
+            raise ValueError("Empty content id provided")
+
+        first_char = cid[0]
+        if first_char in ('b', 'B', 'z', 'Z'):
+            return cls.from_ipfs(cid)
+
+        try:
+            cid_version = int.from_bytes(b58decode(cid)[0:1], 'big')
+        except Exception:
+            cid_version = None
+
         if cid_version == 1:
-            return cls.from_v1(cid)
-        elif cid_version == 2:
-            return cls.from_v2(cid)
-        else:
-            raise ValueError("Invalid cid version")
+            obj = cls.from_v1(cid)
+            obj._raw_value = cid
+            return obj
+        if cid_version == 2:
+            obj = cls.from_v2(cid)
+            obj._raw_value = cid
+            return obj
+
+        try:
+            return cls.from_ipfs(cid)
+        except Exception as exc:
+            raise ValueError(f"Invalid cid format: {exc}") from exc
 
     def json_format(self):
         return {
@@ -147,7 +200,130 @@ class ContentId:
             "content_hash": self.content_hash_b58,
             "onchain_index": self.safe_onchain_index,
             "accept_type": self.accept_type,
-            "encryption_key_sha256": b58encode(self.encryption_key_sha256).decode() if self.encryption_key_sha256 else None
+            "encryption_key_sha256": b58encode(self.encryption_key_sha256).decode() if self.encryption_key_sha256 else None,
+            "format": self.cid_format,
+            "raw": self.serialize_v2() if self.cid_format == 'ipfs' else None,
         }
 
+    # --- helpers for IPFS/ENCF CID handling ---------------------------------
 
+    @staticmethod
+    def _decode_multibase(cid_str: str) -> Tuple[bytes, Optional[str]]:
+        prefix = cid_str[0]
+        if prefix in ('b', 'B'):
+            payload = cid_str[1:]
+            padding = (-len(payload)) % 8
+            decoded = b32decode(payload.upper() + ('=' * padding), casefold=True)
+            return decoded, prefix.lower()
+        if prefix in ('z', 'Z'):
+            return b58decode(cid_str[1:]), prefix.lower()
+        # CIDv0 without explicit prefix
+        return b58decode(cid_str), None
+
+    @staticmethod
+    def _read_varint(data: bytes, offset: int) -> Tuple[int, int]:
+        result = 0
+        shift = 0
+        while True:
+            if offset >= len(data):
+                raise ValueError("truncated varint")
+            byte = data[offset]
+            offset += 1
+            result |= (byte & 0x7F) << shift
+            if not (byte & 0x80):
+                break
+            shift += 7
+            if shift > 63:
+                raise ValueError("varint overflow")
+        return result, offset
+
+    @classmethod
+    def from_ipfs(cls, cid: str):
+        cid = cid.strip()
+        payload, multibase_prefix = cls._decode_multibase(cid)
+
+        idx = 0
+        version: Optional[int] = None
+        codec: Optional[int] = None
+
+        if multibase_prefix is not None:
+            version, idx = cls._read_varint(payload, idx)
+            if version not in (0, 1):
+                raise ValueError(f"unsupported CID version: {version}")
+            if version == 1:
+                codec, idx = cls._read_varint(payload, idx)
+            else:
+                codec = 0x70  # dag-pb default for CIDv0
+        else:
+            # CIDv0 without explicit version/codec
+            version = 0
+            codec = 0x70
+
+        multihash_code, idx = cls._read_varint(payload, idx)
+        multihash_length, idx = cls._read_varint(payload, idx)
+        digest = payload[idx:idx + multihash_length]
+        if len(digest) != multihash_length:
+            raise ValueError("truncated multihash digest")
+        if multihash_length != 32:
+            raise ValueError("unsupported multihash length (expected 32 bytes)")
+        if multihash_code != 0x12:
+            raise ValueError(f"unsupported multihash code: {hex(multihash_code)}")
+
+        return cls(
+            version=version,
+            content_hash=digest,
+            onchain_index=None,
+            accept_type=None,
+            encryption_key_sha256=None,
+            raw_value=cid,
+            cid_format='ipfs',
+            multibase_prefix=multibase_prefix,
+            multicodec=codec,
+            multihash_code=multihash_code,
+            multihash_length=multihash_length,
+        )
+
+    def _serialize_ipfs(self) -> str:
+        if not self.content_hash:
+            raise ValueError("Cannot serialize IPFS CID without content hash")
+        if self.multibase_prefix is None:
+            # default to CIDv0 (base58btc) dag-pb
+            multihash = self._encode_varint(self.multihash_code or 0x12) + self._encode_varint(self.multihash_length or len(self.content_hash)) + self.content_hash
+            return b58encode(multihash).decode()
+
+        version_bytes = self._encode_varint(self.version or 1)
+        codec_bytes = b''
+        if (self.version or 1) == 1:
+            codec_bytes = self._encode_varint(self.multicodec or 0x70)
+
+        multihash = (
+            version_bytes +
+            codec_bytes +
+            self._encode_varint(self.multihash_code or 0x12) +
+            self._encode_varint(self.multihash_length or len(self.content_hash)) +
+            self.content_hash
+        )
+
+        if self.multibase_prefix == 'z':
+            return 'z' + b58encode(multihash).decode()
+        if self.multibase_prefix == 'b':
+            from base64 import b32encode
+            encoded = b32encode(multihash).decode().rstrip('=').lower()
+            return 'b' + encoded
+        # Fallback to base58btc without prefix
+        return b58encode(multihash).decode()
+
+    @staticmethod
+    def _encode_varint(value: int) -> bytes:
+        if value < 0:
+            raise ValueError("varint cannot encode negative values")
+        out = bytearray()
+        while True:
+            to_write = value & 0x7F
+            value >>= 7
+            if value:
+                out.append(to_write | 0x80)
+            else:
+                out.append(to_write)
+                break
+        return bytes(out)

app/core/models/_telegram/templates/player.py

@@ -11,6 +11,7 @@ import json
 import urllib
 
 from app.core.models.transaction import StarsInvoice
+from app.core._utils.share_links import build_content_links
 
 
 class PlayerTemplates:
@@ -59,12 +60,26 @@ class PlayerTemplates:
                 cd_log += f"Can't get cover content: {e}. "
                 cover_content = None
             
+            share_target = user_existing_license.onchain_address if user_existing_license else content.cid.serialize_v2()
+            ref_id = (self.user.meta or {}).get('ref_id')
+            if not ref_id:
+                ref_id = self.user.ensure_ref_id()
+                if self.db_session:
+                    await self.db_session.commit()
+
+            _, startapp_url, web_app_url = build_content_links(
+                share_target,
+                ref_id,
+                project_host=PROJECT_HOST,
+                bot_username=CLIENT_TELEGRAM_BOT_USERNAME
+            )
+
             content_share_link = {
                 'text': self.user.translated('p_shareLinkContext').format(title=content_metadata_json.get('name', "")),
-                'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}/content?startapp={content.cid.serialize_v2()}"
+                'url': startapp_url,
+                'web_url': web_app_url,
+                'ref_id': ref_id
             }
-            if user_existing_license:
-                content_share_link['url'] = f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}/content?startapp={user_existing_license.onchain_address}"
 
             if cover_content:
                 template_kwargs['photo'] = URLInputFile(cover_content.web_url)
@@ -85,7 +100,8 @@ class PlayerTemplates:
 {content_hashtags}
 Этот контент был загружен в MY
 \t/ p2p content market /
-<blockquote><a href="{content_share_link['url']}">🔴 «открыть в MY»</a></blockquote>"""
+<blockquote><a href="{content_share_link['url']}">🔴 «открыть в MY»</a></blockquote>
+<blockquote><a href="{content_share_link['web_url']}">🌐 «открыть в браузере»</a></blockquote>"""
 
         make_log("TG-Player", f"Send content {content_type} ({content_encoding}) to chat {self._chat_id}. {cd_log}")
         kmsgs = (await self.db_session.execute(select(KnownTelegramMessage).where(

app/core/models/content/indexation_mixins.py

@@ -1,7 +1,9 @@
+import os
 import traceback
 
 import base58
-from sqlalchemy import and_, select
+from sqlalchemy import select
+from datetime import datetime
 
 from app.core.logger import make_log
 from app.core.models import StoredContent
@@ -42,6 +44,9 @@ class NodeStorageIndexationMixin:
     pass # async def fetch_onchain_metadata(self):
 
 
+MIN_ONCHAIN_INDEX = int(os.getenv("MIN_ONCHAIN_INDEX", "8"))
+
+
 class UserContentIndexationMixin:
     async def sync_with_chain(self, db_session):
         errored = False
@@ -54,12 +59,29 @@ class UserContentIndexationMixin:
                 cc_indexator_data = unpack_item_indexator_data(cc_indexator_result)
                 assert cc_indexator_data['type'] == 1, "Type is not a content"
                 assert cc_indexator_data['address'] == self.onchain_address, "Address is not equal"
+                if cc_indexator_data['index'] < MIN_ONCHAIN_INDEX:
+                    make_log(
+                        "UserContent",
+                        f"Skip license {self.onchain_address} with index {cc_indexator_data['index']} < MIN_ONCHAIN_INDEX={MIN_ONCHAIN_INDEX}",
+                        level="info"
+                    )
+                    self.type = 'nft/ignored'
+                    self.content_id = None
+                    self.updated = datetime.utcnow()
+                    await db_session.commit()
+                    return
                 values_slice = cc_indexator_data['values'].begin_parse()
                 content_hash_b58 = base58.b58encode(bytes.fromhex(hex(values_slice.read_uint(256))[2:])).decode()
                 make_log("UserContent", f"License ({self.onchain_address}) content hash: {content_hash_b58}", level="info")
                 stored_content = (await db_session.execute(select(StoredContent).where(
-                    and_(StoredContent.type == 'onchain/content', StoredContent.hash == content_hash_b58)
+                    StoredContent.hash == content_hash_b58
                 ))).scalars().first()
+                if not stored_content:
+                    raise AssertionError(f"Stored content not found for hash={content_hash_b58}")
+                if not (stored_content.type or '').startswith('onchain/content'):
+                    stored_content.type = 'onchain/content' if stored_content.key_id else 'onchain/content_unknown'
+                    stored_content.onchain_index = stored_content.onchain_index or cc_indexator_data['index']
+                    stored_content.owner_address = stored_content.owner_address or cc_indexator_data['owner_address']
                 trusted_cop_address_result = await toncenter.run_get_method(stored_content.meta['item_address'], 'get_nft_address_by_index', [['num', cc_indexator_data['index']]])
                 assert trusted_cop_address_result.get('exit_code', -1) == 0, "Trusted cop address error"
                 trusted_cop_address = Cell.one_from_boc(b64decode(trusted_cop_address_result['stack'][0][1]['bytes'])).begin_parse().read_msg_addr().to_string(1, 1, 1)
@@ -77,5 +99,3 @@ class UserContentIndexationMixin:
             self.type = 'nft/unknown'
             self.content_id = None
             await db_session.commit()
-
-

app/core/models/node_storage.py

@@ -53,6 +53,11 @@ class StoredContent(AlchemyBase, AudioContentMixin):
 
     @property
     def cid(self) -> ContentId:
+        if self.content_id:
+            try:
+                return ContentId.deserialize(self.content_id)
+            except Exception as exc:
+                make_log("StoredContent", f"Failed to deserialize stored content_id '{self.content_id}': {exc}", level='warning')
         return ContentId(
             content_hash=b58decode(self.hash),
             onchain_index=self.onchain_index,

app/core/models/user/__init__.py

@@ -9,6 +9,10 @@ from app.core.translation import TranslationCore
 from ..base import AlchemyBase
 
 
+_BASE62_ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+_BASE62 = len(_BASE62_ALPHABET)
+
+
 class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1, WalletMixin):
     LOCALE_DOMAIN = 'sanic_telegram_bot'
 
@@ -32,3 +36,26 @@ class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1, W
     def __str__(self):
         return f"User, {self.id}_{self.telegram_id} | Username: {self.username} " + '\\'
 
+    def ensure_ref_id(self):
+        """Return a short referral identifier, generating it if missing."""
+        meta = self.meta or {}
+        ref_id = meta.get('ref_id')
+        if isinstance(ref_id, str) and ref_id:
+            return ref_id
+
+        ref_id = self._generate_ref_id()
+        self.meta = {**meta, 'ref_id': ref_id}
+        return ref_id
+
+    def _generate_ref_id(self):
+        user_id = int(self.id or 0)
+        if user_id <= 0:
+            return '000'
+
+        value = user_id % (_BASE62 ** 3)
+        chars = []
+        for _ in range(3):
+            chars.append(_BASE62_ALPHABET[value % _BASE62])
+            value //= _BASE62
+
+        return ''.join(reversed(chars)) or '000'

app/core/models/user/wallet_mixin.py

@@ -7,6 +7,11 @@ from tonsdk.utils import Address
 from datetime import datetime, timedelta
 from app.core.logger import make_log
 from httpx import AsyncClient
+from app.core.models.content.indexation_mixins import unpack_item_indexator_data, MIN_ONCHAIN_INDEX
+
+def _platform_address_str() -> str:
+    from app.core._blockchain.ton.platform import platform
+    return platform.address.to_string(1, 1, 1)
 
 
 class WalletMixin:
@@ -43,6 +48,43 @@ class WalletMixin:
             item_address = Address(nft_item['address']).to_string(1, 1, 1)
             owner_address = Address(nft_item['owner']['address']).to_string(1, 1, 1)
 
+            platform_address = _platform_address_str()
+            collection_address = None
+            if isinstance(nft_item, dict):
+                collection_data = nft_item.get('collection')
+                if isinstance(collection_data, dict):
+                    collection_address = collection_data.get('address')
+                collection_address = collection_address or nft_item.get('collection_address')
+            if collection_address:
+                try:
+                    normalized_collection = Address(collection_address).to_string(1, 1, 1)
+                except Exception:
+                    normalized_collection = collection_address
+                if normalized_collection != platform_address:
+                    make_log(self, f"Skip foreign NFT {item_address} from collection {normalized_collection}", level='debug')
+                    continue
+
+            item_index = None
+            # Prefer index from tonapi payload if available
+            raw_index = nft_item.get('index') if isinstance(nft_item, dict) else None
+            if isinstance(raw_index, int):
+                item_index = raw_index
+            if item_index is None:
+                try:
+                    indexator_raw = await toncenter.run_get_method(item_address, 'indexator_data')
+                    if indexator_raw.get('exit_code', -1) == 0:
+                        item_index = unpack_item_indexator_data(indexator_raw)['index']
+                except BaseException as err:
+                    make_log(self, f"Failed to fetch indexator data for {item_address}: {err}", level='warning')
+
+            if item_index is None:
+                make_log(self, f"Skip NFT {item_address}: unable to resolve on-chain index", level='warning')
+                continue
+
+            if item_index is not None and item_index < MIN_ONCHAIN_INDEX:
+                make_log(self, f"Ignore NFT {item_address} with index {item_index} < MIN_ONCHAIN_INDEX={MIN_ONCHAIN_INDEX}", level='debug')
+                continue
+
             from sqlalchemy import select
             user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
             if user_content:
@@ -83,6 +125,33 @@ class WalletMixin:
                     item_address = Address(nft_item['address']).to_string(1, 1, 1)
                     owner_address = Address(nft_item['owner_address']).to_string(1, 1, 1)
 
+                    platform_address = _platform_address_str()
+                    collection_address = nft_item.get('collection_address') if isinstance(nft_item, dict) else None
+                    if collection_address:
+                        try:
+                            normalized_collection = Address(collection_address).to_string(1, 1, 1)
+                        except Exception:
+                            normalized_collection = collection_address
+                        if normalized_collection != platform_address:
+                            make_log(self, f"Skip foreign NFT {item_address} from collection {normalized_collection}", level='debug')
+                            continue
+
+                    item_index = None
+                    try:
+                        indexator_raw = await toncenter.run_get_method(item_address, 'indexator_data')
+                        if indexator_raw.get('exit_code', -1) == 0:
+                            item_index = unpack_item_indexator_data(indexator_raw)['index']
+                    except BaseException as err:
+                        make_log(self, f"Failed to fetch indexator data for {item_address}: {err}", level='warning')
+
+                    if item_index is None:
+                        make_log(self, f"Skip NFT {item_address}: unable to resolve on-chain index", level='warning')
+                        continue
+
+                    if item_index is not None and item_index < MIN_ONCHAIN_INDEX:
+                        make_log(self, f"Ignore NFT {item_address} with index {item_index} < MIN_ONCHAIN_INDEX={MIN_ONCHAIN_INDEX}", level='debug')
+                        continue
+
                     from sqlalchemy import select
                     user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
                     if user_content:

app/core/projscale_logger.py

@@ -1,13 +1,15 @@
-from datetime import datetime
 import logging
 import time
 import httpx
 import threading
 import os
+from logging.handlers import TimedRotatingFileHandler
 
 PROJSCALE_APP_NAME = os.getenv('APP_PROJSCALE_NAME', 'my-uploader')
 LOGS_DIRECTORY = os.getenv('APP_LOGS_DIRECTORY', 'logs')
 os.makedirs(LOGS_DIRECTORY, exist_ok=True)
+LOG_FILE_BASENAME = os.getenv('APP_LOG_FILE_BASENAME', 'app.log')
+LOG_ROTATION_KEEP_HOURS = max(int(os.getenv('APP_LOG_ROTATION_KEEP_HOURS', '168')), 1)
 
 FORMAT_STRING = '%(asctime)s - %(levelname)s – %(pathname)s – %(funcName)s – %(lineno)d - %(message)s'
 
@@ -62,8 +64,14 @@ projscale_handler = ProjscaleLoggingHandler()
 projscale_handler.setLevel(logging.DEBUG)
 logger.addHandler(projscale_handler)
 
-log_filepath = f"{LOGS_DIRECTORY}/{datetime.now().strftime('%Y-%m-%d_%H')}.log"
-file_handler = logging.FileHandler(log_filepath)
+log_filepath = os.path.join(LOGS_DIRECTORY, LOG_FILE_BASENAME)
+file_handler = TimedRotatingFileHandler(
+    log_filepath,
+    when='H',
+    interval=1,
+    backupCount=LOG_ROTATION_KEEP_HOURS,
+    utc=False
+)
 file_handler.setLevel(logging.DEBUG)
 file_handler.setFormatter(logging.Formatter(FORMAT_STRING))
 logger.addHandler(file_handler)