sh startup

Dockerfile

@@ -12,7 +12,8 @@ RUN apt-get update && apt-get install -y \
     ca-certificates \
     curl \
     gnupg \
-    lsb-release && \
+    lsb-release \
+    ffmpeg && \
     install -m 0755 -d /etc/apt/keyrings && \
     curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \
     chmod a+r /etc/apt/keyrings/docker.asc && \

ENDPOINTS.md

@@ -48,6 +48,18 @@ TODO: реализовать поиск файла на других нодах
 16. GET /api/v1/content.view
 
 
+17. GET /api/v1/network.info
+Возвращает информацию о ноде: id/public_key, version, node_type, metrics, capabilities.
+
+18. GET /api/v1/network.nodes
+Возвращает список известных публичных нод с совместимостью и метаданными.
+
+19. POST /api/v1/network.handshake
+Рукопожатие между нодами. Тело запроса подписано приватным ключом ноды; ответ подписан приватным ключом сервера.
+Поля запроса: version, public_key (base58), node_type, metrics, capabilities, timestamp, nonce, signature.
+Поле public_host обязательно для public-нод, и опционально/пустое для private-нод.
+Поля ответа: compatibility, node, known_public_nodes, timestamp, server_public_key, server_signature (+ warning при несовпадении MINOR).
+Private-ноды не сохраняются на стороне принимающей ноды (никакого учета peer-а), но получают список публичных нод и могут синхронизироваться через них.
 
 
 

app/__main__.py

@@ -12,17 +12,12 @@ try:
 except BaseException:
     pass
 
-from app.core._utils.create_maria_tables import create_maria_tables
+from app.core._utils.create_maria_tables import create_db_tables
 from app.core.storage import engine
-if startup_target == '__main__':
+if startup_target != '__main__':
-    create_maria_tables(engine)
+    # Background services get a short delay before startup
-else:
     time.sleep(7)
 
-from app.api import app
-from app.bot import dp as uploader_bot_dp
-from app.client_bot import dp as client_bot_dp
-from app.core._config import SANIC_PORT, MYSQL_URI, PROJECT_HOST
 from app.core.logger import make_log
 
 if int(os.getenv("SANIC_MAINTENANCE", '0')) == 1:
@@ -52,7 +47,11 @@ async def execute_queue(app):
     make_log(None, f"Application normally started. HTTP port: {SANIC_PORT}")
     make_log(None, f"Telegram bot: https://t.me/{telegram_bot_username}")
     make_log(None, f"Client Telegram bot: https://t.me/{client_telegram_bot_username}")
-    make_log(None, f"MariaDB host: {MYSQL_URI.split('@')[1].split('/')[0].replace('/', '')}")
+    try:
+        _db_host = DATABASE_URL.split('@')[1].split('/')[0].replace('/', '')
+    except Exception:
+        _db_host = 'postgres://'
+    make_log(None, f"PostgreSQL host: {_db_host}")
     make_log(None, f"API host: {PROJECT_HOST}")
     while True:
         try:
@@ -81,16 +80,47 @@ async def execute_queue(app):
 if __name__ == '__main__':
     main_memory = Memory()
     if startup_target == '__main__':
+        # Defer heavy imports to avoid side effects in background services
+        # Mark this process as the primary node for seeding/config init
+        os.environ.setdefault('NODE_ROLE', 'primary')
+        # Create DB tables synchronously before importing HTTP app to satisfy _secrets
+        try:
+            from sqlalchemy import create_engine
+            from app.core.models import AlchemyBase  # imports all models
+            db_url = os.environ.get('DATABASE_URL')
+            if not db_url:
+                raise RuntimeError('DATABASE_URL is not set')
+            # Normalize to sync driver
+            if '+asyncpg' in db_url:
+                db_url_sync = db_url.replace('+asyncpg', '+psycopg2')
+            else:
+                db_url_sync = db_url
+            sync_engine = create_engine(db_url_sync, pool_pre_ping=True)
+            AlchemyBase.metadata.create_all(sync_engine)
+        except Exception as e:
+            make_log('Startup', f'DB sync init failed: {e}', level='error')
+        from app.api import app
+        from app.bot import dp as uploader_bot_dp
+        from app.client_bot import dp as client_bot_dp
+        from app.core._config import SANIC_PORT, PROJECT_HOST, DATABASE_URL
+        from app.core.network.nodes import network_handshake_daemon, bootstrap_once_and_exit_if_failed
+
         app.ctx.memory = main_memory
         for _target in [uploader_bot_dp, client_bot_dp]:
             _target._s_memory = app.ctx.memory
 
         app.ctx.memory._app = app
 
+        # Ensure DB schema exists using the same event loop as Sanic (idempotent)
+        app.add_task(create_db_tables(engine))
+
         app.add_task(execute_queue(app))
         app.add_task(queue_daemon(app))
         app.add_task(uploader_bot_dp.start_polling(app.ctx.memory._telegram_bot))
         app.add_task(client_bot_dp.start_polling(app.ctx.memory._client_telegram_bot))
+        # Start network handshake daemon and bootstrap step
+        app.add_task(network_handshake_daemon(app))
+        app.add_task(bootstrap_once_and_exit_if_failed())
 
         app.run(host='0.0.0.0', port=SANIC_PORT)
     else:
@@ -112,6 +142,15 @@ if __name__ == '__main__':
         elif startup_target == 'convert_process':
             from app.core.background.convert_service import main_fn as target_fn
             time.sleep(9)
+        elif startup_target == 'convert_v3':
+            from app.core.background.convert_v3_service import main_fn as target_fn
+            time.sleep(9)
+        elif startup_target == 'index_scout_v3':
+            from app.core.background.index_scout_v3 import main_fn as target_fn
+            time.sleep(7)
+        elif startup_target == 'derivative_janitor':
+            from app.core.background.derivative_cache_janitor import main_fn as target_fn
+            time.sleep(5)
 
         startup_fn = startup_fn or target_fn
         assert startup_fn
@@ -126,6 +165,7 @@ if __name__ == '__main__':
 
         loop = asyncio.get_event_loop()
         try:
+            # Background services no longer perform schema initialization
             loop.run_until_complete(wrapped_startup_fn(main_memory))
         except BaseException as e:
             make_log(startup_target[0].upper() + startup_target[1:], f"Error: {e}" + '\n' + str(traceback.format_exc()),

app/api/__init__.py

@@ -1,6 +1,8 @@
 import traceback
 
 from sanic import Sanic, response
+from uuid import uuid4
+import traceback as _traceback
 
 from app.core.logger import make_log
 
@@ -13,16 +15,45 @@ app.register_middleware(close_db_session, "response")
 
 from app.api.routes._index import s_index, s_favicon
 from app.api.routes._system import s_api_v1_node, s_api_system_version, s_api_system_send_status, s_api_v1_node_friendly
+from app.api.routes.network import (
+    s_api_v1_network_info,
+    s_api_v1_network_nodes,
+    s_api_v1_network_handshake,
+)
 from app.api.routes.auth import s_api_v1_auth_twa, s_api_v1_auth_select_wallet, s_api_v1_auth_me
 from app.api.routes.statics import s_api_tonconnect_manifest, s_api_platform_metadata
 from app.api.routes.node_storage import s_api_v1_storage_post, s_api_v1_storage_get, \
     s_api_v1_storage_decode_cid
 from app.api.routes.progressive_storage import s_api_v1_5_storage_get, s_api_v1_5_storage_post
+from app.api.routes.upload_tus import s_api_v1_upload_tus_hook
 from app.api.routes.account import s_api_v1_account_get
 from app.api.routes._blockchain import s_api_v1_blockchain_send_new_content_message, \
     s_api_v1_blockchain_send_purchase_content_message
 from app.api.routes.content import s_api_v1_content_list, s_api_v1_content_view, s_api_v1_content_friendly_list, s_api_v1_5_content_list
+from app.api.routes.content_index import s_api_v1_content_index, s_api_v1_content_delta
+from app.api.routes.derivatives import s_api_v1_content_derivatives
+from app.api.routes.admin import (
+    s_api_v1_admin_blockchain,
+    s_api_v1_admin_cache_cleanup,
+    s_api_v1_admin_cache_setlimits,
+    s_api_v1_admin_licenses,
+    s_api_v1_admin_login,
+    s_api_v1_admin_logout,
+    s_api_v1_admin_node_setrole,
+    s_api_v1_admin_nodes,
+    s_api_v1_admin_overview,
+    s_api_v1_admin_stars,
+    s_api_v1_admin_status,
+    s_api_v1_admin_storage,
+    s_api_v1_admin_sync_setlimits,
+    s_api_v1_admin_system,
+    s_api_v1_admin_uploads,
+    s_api_v1_admin_users,
+)
 from app.api.routes.tonconnect import s_api_v1_tonconnect_new, s_api_v1_tonconnect_logout
+from app.api.routes.keys import s_api_v1_keys_request
+from app.api.routes.sync import s_api_v1_sync_pin, s_api_v1_sync_status
+from app.api.routes.upload_status import s_api_v1_upload_status
 
 
 app.add_route(s_index, "/", methods=["GET", "OPTIONS"])
@@ -32,6 +63,9 @@ app.add_route(s_api_v1_node, "/api/v1/node", methods=["GET", "OPTIONS"])
 app.add_route(s_api_v1_node_friendly, "/api/v1/nodeFriendly", methods=["GET", "OPTIONS"])
 app.add_route(s_api_system_version, "/api/system.version", methods=["GET", "OPTIONS"])
 app.add_route(s_api_system_send_status, "/api/system.sendStatus", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_network_info, "/api/v1/network.info", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_network_nodes, "/api/v1/network.nodes", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_network_handshake, "/api/v1/network.handshake", methods=["POST", "OPTIONS"])
 
 app.add_route(s_api_tonconnect_manifest, "/api/tonconnect-manifest.json", methods=["GET", "OPTIONS"])
 app.add_route(s_api_platform_metadata, "/api/platform-metadata.json", methods=["GET", "OPTIONS"])
@@ -59,22 +93,87 @@ app.add_route(s_api_v1_content_list, "/api/v1/content.list", methods=["GET", "OP
 app.add_route(s_api_v1_content_view, "/api/v1/content.view/<content_address>", methods=["GET", "OPTIONS"])
 app.add_route(s_api_v1_content_friendly_list, "/api/v1/content.friendlyList", methods=["GET", "OPTIONS"])
 app.add_route(s_api_v1_5_content_list, "/api/v1.5/content.list", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_content_index, "/api/v1/content.index", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_content_delta, "/api/v1/content.delta", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_content_derivatives, "/api/v1/content.derivatives", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_login, "/api/v1/admin.login", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_admin_logout, "/api/v1/admin.logout", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_admin_overview, "/api/v1/admin.overview", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_storage, "/api/v1/admin.storage", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_uploads, "/api/v1/admin.uploads", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_users, "/api/v1/admin.users", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_licenses, "/api/v1/admin.licenses", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_stars, "/api/v1/admin.stars", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_system, "/api/v1/admin.system", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_blockchain, "/api/v1/admin.blockchain", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_node_setrole, "/api/v1/admin.node.setRole", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_admin_nodes, "/api/v1/admin.nodes", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_status, "/api/v1/admin.status", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_cache_setlimits, "/api/v1/admin.cache.setLimits", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_admin_cache_cleanup, "/api/v1/admin.cache.cleanup", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_admin_sync_setlimits, "/api/v1/admin.sync.setLimits", methods=["POST", "OPTIONS"])
+
+# tusd HTTP hooks
+app.add_route(s_api_v1_upload_tus_hook, "/api/v1/upload.tus-hook", methods=["POST", "OPTIONS"])
+
+# Keys auto-grant
+app.add_route(s_api_v1_keys_request, "/api/v1/keys.request", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_sync_pin, "/api/v1/sync.pin", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_sync_status, "/api/v1/sync.status", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_upload_status, "/api/v1/upload.status/<upload_id>", methods=["GET", "OPTIONS"])
 
 
 @app.exception(BaseException)
 async def s_handle_exception(request, exception):
-    response_buffer = response.json({"error": "An internal server error occurred"}, status=500)
+    # Correlate error to request
+    session_id = getattr(request.ctx, 'session_id', None) or uuid4().hex[:16]
+    error_id = uuid4().hex[:8]
+
+    status = 500
+    code = type(exception).__name__
+    message = "Internal HTTP Error"
+
     try:
         raise exception
     except AssertionError as e:
-        response_buffer = response.json({"error": str(e)}, status=400)
+        status = 400
+        code = 'AssertionError'
+        message = str(e) or 'Bad Request'
     except BaseException as e:
-        make_log("sanic_exception", f"Exception: {e}" + '\n' + str(traceback.format_exc()), level='error')
+        # keep default 500, but expose exception message to aid debugging
+        message = str(e) or message
 
+    # Build structured log with full context and traceback
+    try:
+        tb = _traceback.format_exc()
+        user_id = getattr(getattr(request.ctx, 'user', None), 'id', None)
+        log_ctx = {
+            'sid': session_id,
+            'eid': error_id,
+            'path': request.path,
+            'method': request.method,
+            'query': dict(request.args) if hasattr(request, 'args') else {},
+            'user_id': user_id,
+            'remote': (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip),
+            'code': code,
+            'message': message,
+            'traceback': tb,
+        }
+        make_log('http_exception', 'API exception', level='error', **log_ctx)
+    except BaseException:
+        pass
+
+    # Return enriched error response for the client
+    payload = {
+        'error': True,
+        'code': code,
+        'message': message,
+        'session_id': session_id,
+        'error_id': error_id,
+        'path': request.path,
+        'method': request.method,
+    }
+
+    response_buffer = response.json(payload, status=status)
     response_buffer = await close_db_session(request, response_buffer)
-    response_buffer.headers["Access-Control-Allow-Origin"] = "*"
-    response_buffer.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
-    response_buffer.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site"
-    response_buffer.headers["Access-Control-Allow-Credentials"] = "true"
     return response_buffer
-

app/api/middleware.py

@@ -1,5 +1,7 @@
+import os
 from base58 import b58decode
 from sanic import response as sanic_response
+from uuid import uuid4
 
 from app.core._crypto.signer import Signer
 from app.core._secrets import hot_seed
@@ -8,17 +10,37 @@ from app.core.models.keys import KnownKey
 from app.core.models._telegram.wrapped_bot import Wrapped_CBotChat
 from app.core.models.user_activity import UserActivity
 from app.core.models.user import User
-from app.core.storage import Session
+from sqlalchemy import select
+from app.core.storage import new_session
 from datetime import datetime, timedelta
+from app.core.log_context import (
+    ctx_session_id, ctx_user_id, ctx_method, ctx_path, ctx_remote
+)
 
 
-def attach_headers(response):
+ENABLE_INTERNAL_CORS = os.getenv("ENABLE_INTERNAL_CORS", "1").lower() in {"1", "true", "yes"}
-    response.headers["Access-Control-Allow-Origin"] = "*"
+
-    response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
+
-    response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id"
+def attach_headers(response, request=None):
-    # response.headers["Access-Control-Allow-Credentials"] = "true"
+    response.headers.pop("Access-Control-Allow-Origin", None)
+    response.headers.pop("Access-Control-Allow-Methods", None)
+    response.headers.pop("Access-Control-Allow-Headers", None)
+    response.headers.pop("Access-Control-Allow-Credentials", None)
+
+    if not ENABLE_INTERNAL_CORS:
         return response
 
+    response.headers["Access-Control-Allow-Origin"] = "*"
+    response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS, PATCH, HEAD"
+    response.headers["Access-Control-Allow-Headers"] = (
+        "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, "
+        "Sec-Fetch-Site, Tus-Resumable, tus-resumable, Upload-Length, upload-length, Upload-Offset, upload-offset, "
+        "Upload-Metadata, upload-metadata, Upload-Defer-Length, upload-defer-length, Upload-Concat, upload-concat, "
+        "x-file-name, x-last-chunk, x-chunk-start, x-upload-id, x-request-id"
+    )
+    return response
+
+
 
 async def try_authorization(request):
     token = request.headers.get("Authorization")
@@ -30,7 +52,8 @@ async def try_authorization(request):
         make_log("auth", "Invalid token length", level="warning")
         return
 
-    known_key = request.ctx.db_session.query(KnownKey).filter(KnownKey.seed == token).first()
+    result = await request.ctx.db_session.execute(select(KnownKey).where(KnownKey.seed == token))
+    known_key = result.scalars().first()
     if not known_key:
         make_log("auth", "Unknown key", level="warning")
         return
@@ -58,7 +81,8 @@ async def try_authorization(request):
         make_log("auth", f"User ID mismatch: {known_key.meta.get('I_user_id', -1)} != {user_id}", level="warning")
         return
 
-    user = request.ctx.db_session.query(User).filter(User.id == known_key.meta['I_user_id']).first()
+    result = await request.ctx.db_session.execute(select(User).where(User.id == known_key.meta['I_user_id']))
+    user = result.scalars().first()
     if not user:
         make_log("auth", "No user from key", level="warning")
         return
@@ -118,7 +142,14 @@ async def save_activity(request):
         pass
 
     try:
-        activity_meta["headers"] = dict(request.headers)
+        # Sanitize sensitive headers
+        headers = dict(request.headers)
+        for hk in list(headers.keys()):
+            if str(hk).lower() in [
+                'authorization', 'cookie', 'x-service-signature', 'x-message-hash'
+            ]:
+                headers[hk] = '<redacted>'
+        activity_meta["headers"] = headers
     except:
         pass
 
@@ -127,23 +158,51 @@ async def save_activity(request):
         meta=activity_meta,
         user_id=request.ctx.user.id if request.ctx.user else None,
         user_ip=activity_meta.get("ip", "0.0.0.0"),
-        created=datetime.now()
+        created=datetime.utcnow()
     )
     request.ctx.db_session.add(new_user_activity)
-    request.ctx.db_session.commit()
+    await request.ctx.db_session.commit()
 
 
 async def attach_user_to_request(request):
     if request.method == 'OPTIONS':
-        return attach_headers(sanic_response.text("OK"))
+        return attach_headers(sanic_response.text("OK"), request)
 
-    request.ctx.db_session = Session()
+    request.ctx.db_session = new_session()
     request.ctx.verified_hash = None
     request.ctx.user = None
     request.ctx.user_key = None
     request.ctx.user_uploader_wrapper = Wrapped_CBotChat(request.app.ctx.memory._telegram_bot, db_session=request.ctx.db_session)
     request.ctx.user_client_wrapper = Wrapped_CBotChat(request.app.ctx.memory._client_telegram_bot, db_session=request.ctx.db_session)
+    # Correlation/session id for this request: prefer proxy-provided X-Request-ID
+    incoming_req_id = request.headers.get('X-Request-Id') or request.headers.get('X-Request-ID')
+    request.ctx.session_id = (incoming_req_id or uuid4().hex)[:32]
+    # Populate contextvars for automatic logging context
+    try:
+        ctx_session_id.set(request.ctx.session_id)
+        ctx_method.set(request.method)
+        ctx_path.set(request.path)
+        _remote = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip)
+        if _remote and isinstance(_remote, str) and ',' in _remote:
+            _remote = _remote.split(',')[0].strip()
+        ctx_remote.set(_remote)
+    except BaseException:
+        pass
+    try:
+        make_log(
+            "HTTP",
+            f"Request start sid={request.ctx.session_id} {request.method} {request.path}",
+            level='info'
+        )
+    except BaseException:
+        pass
     await try_authorization(request)
+    # Update user_id in context after auth
+    try:
+        if request.ctx.user and request.ctx.user.id:
+            ctx_user_id.set(request.ctx.user.id)
+    except BaseException:
+        pass
     await save_activity(request)
     await try_service_authorization(request)
 
@@ -152,17 +211,34 @@ async def close_request_handler(request, response):
     if request.method == 'OPTIONS':
         response = sanic_response.text("OK")
 
+    response = attach_headers(response, request)
+
     try:
-        request.ctx.db_session.close()
+        await request.ctx.db_session.close()
-    except BaseException as e:
+    except BaseException:
         pass
 
-    response = attach_headers(response)
+    try:
+        make_log(
+            "HTTP",
+            f"Request end sid={getattr(request.ctx, 'session_id', None)} {request.method} {request.path} status={getattr(response, 'status', None)}",
+            level='info'
+        )
+    except BaseException:
+        pass
 
     return request, response
 
 
 async def close_db_session(request, response):
     request, response = await close_request_handler(request, response)
-    response = attach_headers(response)
+    # Clear contextvars
+    try:
+        ctx_session_id.set(None)
+        ctx_user_id.set(None)
+        ctx_method.set(None)
+        ctx_path.set(None)
+        ctx_remote.set(None)
+    except BaseException:
+        pass
     return response

app/api/routes/_blockchain.py

@@ -3,11 +3,10 @@ from datetime import datetime
 import traceback
 
 from sanic import response
-from sqlalchemy import and_
+from sqlalchemy import and_, select, func
 from tonsdk.boc import begin_cell, begin_dict
 from tonsdk.utils import Address
 
-from base58 import b58encode
 from app.core._blockchain.ton.connect import TonConnect, wallet_obj_by_name
 from app.core._blockchain.ton.platform import platform
 from app.core._config import PROJECT_HOST
@@ -46,7 +45,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
         for field_key, field_value in {
             'title': lambda x: isinstance(x, str),
             'authors': lambda x: isinstance(x, list),
-            'content': lambda x: isinstance(x, str),
+            'content': lambda x: isinstance(x, str),  # may be plaintext CID (legacy) or encrypted IPFS CID (bafy...)
             'image': lambda x: isinstance(x, str),
             'description': lambda x: isinstance(x, str),
             'price': lambda x: (isinstance(x, str) and x.isdigit()),
@@ -57,26 +56,29 @@ async def s_api_v1_blockchain_send_new_content_message(request):
             assert field_key in request.json, f"No {field_key} provided"
             assert field_value(request.json[field_key]), f"Invalid {field_key} provided"
 
-        decrypted_content_cid, err = resolve_content(request.json['content'])
+        # Support legacy: 'content' as decrypted ContentId; and new: 'content' as encrypted IPFS CID
-        assert not err, f"Invalid content CID"
+        source_content_cid, cid_err = resolve_content(request.json['content'])
+        assert not cid_err, f"Invalid content CID provided: {cid_err}"
 
-        # Поиск исходного файла загруженного
+        encrypted_content_cid = None
-        decrypted_content = request.ctx.db_session.query(StoredContent).filter(
+        decrypted_content = (await request.ctx.db_session.execute(
-            StoredContent.hash == decrypted_content_cid.content_hash_b58
+            select(StoredContent).where(StoredContent.hash == source_content_cid.content_hash_b58)
-        ).first()
+        )).scalars().first()
-        assert decrypted_content, "No content locally found"
-        assert decrypted_content.type == "local/content_bin", "Invalid content type"
 
-        # Создание фиктивного encrypted_content. Не шифруем для производительности, тк зашифрованная нигде дальше не используется
+        if decrypted_content and decrypted_content.type == "local/content_bin":
             encrypted_content = await create_encrypted_content(request.ctx.db_session, decrypted_content)
             encrypted_content_cid = encrypted_content.cid
+        elif source_content_cid.cid_format == 'ipfs':
+            encrypted_content_cid = source_content_cid
+        else:
+            raise AssertionError("Provided content is neither locally available nor a valid encrypted CID")
 
         if request.json['image']:
             image_content_cid, err = resolve_content(request.json['image'])
             assert not err, f"Invalid image CID"
-            image_content = request.ctx.db_session.query(StoredContent).filter(
+            image_content = (await request.ctx.db_session.execute(
-                StoredContent.hash == image_content_cid.content_hash_b58
+                select(StoredContent).where(StoredContent.hash == image_content_cid.content_hash_b58)
-            ).first()
+            )).scalars().first()
             assert image_content, "No image locally found"
         else:
             image_content_cid = None
@@ -94,6 +96,19 @@ async def s_api_v1_blockchain_send_new_content_message(request):
             downloadable=request.json['downloadable'] if 'downloadable' in request.json else False,
         )
 
+        # Try to update ContentIndexItem with cover_url for this encrypted content
+        try:
+            from app.core.models.content_v3 import ContentIndexItem
+            ecid_str = encrypted_content_cid.serialize_v2()
+            row = (await request.ctx.db_session.execute(select(ContentIndexItem).where(ContentIndexItem.encrypted_cid == ecid_str))).scalars().first()
+            if row:
+                payload = row.payload or {}
+                payload['cover_url'] = f"{PROJECT_HOST}/api/v1.5/storage/{image_content_cid.serialize_v2()}" if image_content_cid else None
+                row.payload = payload
+                await request.ctx.db_session.commit()
+        except Exception as _e:
+            make_log("Blockchain", f"index cover update failed: {_e}", level='warning')
+
         royalties_dict = begin_dict(8)
         i = 0
         for royalty_param in request.json['royaltyParams']:
@@ -105,18 +120,22 @@ async def s_api_v1_blockchain_send_new_content_message(request):
             )
             i += 1
 
-        promo_free_upload_available = (
+        _cnt = (await request.ctx.db_session.execute(
-            3 - (request.ctx.db_session.query(PromoAction).filter(
+            select(func.count()).select_from(PromoAction).where(
-                PromoAction.user_internal_id == request.ctx.user.id,
-                PromoAction.action_type == 'freeUpload',
-            ).count())
-        )
-        if request.ctx.db_session.query(BlockchainTask).filter(
                 and_(
-                BlockchainTask.user_id == request.ctx.user.id,
+                    PromoAction.user_internal_id == request.ctx.user.id,
-                BlockchainTask.status != 'done',
+                    PromoAction.action_type == 'freeUpload'
                 )
-        ).first():
+            )
+        )).scalar()
+        promo_free_upload_available = 3 - int(_cnt or 0)
+
+        has_pending_task = (await request.ctx.db_session.execute(
+            select(BlockchainTask).where(
+                and_(BlockchainTask.user_id == request.ctx.user.id, BlockchainTask.status != 'done')
+            )
+        )).scalars().first()
+        if has_pending_task:
             make_log("Blockchain", f"User {request.ctx.user.id} already has a pending task", level='warning')
             promo_free_upload_available = 0
 
@@ -127,7 +146,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                 user_id = str(request.ctx.user.id),
                 user_internal_id=request.ctx.user.id,
                 action_type='freeUpload',
-                action_ref=str(encrypted_content_cid.content_hash),
+                action_ref=encrypted_content_cid.serialize_v2(),
                 created=datetime.now()
             )
             request.ctx.db_session.add(promo_action)
@@ -139,7 +158,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                     begin_cell()
                         .store_uint(0x5491d08c, 32)
                         .store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
-                        .store_address(Address(request.ctx.user.wallet_address(request.ctx.db_session)))
+                        .store_address(Address(await request.ctx.user.wallet_address_async(request.ctx.db_session)))
                         .store_ref(
                             begin_cell()
                                 .store_ref(
@@ -177,14 +196,14 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                 user_id = request.ctx.user.id
             )
             request.ctx.db_session.add(blockchain_task)
-            request.ctx.db_session.commit()
+            await request.ctx.db_session.commit()
 
             await request.ctx.user_uploader_wrapper.send_message(
                 request.ctx.user.translated('p_uploadContentTxPromo').format(
                     title=content_title,
                     free_count=(promo_free_upload_available - 1)
                 ), message_type='hint', message_meta={
-                    'encrypted_content_hash': b58encode(encrypted_content_cid.content_hash).decode(),
+                    'encrypted_content_hash': encrypted_content_cid.content_hash_b58,
                     'hint_type': 'uploadContentTxRequested'
                 }
             )
@@ -194,23 +213,23 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                 'payload': ""
             })
 
+        user_wallet_address = await request.ctx.user.wallet_address_async(request.ctx.db_session)
+        assert user_wallet_address, "Wallet address is not linked"
+
         await request.ctx.user_uploader_wrapper.send_message(
             request.ctx.user.translated('p_uploadContentTxRequested').format(
                 title=content_title,
             ), message_type='hint', message_meta={
-                'encrypted_content_hash': b58encode(encrypted_content_cid.content_hash).decode(),
+                'encrypted_content_hash': encrypted_content_cid.content_hash_b58,
                 'hint_type': 'uploadContentTxRequested'
             }
         )
 
-        return response.json({
+        payload_cell = (
-            'address': platform.address.to_string(1, 1, 1),
-            'amount': str(int(0.03 * 10 ** 9)),
-            'payload': b64encode(
             begin_cell()
                 .store_uint(0x5491d08c, 32)
                 .store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
-                    .store_uint(0, 2)
+                .store_address(Address(user_wallet_address))
                 .store_ref(
                     begin_cell()
                         .store_ref(
@@ -240,8 +259,13 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                         )
                         .end_cell()
                 )
-                    .end_cell().to_boc(False)
+                .end_cell()
-            ).decode()
+        )
+
+        return response.json({
+            'address': platform.address.to_string(1, 1, 1),
+            'amount': str(int(0.03 * 10 ** 9)),
+            'payload': b64encode(payload_cell.to_boc(False)).decode()
         })
     except BaseException as e:
         make_log("Blockchain", f"Error while sending new content message: {e}" + '\n' + traceback.format_exc(), level='error')
@@ -258,18 +282,38 @@ async def s_api_v1_blockchain_send_purchase_content_message(request):
         assert field_key in request.json, f"No {field_key} provided"
         assert field_value(request.json[field_key]), f"Invalid {field_key} provided"
 
-    if not request.ctx.user.wallet_address(request.ctx.db_session):
+    if not (await request.ctx.user.wallet_address_async(request.ctx.db_session)):
         return response.json({"error": "No wallet address provided"}, status=400)
 
-    license_exist = request.ctx.db_session.query(UserContent).filter_by(
+    from sqlalchemy import select
-        onchain_address=request.json['content_address'],
+    license_exist = (await request.ctx.db_session.execute(select(UserContent).where(
-    ).first()
+        UserContent.onchain_address == request.json['content_address']
-    if license_exist:
+    ))).scalars().first()
-        r_content = StoredContent.from_cid(request.ctx.db_session, license_exist.content.cid.serialize_v2())
+    from app.core.content.content_id import ContentId
-    else:
-        r_content = StoredContent.from_cid(request.ctx.db_session, request.json['content_address'])
 
-    content = r_content.open_content(request.ctx.db_session)
+    if license_exist and license_exist.content_id:
+        r_content = (await request.ctx.db_session.execute(select(StoredContent).where(
+            StoredContent.id == license_exist.content_id
+        ))).scalars().first()
+    else:
+        requested_cid = ContentId.deserialize(request.json['content_address'])
+        r_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == requested_cid.content_hash_b58))).scalars().first()
+    
+    async def open_content_async(session, sc: StoredContent):
+        if not sc.encrypted:
+            decrypted = sc
+            encrypted = (await session.execute(select(StoredContent).where(StoredContent.decrypted_content_id == sc.id))).scalars().first()
+        else:
+            encrypted = sc
+            decrypted = (await session.execute(select(StoredContent).where(StoredContent.id == sc.decrypted_content_id))).scalars().first()
+        assert decrypted and encrypted, "Can't open content"
+        ctype = decrypted.json_format().get('content_type', 'application/x-binary')
+        try:
+            content_type = ctype.split('/')[0]
+        except Exception:
+            content_type = 'application'
+        return {'encrypted_content': encrypted, 'decrypted_content': decrypted, 'content_type': content_type}
+    content = await open_content_async(request.ctx.db_session, r_content)
 
     licenses_cost = content['encrypted_content'].json_format()['license']
     assert request.json['license_type'] in licenses_cost

app/api/routes/_system.py

@@ -6,6 +6,7 @@ from base58 import b58encode, b58decode
 from sanic import response
 
 from app.core.models.node_storage import StoredContent
+from sqlalchemy import select
 from app.core._blockchain.ton.platform import platform
 from app.core._crypto.signer import Signer
 from app.core._secrets import hot_pubkey, service_wallet, hot_seed
@@ -19,10 +20,10 @@ def get_git_info():
 
 
 async def s_api_v1_node(request):  # /api/v1/node
-    last_known_index = request.ctx.db_session.query(StoredContent).filter(
+    last_known_index_obj = (await request.ctx.db_session.execute(
-        StoredContent.onchain_index != None
+        select(StoredContent).where(StoredContent.onchain_index != None).order_by(StoredContent.onchain_index.desc())
-    ).order_by(StoredContent.onchain_index.desc()).first()
+    )).scalars().first()
-    last_known_index = last_known_index.onchain_index if last_known_index else 0
+    last_known_index = last_known_index_obj.onchain_index if last_known_index_obj else 0
     last_known_index = max(last_known_index, 0)
     return response.json({
         'id': b58encode(hot_pubkey).decode(),
@@ -39,10 +40,10 @@ async def s_api_v1_node(request):  # /api/v1/node
     })
 
 async def s_api_v1_node_friendly(request):
-    last_known_index = request.ctx.db_session.query(StoredContent).filter(
+    last_known_index_obj = (await request.ctx.db_session.execute(
-        StoredContent.onchain_index != None
+        select(StoredContent).where(StoredContent.onchain_index != None).order_by(StoredContent.onchain_index.desc())
-    ).order_by(StoredContent.onchain_index.desc()).first()
+    )).scalars().first()
-    last_known_index = last_known_index.onchain_index if last_known_index else 0
+    last_known_index = last_known_index_obj.onchain_index if last_known_index_obj else 0
     last_known_index = max(last_known_index, 0)
     response_plain_text = f"""
     Node address: {service_wallet.address.to_string(1, 1, 1)}

app/api/routes/auth.py

@@ -37,7 +37,9 @@ async def s_api_v1_auth_twa(request):
         make_log("auth", "Invalid TWA data", level="warning")
         return response.json({"error": "Invalid TWA data"}, status=401)
 
-    known_user = request.ctx.db_session.query(User).filter(User.telegram_id == twa_data.user.id).first()
+    known_user = (await request.ctx.db_session.execute(
+        select(User).where(User.telegram_id == twa_data.user.id)
+    )).scalars().first()
     if not known_user:
         new_user = User(
             telegram_id=twa_data.user.id,
@@ -52,11 +54,28 @@ async def s_api_v1_auth_twa(request):
             created=datetime.now()
         )
         request.ctx.db_session.add(new_user)
-        request.ctx.db_session.commit()
+        await request.ctx.db_session.commit()
 
-        known_user = request.ctx.db_session.query(User).filter(User.telegram_id == twa_data.user.id).first()
+        known_user = (await request.ctx.db_session.execute(
+            select(User).where(User.telegram_id == twa_data.user.id)
+        )).scalars().first()
         assert known_user, "User not created"
 
+    meta_updated = False
+    if not (known_user.meta or {}).get('ref_id'):
+        known_user.ensure_ref_id()
+        meta_updated = True
+
+    incoming_ref_id = auth_data.get('ref_id')
+    stored_ref_id = (known_user.meta or {}).get('ref_id')
+    if incoming_ref_id and incoming_ref_id != stored_ref_id:
+        if (known_user.meta or {}).get('referrer_id') != incoming_ref_id:
+            known_user.meta = {
+                **(known_user.meta or {}),
+                'referrer_id': incoming_ref_id
+            }
+            meta_updated = True
+
     new_user_key = await known_user.create_api_token_v1(request.ctx.db_session, "USER_API_V1")
     if auth_data['ton_proof']:
         try:
@@ -65,12 +84,12 @@ async def s_api_v1_auth_twa(request):
             wallet_info.account = Account.from_dict(auth_data['ton_proof']['account'])
             wallet_info.ton_proof = TonProof.from_dict({'proof': auth_data['ton_proof']['ton_proof']})
             connection_payload = auth_data['ton_proof']['ton_proof']['payload']
-            known_payload = (request.ctx.db_session.execute(select(KnownKey).where(KnownKey.seed == connection_payload))).scalars().first()
+            known_payload = (await request.ctx.db_session.execute(select(KnownKey).where(KnownKey.seed == connection_payload))).scalars().first()
             assert known_payload, "Unknown payload"
             assert known_payload.meta['I_user_id'] == known_user.id, "Invalid user_id"
             assert wallet_info.check_proof(connection_payload), "Invalid proof"
 
-            for known_connection in (request.ctx.db_session.execute(select(WalletConnection).where(
+            for known_connection in (await request.ctx.db_session.execute(select(WalletConnection).where(
                     and_(
                         WalletConnection.user_id == known_user.id,
                         WalletConnection.network == 'ton'
@@ -78,7 +97,7 @@ async def s_api_v1_auth_twa(request):
             ))).scalars().all():
                 known_connection.invalidated = True
 
-            for other_connection in (request.ctx.db_session.execute(select(WalletConnection).where(
+            for other_connection in (await request.ctx.db_session.execute(select(WalletConnection).where(
                     WalletConnection.wallet_address == Address(wallet_info.account.address).to_string(1, 1, 1)
             ))).scalars().all():
                 other_connection.invalidated = True
@@ -99,12 +118,12 @@ async def s_api_v1_auth_twa(request):
                 without_pk=False
             )
             request.ctx.db_session.add(new_connection)
-            request.ctx.db_session.commit()
+            await request.ctx.db_session.commit()
         except BaseException as e:
             make_log("auth", f"Invalid ton_proof: {e}", level="warning")
             return response.json({"error": "Invalid ton_proof"}, status=400)
 
-    ton_connection = (request.ctx.db_session.execute(select(WalletConnection).where(
+    ton_connection = (await request.ctx.db_session.execute(select(WalletConnection).where(
         and_(
             WalletConnection.user_id == known_user.id,
             WalletConnection.network == 'ton',
@@ -112,7 +131,9 @@ async def s_api_v1_auth_twa(request):
         )
     ).order_by(WalletConnection.created.desc()))).scalars().first()
     known_user.last_use = datetime.now()
-    request.ctx.db_session.commit()
+    if meta_updated:
+        known_user.updated = datetime.now()
+    await request.ctx.db_session.commit()
 
     return response.json({
         'user': known_user.json_format(),
@@ -124,7 +145,7 @@ async def s_api_v1_auth_me(request):
     if not request.ctx.user:
         return response.json({"error": "Unauthorized"}, status=401)
 
-    ton_connection = (request.ctx.db_session.execute(
+    ton_connection = (await request.ctx.db_session.execute(
         select(WalletConnection).where(
             and_(
                 WalletConnection.user_id == request.ctx.user.id,
@@ -159,10 +180,12 @@ async def s_api_v1_auth_select_wallet(request):
     user = request.ctx.user
 
     # Check if a WalletConnection already exists for this user with the given canonical wallet address
-    existing_connection = db_session.query(WalletConnection).filter(
+    existing_connection = (await db_session.execute(select(WalletConnection).where(
+        and_(
             WalletConnection.user_id == user.id,
             WalletConnection.wallet_address == canonical_address
-    ).first()
+        )
+    ))).scalars().first()
 
     if not existing_connection:
         return response.json({"error": "Wallet connection not found"}, status=404)
@@ -185,6 +208,6 @@ async def s_api_v1_auth_select_wallet(request):
         without_pk=False
     )
     db_session.add(new_connection)
-    db_session.commit()
+    await db_session.commit()
 
     return response.empty(status=200)

app/api/routes/content.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta
 from sanic import response
+from sqlalchemy import select, and_, func
 from aiogram import Bot, types
 from sqlalchemy import and_
 from app.core.logger import make_log
@@ -9,6 +10,8 @@ from app.core.models.keys import KnownKey
 from app.core.models import StarsInvoice
 from app.core.models.content.user_content import UserContent
 from app.core._config import CLIENT_TELEGRAM_API_KEY, PROJECT_HOST
+from app.core.models.content_v3 import EncryptedContent as ECv3, ContentDerivative as CDv3, UploadSession
+from app.core.content.content_id import ContentId
 import json
 import uuid
 
@@ -22,13 +25,20 @@ async def s_api_v1_content_list(request):
     store = request.args.get('store', 'local')
     assert store in ('local', 'onchain'), "Invalid store"
 
-    content_list = request.ctx.db_session.query(StoredContent).filter(
+    stmt = (
+        select(StoredContent)
+        .where(
             StoredContent.type.like(store + '%'),
-        StoredContent.disabled == False
+            StoredContent.disabled.is_(None)
-    ).order_by(StoredContent.created.desc()).offset(offset).limit(limit)
+        )
-    make_log("Content", f"Listed {content_list.count()} contents", level='info')
+        .order_by(StoredContent.created.desc())
+        .offset(offset)
+        .limit(limit)
+    )
+    rows = (await request.ctx.db_session.execute(stmt)).scalars().all()
+    make_log("Content", f"Listed {len(rows)} contents", level='info')
     result = {}
-    for content in content_list.all():
+    for content in rows:
         content_json = content.json_format()
         result[content_json["cid"]] = content_json
 
@@ -38,23 +48,51 @@ async def s_api_v1_content_list(request):
 async def s_api_v1_content_view(request, content_address: str):
     # content_address can be CID or TON address
 
-    license_exist = request.ctx.db_session.query(UserContent).filter_by(
+    license_exist = (await request.ctx.db_session.execute(
-        onchain_address=content_address,
+        select(UserContent).where(UserContent.onchain_address == content_address)
-    ).first()
+    )).scalars().first()
+    license_address = None
     if license_exist:
-        content_address = license_exist.content.cid.serialize_v2()
+        license_address = license_exist.onchain_address
+        if license_exist.content_id:
+            linked_content = (await request.ctx.db_session.execute(
+                select(StoredContent).where(StoredContent.id == license_exist.content_id)
+            )).scalars().first()
+            if linked_content:
+                content_address = linked_content.cid.serialize_v2()
 
-    r_content = StoredContent.from_cid(request.ctx.db_session, content_address)
+    from app.core.content.content_id import ContentId
-    content = r_content.open_content(request.ctx.db_session)
+    cid = ContentId.deserialize(content_address)
+    r_content = (await request.ctx.db_session.execute(
+        select(StoredContent).where(StoredContent.hash == cid.content_hash_b58)
+    )).scalars().first()
+    async def open_content_async(session, sc: StoredContent):
+        if not sc.encrypted:
+            decrypted = sc
+            encrypted = (await session.execute(select(StoredContent).where(StoredContent.decrypted_content_id == sc.id))).scalars().first()
+        else:
+            encrypted = sc
+            decrypted = (await session.execute(select(StoredContent).where(StoredContent.id == sc.decrypted_content_id))).scalars().first()
+        assert decrypted and encrypted, "Can't open content"
+        ctype = decrypted.json_format().get('content_type', 'application/x-binary')
+        try:
+            content_type = ctype.split('/')[0]
+        except Exception:
+            content_type = 'application'
+        return {'encrypted_content': encrypted, 'decrypted_content': decrypted, 'content_type': content_type}
+    content = await open_content_async(request.ctx.db_session, r_content)
 
+    master_address = content['encrypted_content'].meta.get('item_address', '')
     opts = {
         'content_type': content['content_type'],  # возможно с ошибками, нужно переделать на ffprobe
-        'content_address': content['encrypted_content'].meta.get('item_address', '')
+        'content_address': license_address or master_address,
+        'license_address': license_address,
+        'master_address': master_address,
     }
     if content['encrypted_content'].key_id:
-        known_key = request.ctx.db_session.query(KnownKey).filter(
+        known_key = (await request.ctx.db_session.execute(
-            KnownKey.id == content['encrypted_content'].key_id
+            select(KnownKey).where(KnownKey.id == content['encrypted_content'].key_id)
-        ).first()
+        )).scalars().first()
         if known_key:
             opts['key_hash'] = known_key.seed_hash  # нахер не нужно на данный момент
 
@@ -64,22 +102,23 @@ async def s_api_v1_content_view(request, content_address: str):
 
     have_access = False
     if request.ctx.user:
-        user_wallet_address = request.ctx.user.wallet_address(request.ctx.db_session)
+        user_wallet_address = await request.ctx.user.wallet_address_async(request.ctx.db_session)
         have_access = (
                 (content['encrypted_content'].owner_address == user_wallet_address)
-                or bool(request.ctx.db_session.query(UserContent).filter_by(owner_address=user_wallet_address, status='active',
+                or bool((await request.ctx.db_session.execute(select(UserContent).where(
-                                                                     content_id=content['encrypted_content'].id).first()) \
+                    and_(UserContent.owner_address == user_wallet_address, UserContent.status == 'active', UserContent.content_id == content['encrypted_content'].id)
-                or bool(request.ctx.db_session.query(StarsInvoice).filter(
+                ))).scalars().first()) \
+                or bool((await request.ctx.db_session.execute(select(StarsInvoice).where(
                     and_(
                         StarsInvoice.user_id == request.ctx.user.id,
                         StarsInvoice.content_hash == content['encrypted_content'].hash,
                         StarsInvoice.paid == True
                     )
-                ).first())
+                ))).scalars().first())
         )
 
         if not have_access:
-            current_star_rate = ServiceConfig(request.ctx.db_session).get('live_tonPerStar', [0, 0])[0]
+            current_star_rate = (await ServiceConfig(request.ctx.db_session).get('live_tonPerStar', [0, 0]))[0]
             if current_star_rate < 0:
                 current_star_rate = 0.00000001
 
@@ -88,14 +127,14 @@ async def s_api_v1_content_view(request, content_address: str):
                 stars_cost = 2
 
             invoice_id = f"access_{uuid.uuid4().hex}"
-            exist_invoice = request.ctx.db_session.query(StarsInvoice).filter(
+            exist_invoice = (await request.ctx.db_session.execute(select(StarsInvoice).where(
                 and_(
                     StarsInvoice.user_id == request.ctx.user.id,
                     StarsInvoice.created > datetime.now() - timedelta(minutes=25),
                     StarsInvoice.amount == stars_cost,
                     StarsInvoice.content_hash == content['encrypted_content'].hash,
                 )
-            ).first()
+            ))).scalars().first()
             if exist_invoice:
                 invoice_url = exist_invoice.invoice_url
             else:
@@ -119,7 +158,7 @@ async def s_api_v1_content_view(request, content_address: str):
                             invoice_url=invoice_url
                         )
                     )
-                    request.ctx.db_session.commit()
+                    await request.ctx.db_session.commit()
                 except BaseException as e:
                     make_log("Content", f"Can't create invoice link: {e}", level='warning')
 
@@ -129,37 +168,169 @@ async def s_api_v1_content_view(request, content_address: str):
                     'amount': stars_cost,
                 }
 
-    display_options = {
+    display_options = {'content_url': None}
-        'content_url': None,
-    }
 
     if have_access:
         opts['have_licenses'].append('listen')
 
-    converted_content = content['encrypted_content'].meta.get('converted_content')
+    enc_cid = content['encrypted_content'].meta.get('content_cid') or content['encrypted_content'].meta.get('encrypted_cid')
-    if converted_content:
+    ec_v3 = None
-        user_content_option = 'low_preview'
+    derivative_rows = []
+    if enc_cid:
+        ec_v3 = (await request.ctx.db_session.execute(select(ECv3).where(ECv3.encrypted_cid == enc_cid))).scalars().first()
+        if ec_v3:
+            derivative_rows = (await request.ctx.db_session.execute(select(CDv3).where(CDv3.content_id == ec_v3.id))).scalars().all()
+
+    upload_row = None
+    if enc_cid:
+        upload_row = (await request.ctx.db_session.execute(select(UploadSession).where(UploadSession.encrypted_cid == enc_cid))).scalars().first()
+
+    converted_meta_map = dict(content['encrypted_content'].meta.get('converted_content') or {})
+
+    derivative_latest = {}
+    if derivative_rows:
+        derivative_sorted = sorted(derivative_rows, key=lambda row: row.created_at or datetime.min)
+        for row in derivative_sorted:
+            derivative_latest[row.kind] = row
+
+    def _row_to_hash_and_url(row):
+        if not row or not row.local_path:
+            return None, None
+        file_hash = row.local_path.split('/')[-1]
+        return file_hash, f"{PROJECT_HOST}/api/v1.5/storage/{file_hash}"
+
+    chosen_row = None
     if have_access:
-            user_content_option = 'low'  # TODO: подключать high если человек внезапно меломан
+        for key in ('decrypted_low', 'decrypted_high'):
+            if key in derivative_latest:
+                chosen_row = derivative_latest[key]
+                break
+    else:
+        for key in ('decrypted_preview', 'decrypted_low'):
+            if key in derivative_latest:
+                chosen_row = derivative_latest[key]
+                break
 
-        converted_content = request.ctx.db_session.query(StoredContent).filter(
+    if chosen_row:
-            StoredContent.hash == converted_content[user_content_option]
+        file_hash, url = _row_to_hash_and_url(chosen_row)
-        ).first()
+        if url:
-        if converted_content:
+            display_options['content_url'] = url
-            display_options['content_url'] = converted_content.web_url
+            opts['content_ext'] = (chosen_row.content_type or '').split('/')[-1] if chosen_row.content_type else None
-            opts['content_ext'] = converted_content.filename.split('.')[-1]
+            converted_meta_map.setdefault('low' if have_access else 'low_preview', file_hash)
 
+    if not display_options['content_url'] and converted_meta_map:
+        preference = ['low', 'high', 'low_preview'] if have_access else ['low_preview', 'low', 'high']
+        for key in preference:
+            hash_value = converted_meta_map.get(key)
+            if not hash_value:
+                continue
+            stored = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == hash_value))).scalars().first()
+            if stored:
+                display_options['content_url'] = stored.web_url
+                opts['content_ext'] = stored.filename.split('.')[-1]
+                break
+
+    # Metadata fallback
     content_meta = content['encrypted_content'].json_format()
-    content_metadata = StoredContent.from_cid(request.ctx.db_session, content_meta.get('metadata_cid') or None)
+    content_metadata_json = None
+    _mcid = content_meta.get('metadata_cid') or None
+    if _mcid:
+        _cid = ContentId.deserialize(_mcid)
+        content_metadata = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
+        if content_metadata:
+            try:
                 with open(content_metadata.filepath, 'r') as f:
                     content_metadata_json = json.loads(f.read())
+            except Exception as exc:
+                make_log("Content", f"Can't read metadata file: {exc}", level='warning')
+
+    if not content_metadata_json:
+        fallback_name = (ec_v3.title if ec_v3 else None) or content_meta.get('title') or content_meta.get('cid')
+        fallback_description = (ec_v3.description if ec_v3 else '') or ''
+        content_metadata_json = {
+            'name': fallback_name or 'Без названия',
+            'description': fallback_description,
+            'downloadable': False,
+        }
+        cover_cid = content_meta.get('cover_cid')
+        if cover_cid:
+            content_metadata_json.setdefault('image', f"{PROJECT_HOST}/api/v1.5/storage/{cover_cid}")
 
     display_options['metadata'] = content_metadata_json
+
     opts['downloadable'] = content_metadata_json.get('downloadable', False)
-    if opts['downloadable']:
+    if opts['downloadable'] and 'listen' not in opts['have_licenses']:
-        if not ('listen' in opts['have_licenses']):
         opts['downloadable'] = False
 
+    # Conversion status summary
+    conversion_summary = {}
+    conversion_details = []
+    derivative_summary_map = {}
+    for row in derivative_latest.values():
+        conversion_summary[row.status] = conversion_summary.get(row.status, 0) + 1
+        derivative_summary_map[row.kind] = row
+        conversion_details.append({
+            'kind': row.kind,
+            'status': row.status,
+            'size_bytes': row.size_bytes,
+            'content_type': row.content_type,
+            'error': row.error,
+            'updated_at': (row.last_access_at or row.created_at).isoformat() + 'Z' if (row.last_access_at or row.created_at) else None,
+        })
+
+    required_kinds = {'decrypted_low', 'decrypted_high'}
+    if ec_v3 and ec_v3.content_type.startswith('video/'):
+        required_kinds.add('decrypted_preview')
+
+    statuses_by_kind = {kind: row.status for kind, row in derivative_summary_map.items() if kind in required_kinds}
+    conversion_state = 'pending'
+    if required_kinds and all(statuses_by_kind.get(kind) == 'ready' for kind in required_kinds):
+        conversion_state = 'ready'
+    elif any(statuses_by_kind.get(kind) == 'failed' for kind in required_kinds):
+        conversion_state = 'failed'
+    elif any(statuses_by_kind.get(kind) in ('processing', 'pending') for kind in required_kinds):
+        conversion_state = 'processing'
+    elif statuses_by_kind:
+        conversion_state = 'partial'
+
+    if display_options['content_url']:
+        conversion_state = 'ready'
+
+    upload_info = None
+    if upload_row:
+        upload_info = {
+            'id': upload_row.id,
+            'state': upload_row.state,
+            'error': upload_row.error,
+            'created_at': upload_row.created_at.isoformat() + 'Z' if upload_row.created_at else None,
+            'updated_at': upload_row.updated_at.isoformat() + 'Z' if upload_row.updated_at else None,
+        }
+
+    final_state = 'ready' if display_options['content_url'] else None
+    if final_state != 'ready':
+        upload_state = upload_row.state if upload_row else None
+        if conversion_state == 'failed' or upload_state in ('failed', 'conversion_failed'):
+            final_state = 'failed'
+        elif conversion_state in ('processing', 'partial') or upload_state in ('processing', 'pinned'):
+            final_state = 'processing'
+        else:
+            final_state = 'uploaded'
+
+    conversion_info = {
+        'state': conversion_state,
+        'summary': conversion_summary,
+        'details': conversion_details,
+        'required_kinds': list(required_kinds),
+    }
+
+    opts['conversion'] = conversion_info
+    opts['upload'] = upload_info
+    opts['status'] = {
+        'state': final_state,
+        'conversion_state': conversion_state,
+        'upload_state': upload_info['state'] if upload_info else None,
+    }
+
     return response.json({
         **opts,
         'encrypted': content['encrypted_content'].json_format(),
@@ -187,14 +358,17 @@ async def s_api_v1_content_friendly_list(request):
                     </tr>
                 </thead>
 """
-    for content in request.ctx.db_session.query(StoredContent).filter(
+    contents = (await request.ctx.db_session.execute(select(StoredContent).where(
         StoredContent.type == 'onchain/content'
-    ).all():
+    ))).scalars().all()
+    for content in contents:
         if not content.meta.get('metadata_cid'):
             make_log("Content", f"Content {content.cid.serialize_v2()} has no metadata", level='warning')
             continue
 
-        metadata_content = StoredContent.from_cid(request.ctx.db_session, content.meta.get('metadata_cid'))
+        from app.core.content.content_id import ContentId
+        _cid = ContentId.deserialize(content.meta.get('metadata_cid'))
+        metadata_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
         with open(metadata_content.filepath, 'r') as f:
             metadata = json.loads(f.read())
         
@@ -228,10 +402,12 @@ async def s_api_v1_5_content_list(request):
         return response.json({'error': 'Invalid limit'}, status=400)
 
     # Query onchain contents which are not disabled
-    contents = request.ctx.db_session.query(StoredContent).filter(
+    contents = (await request.ctx.db_session.execute(
-        StoredContent.type == 'onchain/content',
+        select(StoredContent)
-        StoredContent.disabled == False
+        .where(StoredContent.type == 'onchain/content', StoredContent.disabled == False)
-    ).order_by(StoredContent.created.desc()).offset(offset).limit(limit).all()
+        .order_by(StoredContent.created.desc())
+        .offset(offset).limit(limit)
+    )).scalars().all()
 
     result = []
     for content in contents:
@@ -240,7 +416,9 @@ async def s_api_v1_5_content_list(request):
         if not metadata_cid:
             continue  # Skip if no metadata_cid is found
 
-        metadata_content = StoredContent.from_cid(request.ctx.db_session, metadata_cid)
+        from app.core.content.content_id import ContentId
+        _cid = ContentId.deserialize(metadata_cid)
+        metadata_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
         try:
             with open(metadata_content.filepath, 'r') as f:
                 metadata = json.load(f)
@@ -256,9 +434,9 @@ async def s_api_v1_5_content_list(request):
         preview_link = None
         converted_content = content.meta.get('converted_content')
         if converted_content:
-            converted_content = request.ctx.db_session.query(StoredContent).filter(
+            converted_content = (await request.ctx.db_session.execute(select(StoredContent).where(
                 StoredContent.hash == converted_content['low_preview']
-            ).first()
+            ))).scalars().first()
             preview_link = converted_content.web_url
             if converted_content.filename.split('.')[-1] in ('mp4', 'mov'):
                 media_type = 'video'

app/api/routes/content_index.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from sanic import response
+from sqlalchemy import select
+from datetime import datetime
+
+from app.core.models.content_v3 import ContentIndexItem
+from app.core.logger import make_log
+
+
+async def s_api_v1_content_index(request):
+    rows = (await request.ctx.db_session.execute(select(ContentIndexItem))).scalars().all()
+    items = [{**r.payload, "encrypted_cid": r.encrypted_cid, "sig": r.sig, "_updated_at": (r.updated_at.isoformat() + 'Z') if r.updated_at else None} for r in rows]
+    # ETag by max updated_at + count
+    max_ts = max((it.get("_updated_at") for it in items if it.get("_updated_at")), default="1970-01-01T00:00:00Z")
+    etag = f'W/"{max_ts}.{len(items)}"'
+    inm = request.headers.get('If-None-Match')
+    if inm and inm == etag:
+        resp = response.empty(status=304)
+        resp.headers['ETag'] = etag
+        return resp
+    for it in items:
+        it.pop("_updated_at", None)
+    make_log("content.index", f"items={len(items)} etag={etag}")
+    resp = response.json({"items": items, "schema": "my-network/index@1"})
+    resp.headers['ETag'] = etag
+    return resp
+
+
+async def s_api_v1_content_delta(request):
+    since = request.args.get('since')
+    if not since:
+        # No since provided → act as full index
+        return await s_api_v1_content_index(request)
+    try:
+        # basic parse
+        _ = datetime.fromisoformat(since.replace('Z', '+00:00'))
+    except Exception:
+        return response.json({"error": "BAD_SINCE"}, status=400)
+
+    rows = (await request.ctx.db_session.execute(select(ContentIndexItem))).scalars().all()
+    out = []
+    max_ts = since
+    for r in rows:
+        upd = (r.updated_at.isoformat() + 'Z') if r.updated_at else None
+        if upd and upd > since:
+            out.append({**r.payload, "encrypted_cid": r.encrypted_cid, "sig": r.sig})
+            if upd > max_ts:
+                max_ts = upd
+    resp = response.json({"items": out, "next_since": max_ts, "schema": "my-network/index@1"})
+    # Weak ETag for delta response
+    resp.headers['ETag'] = f'W/"{max_ts}.{len(out)}"'
+    return resp

app/api/routes/derivatives.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from sanic import response
+from sqlalchemy import select
+
+from app.core.models.content_v3 import EncryptedContent, ContentDerivative
+from app.core._config import PROJECT_HOST
+
+
+async def s_api_v1_content_derivatives(request):
+    cid = request.args.get('cid')
+    if not cid:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+    session = request.ctx.db_session
+    ec = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+    if not ec:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id))).scalars().all()
+    out = []
+    for r in rows:
+        # Derive /api/v1.5/storage/<hash> from local_path if possible
+        path_hash = (r.local_path or '').split('/')[-1]
+        storage_url = f"{PROJECT_HOST}/api/v1.5/storage/{path_hash}" if path_hash else None
+        out.append({
+            'kind': r.kind,
+            'interval': [r.interval_start_ms, r.interval_end_ms] if r.interval_start_ms is not None else None,
+            'content_type': r.content_type,
+            'size_bytes': r.size_bytes,
+            'status': r.status,
+            'url': storage_url,
+        })
+    return response.json({'cid': cid, 'derivatives': out})
+

app/api/routes/keys.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import base64
+import json
+import os
+from datetime import datetime
+from typing import Dict, Any
+
+from base58 import b58encode
+from sanic import response
+from sqlalchemy import select
+
+from app.core._secrets import hot_pubkey
+from app.core.logger import make_log
+from app.core.models.content_v3 import EncryptedContent, ContentKey, KeyGrant
+from app.core.network.nodesig import verify_request
+from app.core.network.guard import check_rate_limit
+from app.core.models.my_network import KnownNode
+from app.core.crypto.keywrap import unwrap_dek, KeyWrapError
+
+
+def _b64(b: bytes) -> str:
+    return base64.b64encode(b).decode()
+
+
+async def s_api_v1_keys_request(request):
+    # Rate limit per remote IP (reuse handshake limiter)
+    remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip()
+    if not check_rate_limit(request.app.ctx.memory, remote_ip):
+        return response.json({"error": "RATE_LIMIT"}, status=429)
+
+    # Verify NodeSig
+    ok, hdr_node, reason = verify_request(request, request.app.ctx.memory)
+    if not ok:
+        return response.json({"error": reason or "UNAUTHORIZED"}, status=401)
+
+    data: Dict[str, Any] = request.json or {}
+    cid = data.get("encrypted_cid")
+    requester_node = data.get("requestor_node_id")
+    recipient_box_pub_b64 = data.get("recipient_box_pub")
+    if not cid or not requester_node or not recipient_box_pub_b64:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+
+    if requester_node != hdr_node:
+        return response.json({"error": "NODE_ID_MISMATCH"}, status=401)
+
+    session = request.ctx.db_session
+    row = (await session.execute(select(EncryptedContent, ContentKey).join(ContentKey, ContentKey.content_id == EncryptedContent.id).where(EncryptedContent.encrypted_cid == cid))).first()
+    if not row:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    ec: EncryptedContent = row[0]
+    ck: ContentKey = row[1]
+    # Allow only trusted nodes unless explicitly disabled via env
+    TRUSTED_ONLY = (os.getenv('KEY_AUTO_GRANT_TRUSTED_ONLY', '1') == '1')
+    if TRUSTED_ONLY:
+        kn = (await session.execute(select(KnownNode).where(KnownNode.public_key == requester_node))).scalars().first()
+        role = (kn.meta or {}).get('role') if kn else None
+        if role != 'trusted':
+            return response.json({"error": "DENIED_NOT_TRUSTED"}, status=403)
+    if not ck.allow_auto_grant:
+        return response.json({"error": "DENIED"}, status=403)
+
+    # Seal the DEK for recipient using libsodium sealed box
+    try:
+        dek_plain = unwrap_dek(ck.key_ciphertext_b64)
+        import nacl.public
+        pk = nacl.public.PublicKey(base64.b64decode(recipient_box_pub_b64))
+        box = nacl.public.SealedBox(pk)
+        sealed = box.encrypt(dek_plain)
+        sealed_b64 = _b64(sealed)
+    except KeyWrapError as e:
+        make_log("keys", f"unwrap failed: {e}", level="error")
+        return response.json({"error": "KEY_UNWRAP_FAILED"}, status=500)
+    except Exception as e:
+        make_log("keys", f"seal failed: {e}", level="error")
+        return response.json({"error": "SEAL_FAILED"}, status=500)
+
+    issuer = b58encode(hot_pubkey).decode()
+    purpose = (data.get('purpose') or 'full')
+    ttl_sec = int(os.getenv('KEY_GRANT_PREVIEW_TTL_SEC', '0')) if purpose == 'preview' else 0
+    grant_body = {
+        "encrypted_cid": cid,
+        "to_node_id": requester_node,
+        "sealed_key_b64": sealed_b64,
+        "aead_scheme": ec.aead_scheme,
+        "chunk_bytes": ec.chunk_bytes,
+        "constraints": {"ttl_sec": ttl_sec, "scope": purpose},
+        "issued_at": datetime.utcnow().isoformat(),
+        "issuer_node_id": issuer,
+    }
+    try:
+        from app.core._crypto.signer import Signer
+        from app.core._secrets import hot_seed
+        signer = Signer(hot_seed)
+        blob = json.dumps(grant_body, sort_keys=True, separators=(",", ":")).encode()
+        sig = signer.sign(blob)
+    except Exception:
+        sig = ""
+
+    grant = KeyGrant(
+        encrypted_cid=cid,
+        issuer_node_id=issuer,
+        to_node_id=requester_node,
+        sealed_key_b64=sealed_b64,
+        aead_scheme=ec.aead_scheme,
+        chunk_bytes=ec.chunk_bytes,
+        constraints={"ttl_sec": 0, "scope": "full"},
+        sig=sig,
+    )
+    session.add(grant)
+    await session.commit()
+    grant_row = {
+        **grant_body,
+        "sig": sig,
+        "grant_id": grant.id,
+    }
+    return response.json(grant_row)

app/api/routes/network.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from typing import Dict, Any
+
+from base58 import b58decode
+from sanic import response
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.models.my_network import KnownNode
+from app.core.network.constants import CURRENT_PROTOCOL_VERSION, NODE_TYPE_PRIVATE
+from app.core.network.config import NODE_PRIVACY
+from app.core.network.handshake import build_handshake_payload, compute_node_info, sign_response
+from app.core.network.nodes import upsert_known_node, list_known_public_nodes
+from app.core.network.semver import compatibility
+from app.core.network.guard import check_rate_limit, check_timestamp_fresh, check_and_remember_nonce
+from app.core.network.config import HANDSHAKE_TS_TOLERANCE_SEC
+
+
+async def s_api_v1_network_info(request):
+    async with request.app.ctx.memory.transaction("network.info"):
+        node = await compute_node_info(request.ctx.db_session)
+    make_log("Network", "info served")
+    return response.json({"node": node})
+
+
+async def s_api_v1_network_nodes(request):
+    rows = await list_known_public_nodes(request.ctx.db_session)
+    make_log("Network", f"nodes list count={len(rows)}")
+    return response.json({
+        "count": len(rows),
+        "nodes": rows,
+    })
+
+
+async def s_api_v1_network_handshake(request):
+    # Handshake accepted regardless of our privacy; private nodes typically have no external endpoint
+
+    # Rate limit per remote IP
+    remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip()
+    if not check_rate_limit(request.app.ctx.memory, remote_ip):
+        return response.json({"error": "RATE_LIMIT"}, status=429)
+
+    data = request.json or {}
+    required = ["version", "public_key", "node_type", "metrics", "timestamp", "signature"]
+    for f in required:
+        if f not in data:
+            return response.json({"error": f"Missing field {f}"}, status=400)
+    # public_host is required for public nodes only
+    if data.get("node_type") != "private" and not data.get("public_host"):
+        return response.json({"error": "Missing field public_host"}, status=400)
+
+    # Timestamp freshness
+    if not check_timestamp_fresh(data.get("timestamp")):
+        return response.json({"error": "STALE_TIMESTAMP", "tolerance_sec": HANDSHAKE_TS_TOLERANCE_SEC}, status=400)
+
+    # Nonce replay protection (best-effort)
+    if not data.get("nonce") or not check_and_remember_nonce(request.app.ctx.memory, data.get("public_key"), data.get("nonce")):
+        return response.json({"error": "NONCE_REPLAY"}, status=400)
+
+    peer_version = str(data.get("version"))
+    comp = compatibility(peer_version, CURRENT_PROTOCOL_VERSION)
+    if comp == "blocked":
+        # We still store the node but respond with 409
+        try:
+            await upsert_known_node(
+                request.ctx.db_session,
+                host=data.get("public_host"),
+                port=int(str(data.get("public_host") or "").split(":")[-1]) if ":" in str(data.get("public_host") or "") else 80,
+                public_key=str(data.get("public_key")),
+                meta={
+                    "version": peer_version,
+                    "compatibility": comp,
+                    "is_public": data.get("node_type", "public") != "private",
+                    "public_host": data.get("public_host"),
+                    "unsupported_last_checked_at": datetime.utcnow().isoformat(),
+                }
+            )
+        except Exception:
+            pass
+        make_log("Handshake", f"Reject incompatible peer {data.get('public_host')} peer={peer_version} current={CURRENT_PROTOCOL_VERSION}")
+        return response.json({
+            "error": "INCOMPATIBLE_VERSION",
+            "compatibility": comp,
+            "current": CURRENT_PROTOCOL_VERSION,
+            "peer": peer_version,
+        }, status=409)
+
+    # Verify signature
+    try:
+        # Verify signature over the entire payload except the signature itself
+        signed_fields = {k: v for (k, v) in data.items() if k != "signature"}
+        blob = json.dumps(signed_fields, sort_keys=True, separators=(",", ":")).encode()
+        import nacl.signing, nacl.encoding
+        vk = nacl.signing.VerifyKey(b58decode(data["public_key"]))
+        sig = b58decode(data["signature"])
+        vk.verify(blob, sig)
+        ok = True
+    except Exception:
+        ok = False
+    if not ok:
+        make_log("Handshake", f"Signature verification failed from {data.get('public_host')}", level='warning')
+        return response.json({"error": "BAD_SIGNATURE"}, status=400)
+
+    # Upsert node and respond with our info + known public nodes
+    # Do not persist private peers (ephemeral)
+    if data.get("node_type") != "private" and data.get("public_host"):
+        try:
+            await upsert_known_node(
+                request.ctx.db_session,
+                host=data.get("public_host"),
+                port=int(str(data.get("public_host") or "").split(":")[-1]) if ":" in str(data.get("public_host") or "") else 80,
+                public_key=str(data.get("public_key")),
+                meta={
+                    "version": peer_version,
+                    "compatibility": comp,
+                    "is_public": True,
+                    "public_host": data.get("public_host"),
+                    "last_metrics": data.get("metrics", {}),
+                    "capabilities": data.get("capabilities", {}),
+                }
+            )
+        except Exception as e:
+            make_log("Handshake", f"Upsert peer failed: {e}", level='warning')
+
+    # Merge advertised peers from the caller (optional field)
+    for n in data.get("known_public_nodes", []) or []:
+        try:
+            await upsert_known_node(
+                request.ctx.db_session,
+                host=n.get("public_host") or n.get("host"),
+                port=int(n.get("port") or 80),
+                public_key=n.get("public_key") or "",
+                meta={
+                    "version": n.get("version") or "0.0.0",
+                    "compatibility": compatibility(n.get("version") or "0.0.0", CURRENT_PROTOCOL_VERSION),
+                    "is_public": True,
+                    "public_host": n.get("public_host") or n.get("host"),
+                    "capabilities": n.get("capabilities") or {},
+                }
+            )
+        except Exception:
+            pass
+
+    node = await compute_node_info(request.ctx.db_session)
+    known = await list_known_public_nodes(request.ctx.db_session)
+    resp = sign_response({
+        "compatibility": comp,
+        "node": node,
+        "known_public_nodes": known,
+    })
+    make_log("Handshake", f"OK with {data.get('public_host')} compat={comp}")
+    status = 200
+    if comp == "warning":
+        status = 200
+        resp["warning"] = "MINOR version differs; proceed with caution"
+    return response.json(resp, status=status)

app/api/routes/node_storage.py

@@ -11,6 +11,7 @@ from sanic import response
 import json
 
 from app.core._config import UPLOADS_DIR
+from sqlalchemy import select
 from app.core._utils.resolve_content import resolve_content
 from app.core.logger import make_log
 from app.core.models.node_storage import StoredContent
@@ -52,7 +53,9 @@ async def s_api_v1_storage_post(request):
     try:
         file_hash_bin = hashlib.sha256(file_content).digest()
         file_hash = b58encode(file_hash_bin).decode()
-        stored_content = request.ctx.db_session.query(StoredContent).filter(StoredContent.hash == file_hash).first()
+        stored_content = (await request.ctx.db_session.execute(
+            select(StoredContent).where(StoredContent.hash == file_hash)
+        )).scalars().first()
         if stored_content:
             stored_cid = stored_content.cid.serialize_v1()
             stored_cid_v2 = stored_content.cid.serialize_v2()
@@ -80,7 +83,7 @@ async def s_api_v1_storage_post(request):
             key_id=None,
         )
         request.ctx.db_session.add(new_content)
-        request.ctx.db_session.commit()
+        await request.ctx.db_session.commit()
 
         file_path = os.path.join(UPLOADS_DIR, file_hash)
         async with aiofiles.open(file_path, "wb") as file:
@@ -97,7 +100,7 @@ async def s_api_v1_storage_post(request):
             "content_url": f"dmy://storage?cid={new_cid}",
         })
     except BaseException as e:
-        make_log("Storage", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
+        make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error: {e}" + '\n' + traceback.format_exc(), level="error")
         return response.json({"error": f"Error: {e}"}, status=500)
 
 
@@ -112,14 +115,16 @@ async def s_api_v1_storage_get(request, file_hash=None):
         return response.json({"error": errmsg}, status=400)
 
     content_sha256 = b58encode(cid.content_hash).decode()
-    content = request.ctx.db_session.query(StoredContent).filter(StoredContent.hash == content_sha256).first()
+    content = (await request.ctx.db_session.execute(
+        select(StoredContent).where(StoredContent.hash == content_sha256)
+    )).scalars().first()
     if not content:
         return response.json({"error": "File not found"}, status=404)
 
-    make_log("Storage", f"File {content_sha256} requested by {request.ctx.user}")
+    make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} File {content_sha256} requested by user={getattr(getattr(request.ctx, 'user', None), 'id', None)}")
     file_path = os.path.join(UPLOADS_DIR, content_sha256)
     if not os.path.exists(file_path):
-        make_log("Storage", f"File {content_sha256} not found locally", level="error")
+        make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} File {content_sha256} not found locally", level="error")
         return response.json({"error": "File not found"}, status=404)
 
     async with aiofiles.open(file_path, "rb") as file:
@@ -139,7 +144,16 @@ async def s_api_v1_storage_get(request, file_hash=None):
             tempfile_path += "_mpeg" + (f"_{seconds_limit}" if seconds_limit else "")
             if not os.path.exists(tempfile_path):
                 try:
-                    cover_content = StoredContent.from_cid(content.meta.get('cover_cid'))
+                    # Resolve cover content by CID (async)
+                    from app.core.content.content_id import ContentId
+                    try:
+                        _cid = ContentId.deserialize(content.meta.get('cover_cid'))
+                        _cover_hash = _cid.content_hash_b58
+                        cover_content = (await request.ctx.db_session.execute(
+                            select(StoredContent).where(StoredContent.hash == _cover_hash)
+                        )).scalars().first()
+                    except Exception:
+                        cover_content = None
                     cover_tempfile_path = os.path.join(UPLOADS_DIR, f"tmp_{cover_content.hash}_jpeg")
                     if not os.path.exists(cover_tempfile_path):
                         cover_image = Image.open(cover_content.filepath)
@@ -173,25 +187,25 @@ async def s_api_v1_storage_get(request, file_hash=None):
                         try:
                             audio = AudioSegment.from_file(file_path)
                         except BaseException as e:
-                            make_log("Storage", f"Error loading audio from file: {e}", level="debug")
+                            make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error loading audio from file: {e}", level="debug")
 
                     if not audio:
                         try:
                             audio = AudioSegment(content_file_bin)
                         except BaseException as e:
-                            make_log("Storage", f"Error loading audio from binary: {e}", level="debug")
+                            make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error loading audio from binary: {e}", level="debug")
 
                     audio = audio[:seconds_limit * 1000] if seconds_limit else audio
                     audio.export(tempfile_path, format="mp3", cover=cover_tempfile_path)
                 except BaseException as e:
-                    make_log("Storage", f"Error converting audio: {e}" + '\n' + traceback.format_exc(), level="error")
+                    make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error converting audio: {e}" + '\n' + traceback.format_exc(), level="error")
 
             if os.path.exists(tempfile_path):
                 async with aiofiles.open(tempfile_path, "rb") as file:
                     content_file_bin = await file.read()
 
                 accept_type = 'audio/mpeg'
-                make_log("Storage", f"Audio {content_sha256} converted successfully")
+                make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Audio {content_sha256} converted successfully", level='debug')
             else:
                 tempfile_path = tempfile_path[:-5]
 
@@ -208,13 +222,13 @@ async def s_api_v1_storage_get(request, file_hash=None):
                             break
                         quality -= 5
                 except BaseException as e:
-                    make_log("Storage", f"Error converting image: {e}" + '\n' + traceback.format_exc(), level="error")
+                    make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error converting image: {e}" + '\n' + traceback.format_exc(), level="error")
 
             if os.path.exists(tempfile_path):
                 async with aiofiles.open(tempfile_path, "rb") as file:
                     content_file_bin = await file.read()
 
-                make_log("Storage", f"Image {content_sha256} converted successfully")
+                make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Image {content_sha256} converted successfully", level='debug')
                 accept_type = 'image/jpeg'
             else:
                 tempfile_path = tempfile_path[:-5]

app/api/routes/progressive_storage.py

@@ -11,36 +11,38 @@ from base58 import b58encode
 from sanic import response
 
 from app.core.logger import make_log
+from sqlalchemy import select
 from app.core.models.node_storage import StoredContent
 from app.core._config import UPLOADS_DIR
+from app.core.models.content_v3 import ContentDerivative
 from app.core._utils.resolve_content import resolve_content
 
 
 # POST /api/v1.5/storage
 async def s_api_v1_5_storage_post(request):
     # Log the receipt of a chunk upload request
-    make_log("uploader_v1.5", "Received chunk upload request", level="INFO")
+    make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Received chunk upload request", level="INFO")
     
     # Get the provided file name from header and decode it from base64
     provided_filename_b64 = request.headers.get("X-File-Name")
     if not provided_filename_b64:
-        make_log("uploader_v1.5", "Missing X-File-Name header", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Missing X-File-Name header", level="ERROR")
         return response.json({"error": "Missing X-File-Name header"}, status=400)
     try:
         provided_filename = b64decode(provided_filename_b64).decode("utf-8")
     except Exception as e:
-        make_log("uploader_v1.5", f"Invalid X-File-Name header: {e}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Invalid X-File-Name header: {e}", level="ERROR")
         return response.json({"error": "Invalid X-File-Name header"}, status=400)
     
     # Get X-Chunk-Start header (must be provided) and parse it as integer
     chunk_start_header = request.headers.get("X-Chunk-Start")
     if chunk_start_header is None:
-        make_log("uploader_v1.5", "Missing X-Chunk-Start header", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Missing X-Chunk-Start header", level="ERROR")
         return response.json({"error": "Missing X-Chunk-Start header"}, status=400)
     try:
         chunk_start = int(chunk_start_header)
     except Exception as e:
-        make_log("uploader_v1.5", f"Invalid X-Chunk-Start header: {e}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Invalid X-Chunk-Start header: {e}", level="ERROR")
         return response.json({"error": "Invalid X-Chunk-Start header"}, status=400)
     
     # Enforce maximum chunk size (80 MB) using Content-Length header if provided
@@ -50,7 +52,7 @@ async def s_api_v1_5_storage_post(request):
         try:
             content_length = int(content_length)
             if content_length > max_chunk_size:
-                make_log("uploader_v1.5", f"Chunk size {content_length} exceeds maximum allowed", level="ERROR")
+                make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Chunk size {content_length} exceeds maximum allowed", level="ERROR")
                 return response.json({"error": "Chunk size exceeds maximum allowed (80 MB)"}, status=400)
         except:
             pass
@@ -62,9 +64,9 @@ async def s_api_v1_5_storage_post(request):
         # New upload session: generate a new uuid
         upload_id = str(uuid4())
         is_new_upload = True
-        make_log("uploader_v1.5", f"Starting new upload session with ID: {upload_id}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Start new upload session id={upload_id}", level="INFO")
     else:
-        make_log("uploader_v1.5", f"Resuming upload session with ID: {upload_id}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Resume upload session id={upload_id}", level="DEBUG")
     
     # Determine the temporary file path based on upload_id
     temp_path = os.path.join(UPLOADS_DIR, f"v1.5_upload_{upload_id}")
@@ -76,10 +78,10 @@ async def s_api_v1_5_storage_post(request):
     
     # If the provided chunk_start is less than current_size, the chunk is already received
     if chunk_start < current_size:
-        make_log("uploader_v1.5", f"Chunk starting at {chunk_start} already received, current size: {current_size}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Chunk at {chunk_start} already received; size={current_size}", level="DEBUG")
         return response.json({"upload_id": upload_id, "current_size": current_size})
     elif chunk_start > current_size:
-        make_log("uploader_v1.5", f"Chunk start {chunk_start} does not match current file size {current_size}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Chunk start {chunk_start} != current size {current_size}", level="ERROR")
         return response.json({"error": "Chunk start does not match current file size"}, status=400)
     
     # Append the received chunk to the temporary file
@@ -93,9 +95,9 @@ async def s_api_v1_5_storage_post(request):
                 async for chunk in request.stream:
                     await out_file.write(chunk)
         new_size = os.path.getsize(temp_path)
-        make_log("uploader_v1.5", f"Appended chunk. New file size: {new_size}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Appended chunk. size={new_size}", level="DEBUG")
     except Exception as e:
-        make_log("uploader_v1.5", f"Error saving chunk: {e}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Error saving chunk: {e}", level="ERROR")
         return response.json({"error": "Failed to save chunk"}, status=500)
     
     # If computed hash matches the provided one, the final chunk has been received
@@ -111,28 +113,28 @@ async def s_api_v1_5_storage_post(request):
             stdout, stderr = await proc.communicate()
             if proc.returncode != 0:
                 error_msg = stderr.decode().strip()
-                make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR")
+                make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} sha256sum error: {error_msg}", level="ERROR")
                 return response.json({"error": "Failed to compute file hash"}, status=500)
             computed_hash_hex = stdout.decode().split()[0].strip()
             computed_hash_bytes = bytes.fromhex(computed_hash_hex)
             computed_hash_b58 = b58encode(computed_hash_bytes).decode()
-            make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Computed hash (base58): {computed_hash_b58}", level="INFO")
         except Exception as e:
-            make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Error computing file hash: {e}", level="ERROR")
             return response.json({"error": "Error computing file hash"}, status=500)
 
         final_path = os.path.join(UPLOADS_DIR, f"{computed_hash_b58}")
         try:
             os.rename(temp_path, final_path)
-            make_log("uploader_v1.5", f"Final chunk received. File renamed to: {final_path}", level="INFO")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Final chunk received. Renamed to: {final_path}", level="INFO")
         except Exception as e:
-            make_log("uploader_v1.5", f"Error renaming file: {e}", level="ERROR")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Error renaming file: {e}", level="ERROR")
             return response.json({"error": "Failed to finalize file storage"}, status=500)
         
         db_session = request.ctx.db_session
-        existing = db_session.query(StoredContent).filter_by(hash=computed_hash_b58).first()
+        existing = (await db_session.execute(select(StoredContent).where(StoredContent.hash == computed_hash_b58))).scalars().first()
         if existing:
-            make_log("uploader_v1.5", f"File with hash {computed_hash_b58} already exists in DB", level="INFO")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} File already exists in DB: {computed_hash_b58}", level="INFO")
             serialized_v2 = existing.cid.serialize_v2()
             serialized_v1 = existing.cid.serialize_v1()
             return response.json({
@@ -156,10 +158,10 @@ async def s_api_v1_5_storage_post(request):
                 created=datetime.utcnow()
             )
             db_session.add(new_content)
-            db_session.commit()
+            await db_session.commit()
-            make_log("uploader_v1.5", f"New file stored and indexed for user {user_id} with hash {computed_hash_b58}", level="INFO")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Stored new file user={user_id} hash={computed_hash_b58}", level="INFO")
         except Exception as e:
-            make_log("uploader_v1.5", f"Database error: {e}", level="ERROR")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Database error: {e}", level="ERROR")
             return response.json({"error": "Database error"}, status=500)
         
         serialized_v2 = new_content.cid.serialize_v2()
@@ -178,7 +180,7 @@ async def s_api_v1_5_storage_post(request):
 
 # GET /api/v1.5/storage/<file_hash>
 async def s_api_v1_5_storage_get(request, file_hash):
-    make_log("uploader_v1.5", f"Received file retrieval request for hash: {file_hash}", level="INFO")
+    make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Retrieve file hash={file_hash}", level="INFO")
     
     try:
         file_hash = b58encode(resolve_content(file_hash)[0].content_hash).decode()
@@ -187,11 +189,11 @@ async def s_api_v1_5_storage_get(request, file_hash):
 
     final_path = os.path.join(UPLOADS_DIR, f"{file_hash}")
     if not os.path.exists(final_path):
-        make_log("uploader_v1.5", f"File not found: {final_path}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} File not found: {final_path}", level="ERROR")
         return response.json({"error": "File not found"}, status=404)
 
     db_session = request.ctx.db_session
-    stored = db_session.query(StoredContent).filter_by(hash=file_hash).first()
+    stored = (await db_session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first()
     if stored and stored.filename:
         filename_for_mime = stored.filename
     else:
@@ -204,8 +206,17 @@ async def s_api_v1_5_storage_get(request, file_hash):
     file_size = os.path.getsize(final_path)
     range_header = request.headers.get("Range")
     
+    # touch derivative last_access_at if exists
+    try:
+        cd = (await request.ctx.db_session.execute(select(ContentDerivative).where(ContentDerivative.local_path.like(f"%/{file_hash}")))).scalars().first()
+        if cd:
+            cd.last_access_at = datetime.utcnow()
+            await request.ctx.db_session.commit()
+    except Exception:
+        pass
+
     if range_header:
-        make_log("uploader_v1.5", f"Processing Range header: {range_header}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Processing Range: {range_header}", level="DEBUG")
         range_spec = range_header.strip().lower()
         if not range_spec.startswith("bytes="):
             make_log("uploader_v1.5", f"Invalid Range header: {range_header}", level="ERROR")

app/api/routes/sync.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from datetime import datetime
+from sanic import response
+from sqlalchemy import select
+
+from app.core.ipfs_client import pin_add, pin_ls
+from app.core.logger import make_log
+from app.core.models.content_v3 import EncryptedContent, IpfsSync
+from app.core.network.nodesig import verify_request
+from app.core.network.guard import check_rate_limit
+
+
+async def s_api_v1_sync_pin(request):
+    # Rate limit per IP and require NodeSig for POST
+    remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip()
+    if not check_rate_limit(request.app.ctx.memory, remote_ip):
+        return response.json({"error": "RATE_LIMIT"}, status=429)
+
+    ok, node_id, reason = verify_request(request, request.app.ctx.memory)
+    if not ok:
+        return response.json({"error": reason or "UNAUTHORIZED"}, status=401)
+
+    data = request.json or {}
+    cid = data.get("encrypted_cid")
+    if not cid:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+
+    session = request.ctx.db_session
+    row = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+    if not row:
+        # create record with minimal info (unknown meta)
+        row = EncryptedContent(
+            encrypted_cid=cid,
+            title=cid,
+            description="",
+            content_type="application/octet-stream",
+            preview_enabled=False,
+        )
+        session.add(row)
+        await session.flush()
+    sync = (await session.execute(select(IpfsSync).where(IpfsSync.content_id == row.id))).scalars().first()
+    if not sync:
+        sync = IpfsSync(content_id=row.id, pin_state='queued')
+        session.add(sync)
+        await session.flush()
+
+    try:
+        await pin_add(cid, recursive=True)
+        sync.pin_state = 'pinned'
+        sync.pinned_at = datetime.utcnow()
+    except Exception as e:
+        make_log("sync", f"pin failed: {e}", level="error")
+        sync.pin_state = 'failed'
+        sync.pin_error = str(e)
+    await session.commit()
+    return response.json({"ok": True, "state": sync.pin_state})
+
+
+async def s_api_v1_sync_status(request):
+    cid = request.args.get("cid")
+    if not cid:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+    try:
+        info = await pin_ls(cid)
+        state = 'pinned' if info else 'not_pinned'
+    except Exception:
+        state = 'not_pinned'
+        info = {}
+    return response.json({"cid": cid, "state": state, "info": info})

app/api/routes/tonconnect.py

@@ -4,6 +4,7 @@ from aiogram.utils.web_app import safe_parse_webapp_init_data
 from sanic import response
 
 from app.core._blockchain.ton.connect import TonConnect, unpack_wallet_info, WalletConnection
+from sqlalchemy import select, and_
 from app.core._config import TELEGRAM_API_KEY
 from app.core.models.user import User
 from app.core.logger import make_log
@@ -23,8 +24,19 @@ async def s_api_v1_tonconnect_new(request):
     db_session = request.ctx.db_session
     user = request.ctx.user
     memory = request.ctx.memory
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user)
+    # Try restore last connection from DB
+    ton_connection = (await db_session.execute(select(WalletConnection).where(
+        and_(
+            WalletConnection.user_id == user.id,
+            WalletConnection.invalidated == False,
+            WalletConnection.network == 'ton'
+        )
+    ).order_by(WalletConnection.created.desc()))).scalars().first()
+    if ton_connection:
+        ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"])
         await ton_connect.restore_connection()
+    else:
+        ton_connect = TonConnect()
     make_log("TonConnect_API", f"SDK connected?: {ton_connect.connected}", level='info')
     if ton_connect.connected:
         return response.json({"error": "Already connected"}, status=400)
@@ -47,13 +59,11 @@ async def s_api_v1_tonconnect_logout(request):
     user = request.ctx.user
     memory = request.ctx.memory
 
-    wallet_connections = db_session.query(WalletConnection).filter(
+    result = await db_session.execute(select(WalletConnection).where(
-        WalletConnection.user_id == user.id,
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
-        WalletConnection.invalidated == False
+    ))
-    ).all()
+    wallet_connections = result.scalars().all()
     for wallet_connection in wallet_connections:
         wallet_connection.invalidated = True
-
+    await db_session.commit()
-    db_session.commit()
     return response.json({"success": True})
-

app/api/routes/upload_status.py

@@ -0,0 +1,54 @@
+from sanic import response
+from sqlalchemy import select
+
+from app.core.models.content_v3 import UploadSession, EncryptedContent, ContentDerivative
+from app.core._utils.resolve_content import resolve_content
+
+
+async def s_api_v1_upload_status(request, upload_id: str):
+    session = request.ctx.db_session
+    row = await session.get(UploadSession, upload_id)
+    if not row:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+
+    encrypted_hash = None
+    conversion = {"state": "not_started", "details": []}
+
+    if row.encrypted_cid:
+        cid_obj, err = resolve_content(row.encrypted_cid)
+        if not err:
+            encrypted_hash = cid_obj.content_hash_b58
+        ec = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == row.encrypted_cid))).scalars().first()
+        if ec:
+            derivative_rows = (await session.execute(
+                select(ContentDerivative.kind, ContentDerivative.status).where(ContentDerivative.content_id == ec.id)
+            )).all()
+            details = [
+                {"kind": kind, "status": status}
+                for kind, status in derivative_rows
+            ]
+            required = {"decrypted_high", "decrypted_low"}
+            if ec.preview_enabled and ec.content_type.startswith("video/"):
+                required.add("decrypted_preview")
+            statuses = {kind: status for kind, status in derivative_rows}
+            if required and all(statuses.get(k) == "ready" for k in required):
+                conv_state = "ready"
+            elif any(statuses.get(k) == "failed" for k in required):
+                conv_state = "failed"
+            elif any(statuses.get(k) in ("processing", "pending") for k in required):
+                conv_state = "processing"
+            elif required:
+                conv_state = "pending"
+            else:
+                conv_state = "not_started"
+            conversion = {"state": conv_state, "details": details}
+
+    return response.json({
+        "id": row.id,
+        "state": row.state,
+        "encrypted_cid": row.encrypted_cid,
+        "encrypted_hash": encrypted_hash,
+        "size_bytes": row.size_bytes,
+        "error": row.error,
+        "conversion": conversion,
+    })

app/api/routes/upload_tus.py

@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+import base64
+import json
+import os
+from datetime import datetime
+from typing import Dict, Any
+
+import aiofiles
+from base58 import b58encode
+from sanic import response
+
+from app.core._config import UPLOADS_DIR
+from app.core._secrets import hot_pubkey
+from app.core.crypto.aes_gcm_stream import encrypt_file_to_encf, CHUNK_BYTES
+from app.core.crypto.keywrap import wrap_dek, KeyWrapError
+from app.core.ipfs_client import add_streamed_file
+from app.core.logger import make_log
+from app.core.models.content_v3 import EncryptedContent, ContentKey, IpfsSync, ContentIndexItem, UploadSession
+from app.core.models.node_storage import StoredContent
+from app.core.storage import db_session
+from app.core._utils.resolve_content import resolve_content
+from sqlalchemy import select
+
+
+def _b64(s: bytes) -> str:
+    return base64.b64encode(s).decode()
+
+
+async def s_api_v1_upload_tus_hook(request):
+    """
+    tusd HTTP hook endpoint. We mainly handle post-finish to: encrypt -> IPFS add+pin -> record DB.
+    """
+    try:
+        payload: Dict[str, Any] = request.json
+    except Exception:
+        payload = None
+    if payload is None:
+        raw_body = request.body or b''
+        try:
+            payload = json.loads(raw_body) if raw_body else {}
+        except Exception:
+            payload = {}
+    event = (payload.get("Type") or payload.get("type") or
+             payload.get("Event") or payload.get("event") or
+             payload.get("Hook") or payload.get("hook") or
+             payload.get("HookName") or payload.get("hook_name") or
+             request.headers.get("Hook-Name") or request.headers.get("hook-name"))
+    upload = payload.get("Upload") or payload.get("upload") or {}
+
+    if not event:
+        hook_name = (payload.get("HookName") or payload.get("hook") or
+                     payload.get("hook_name") or request.headers.get("Hook-Name"))
+        raw = request.body or b''
+        preview = raw[:512]
+        make_log("tus-hook", f"Missing event type in hook payload; ignoring (hook={hook_name}, keys={list(payload.keys())}, raw={preview!r})", level="warning")
+        return response.json({"ok": True, "skipped": True})
+
+    if event not in ("post-finish", "postfinish"):
+        # accept but ignore other events
+        return response.json({"ok": True})
+
+    # Extract storage path from tusd payload
+    storage = upload.get("Storage") or {}
+    file_path = storage.get("Path") or storage.get("path")
+    if not file_path:
+        return response.json({"ok": False, "error": "NO_STORAGE_PATH"}, status=400)
+
+    meta = upload.get("MetaData") or {}
+    # Common metadata keys
+    title = meta.get("title") or meta.get("Title") or meta.get("name") or "Untitled"
+    description = meta.get("description") or meta.get("Description") or ""
+    content_type = meta.get("content_type") or meta.get("Content-Type") or "application/octet-stream"
+    preview_enabled = content_type.startswith("audio/") or content_type.startswith("video/")
+    # Optional preview window overrides from tus metadata
+    try:
+        start_ms = int(meta.get("preview_start_ms") or 0)
+        dur_ms = int(meta.get("preview_duration_ms") or 30000)
+    except Exception:
+        start_ms, dur_ms = 0, 30000
+
+    # Record/Update upload session
+    upload_id = upload.get("ID") or upload.get("Id") or upload.get("id")
+    try:
+        size = int(upload.get("Size") or 0)
+    except Exception:
+        size = None
+
+    async with db_session() as session:
+        us = (await session.get(UploadSession, upload_id)) if upload_id else None
+        if not us and upload_id:
+            us = UploadSession(
+                id=upload_id,
+                filename=os.path.basename(file_path),
+                size_bytes=size,
+                state='processing',
+                encrypted_cid=None,
+            )
+            session.add(us)
+            await session.commit()
+
+    # Read & encrypt by streaming (ENCF v1 / AES-GCM)
+    # Generate per-content random DEK and salt
+    dek = os.urandom(32)
+    salt = os.urandom(16)
+    key_fpr = b58encode(hot_pubkey).decode()  # fingerprint as our node id for now
+
+    # Stream encrypt into IPFS add
+    try:
+        wrapped_dek = wrap_dek(dek)
+    except KeyWrapError as e:
+        make_log("tus-hook", f"Key wrap failed: {e}", level="error")
+        async with db_session() as session:
+            if upload_id:
+                us = await session.get(UploadSession, upload_id)
+                if us:
+                    us.state = 'failed'
+                    us.error = str(e)
+                    await session.commit()
+        return response.json({"ok": False, "error": "KEY_WRAP_FAILED"}, status=500)
+
+    try:
+        with open(file_path, 'rb') as f:
+            result = await add_streamed_file(
+                encrypt_file_to_encf(f, dek, CHUNK_BYTES, salt),
+                filename=os.path.basename(file_path),
+                params={},
+            )
+    except Exception as e:
+        make_log("tus-hook", f"Encrypt+add failed: {e}", level="error")
+        # mark failed
+        async with db_session() as session:
+            if upload_id:
+                us = await session.get(UploadSession, upload_id)
+                if us:
+                    us.state = 'failed'
+                    us.error = str(e)
+                    await session.commit()
+        return response.json({"ok": False, "error": "ENCRYPT_ADD_FAILED"}, status=500)
+
+    encrypted_cid = result.get("Hash")
+    try:
+        enc_size = int(result.get("Size") or 0)
+    except Exception:
+        enc_size = None
+
+    encrypted_cid_obj, cid_err = resolve_content(encrypted_cid)
+    if cid_err:
+        make_log("tus-hook", f"Encrypted CID resolve failed: {cid_err}", level="error")
+        return response.json({"ok": False, "error": "INVALID_ENCRYPTED_CID"}, status=500)
+    encrypted_hash_b58 = encrypted_cid_obj.content_hash_b58
+
+    # Persist records
+    async with db_session() as session:
+        ec = EncryptedContent(
+            encrypted_cid=encrypted_cid,
+            title=title,
+            description=description,
+            content_type=content_type,
+            enc_size_bytes=enc_size,
+            plain_size_bytes=os.path.getsize(file_path),
+            preview_enabled=preview_enabled,
+            preview_conf=({"duration_ms": dur_ms, "intervals": [[start_ms, start_ms + dur_ms]]} if preview_enabled else {}),
+            aead_scheme="AES_GCM",
+            chunk_bytes=CHUNK_BYTES,
+            salt_b64=_b64(salt),
+        )
+        session.add(ec)
+        await session.flush()
+
+        ck = ContentKey(
+            content_id=ec.id,
+            key_ciphertext_b64=wrapped_dek,
+            key_fingerprint=key_fpr,
+            issuer_node_id=key_fpr,
+            allow_auto_grant=True,
+        )
+        session.add(ck)
+        await session.flush()
+
+        sync = IpfsSync(
+            content_id=ec.id,
+            pin_state='pinned',
+            bytes_total=enc_size,
+            bytes_fetched=enc_size,
+            pinned_at=datetime.utcnow(),
+        )
+        session.add(sync)
+
+        existing_encrypted_content = (await session.execute(
+            select(StoredContent).where(StoredContent.hash == encrypted_hash_b58)
+        )).scalars().first()
+        if not existing_encrypted_content:
+            placeholder_meta = {
+                'content_type': content_type,
+                'storage': 'ipfs',
+                'encrypted_cid': encrypted_cid,
+                'upload_id': upload_id,
+                'source': 'tusd'
+            }
+            encrypted_stored_content = StoredContent(
+                type="local/encrypted_ipfs",
+                hash=encrypted_hash_b58,
+                content_id=encrypted_cid,
+                filename=os.path.basename(file_path),
+                meta=placeholder_meta,
+                user_id=request.ctx.user.id if request.ctx.user else None,
+                owner_address=None,
+                encrypted=True,
+                decrypted_content_id=None,
+                key_id=None,
+                created=datetime.utcnow(),
+            )
+            session.add(encrypted_stored_content)
+
+        # Publish signed index item
+        item = {
+            "encrypted_cid": encrypted_cid,
+            "title": title,
+            "description": description,
+            "content_type": content_type,
+            "size_bytes": enc_size,
+            "preview_enabled": preview_enabled,
+            "preview_conf": ec.preview_conf,
+            "issuer_node_id": key_fpr,
+            "salt_b64": _b64(salt),
+        }
+        try:
+            from app.core._crypto.signer import Signer
+            from app.core._secrets import hot_seed
+            signer = Signer(hot_seed)
+            blob = json.dumps(item, sort_keys=True, separators=(",", ":")).encode()
+            sig = signer.sign(blob)
+        except Exception:
+            sig = ""
+        session.add(ContentIndexItem(encrypted_cid=encrypted_cid, payload=item, sig=sig))
+
+        await session.commit()
+
+    # Update upload session with result and purge staging to avoid duplicates
+    async with db_session() as session:
+        if upload_id:
+            us = await session.get(UploadSession, upload_id)
+            if us:
+                us.state = 'pinned'
+                us.encrypted_cid = encrypted_cid
+                us.error = None
+                if size:
+                    us.size_bytes = size
+                # prefer using IPFS for downstream conversion; remove staging
+                try:
+                    if file_path and os.path.exists(file_path):
+                        os.remove(file_path)
+                except Exception:
+                    pass
+                us.storage_path = None
+                await session.commit()
+
+    make_log("tus-hook", f"Uploaded+encrypted {file_path} -> {encrypted_cid}")
+    placeholder_path = os.path.join(UPLOADS_DIR, encrypted_hash_b58)
+    if not os.path.exists(placeholder_path):
+        try:
+            async with aiofiles.open(placeholder_path, "wb") as ph:
+                await ph.write(json.dumps({
+                    "ipfs_cid": encrypted_cid,
+                    "note": "Encrypted payload stored in IPFS"
+                }).encode())
+        except Exception as e:
+            make_log("tus-hook", f"Failed to create placeholder for {encrypted_hash_b58}: {e}", level="warning")
+
+    return response.json({"ok": True, "encrypted_cid": encrypted_cid, "upload_id": upload_id})

app/bot/middleware.py

@@ -1,6 +1,7 @@
 from app.core.logger import make_log, logger
 from app.core.models._telegram import Wrapped_CBotChat
 from app.core.models.user import User
+from sqlalchemy import select
 from app.core.storage import db_session
 from aiogram import BaseMiddleware, types
 from app.core.models.messages import KnownTelegramMessage
@@ -21,9 +22,9 @@ class UserDataMiddleware(BaseMiddleware):
 
         # TODO: maybe make users cache
 
-        with db_session(auto_commit=False) as session:
+        async with db_session(auto_commit=False) as session:
             try:
-                user = session.query(User).filter_by(telegram_id=user_id).first()
+                user = (await session.execute(select(User).where(User.telegram_id == user_id))).scalars().first()
             except BaseException as e:
                 logger.error(f"Error when middleware getting user: {e}")
                 user = None
@@ -42,7 +43,7 @@ class UserDataMiddleware(BaseMiddleware):
                     created=datetime.now()
                 )
                 session.add(user)
-                session.commit()
+                await session.commit()
             else:
                 if user.username != update_body.from_user.username:
                     user.username = update_body.from_user.username
@@ -60,7 +61,7 @@ class UserDataMiddleware(BaseMiddleware):
                 }
 
                 user.last_use = datetime.now()
-                session.commit()
+                await session.commit()
 
             data['user'] = user
             data['db_session'] = session
@@ -72,11 +73,11 @@ class UserDataMiddleware(BaseMiddleware):
                 if update_body.text.startswith('/start'):
                     message_type = 'start_command'
 
-                if session.query(KnownTelegramMessage).filter_by(
+                if (await session.execute(select(KnownTelegramMessage).where(
-                    chat_id=update_body.chat.id,
+                    (KnownTelegramMessage.chat_id == update_body.chat.id) &
-                    message_id=update_body.message_id,
+                    (KnownTelegramMessage.message_id == update_body.message_id) &
-                    from_user=True
+                    (KnownTelegramMessage.from_user == True)
-                ).first():
+                ))).scalars().first():
                     make_log("UserDataMiddleware", f"Message {update_body.message_id} already processed", level='debug')
                     return
 
@@ -91,7 +92,7 @@ class UserDataMiddleware(BaseMiddleware):
                     meta={}
                 )
                 session.add(new_message)
-                session.commit()
+                await session.commit()
 
             result = await handler(event, data)
             return result

app/bot/routers/content.py

@@ -1,11 +1,16 @@
 import base58
 from aiogram import types, Router, F
+from collections import defaultdict
+from datetime import datetime
+from typing import Optional
 
 from app.core._config import WEB_APP_URLS
 from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.logger import make_log
 from app.core.models.node_storage import StoredContent
+from app.core.models.content_v3 import UploadSession, EncryptedContent, ContentDerivative
+from sqlalchemy import select, and_, or_
 import json
 
 router = Router()
@@ -17,27 +22,146 @@ def chunks(lst, n):
         yield lst[i:i + n]
 
 
+async def _compute_content_status(db_session, encrypted_cid: Optional[str], fallback_content_type: Optional[str] = None):
+    if not encrypted_cid:
+        return {
+            'final_state': 'uploaded',
+            'conversion_state': 'pending',
+            'upload_state': None,
+            'summary': {},
+            'details': [],
+            'title': None,
+            'content_type': fallback_content_type,
+        }
+
+    ec = (await db_session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == encrypted_cid))).scalars().first()
+    content_type = fallback_content_type or (ec.content_type if ec else None) or 'application/octet-stream'
+
+    derivative_rows = []
+    if ec:
+        derivative_rows = (await db_session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id))).scalars().all()
+    upload_row = (await db_session.execute(select(UploadSession).where(UploadSession.encrypted_cid == encrypted_cid))).scalars().first()
+
+    derivative_sorted = sorted(derivative_rows, key=lambda row: row.created_at or datetime.min)
+    derivative_latest = {}
+    summary = defaultdict(int)
+    details = []
+    for row in derivative_sorted:
+        derivative_latest[row.kind] = row
+    for kind, row in derivative_latest.items():
+        summary[row.status] += 1
+        details.append({
+            'kind': kind,
+            'status': row.status,
+            'size_bytes': row.size_bytes,
+            'error': row.error,
+            'updated_at': (row.last_access_at or row.created_at).isoformat() + 'Z' if (row.last_access_at or row.created_at) else None,
+        })
+
+    required = {'decrypted_low', 'decrypted_high'}
+    if content_type.startswith('video/'):
+        required.add('decrypted_preview')
+
+    statuses_by_kind = {kind: derivative_latest[kind].status for kind in required if kind in derivative_latest}
+    conversion_state = 'pending'
+    if required and all(statuses_by_kind.get(kind) == 'ready' for kind in required):
+        conversion_state = 'ready'
+    elif any(statuses_by_kind.get(kind) == 'failed' for kind in required):
+        conversion_state = 'failed'
+    elif any(statuses_by_kind.get(kind) in ('processing', 'pending') for kind in required):
+        conversion_state = 'processing'
+    elif statuses_by_kind:
+        conversion_state = 'partial'
+
+    upload_state = upload_row.state if upload_row else None
+    final_state = 'ready' if conversion_state == 'ready' else None
+    if not final_state:
+        if conversion_state == 'failed' or upload_state in ('failed', 'conversion_failed'):
+            final_state = 'failed'
+        elif conversion_state in ('processing', 'partial') or upload_state in ('processing', 'pinned'):
+            final_state = 'processing'
+        else:
+            final_state = 'uploaded'
+
+    return {
+        'final_state': final_state,
+        'conversion_state': conversion_state,
+        'upload_state': upload_state,
+        'summary': dict(summary),
+        'details': details,
+        'title': ec.title if ec else None,
+        'content_type': content_type,
+    }
+
+
 async def t_callback_owned_content(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
     message_text = user.translated("ownedContent_menu")
     content_list = []
-    for content in db_session.query(StoredContent).filter_by(
+    user_addr = await user.wallet_address_async(db_session)
-        owner_address=user.wallet_address(db_session),
+    conditions = []
-        type='onchain/content'
+    if user_addr:
-    ).all():
+        conditions.append(and_(StoredContent.owner_address == user_addr, StoredContent.type.like('onchain%')))
+    conditions.append(and_(StoredContent.user_id == user.id, StoredContent.type.like('local/%')))
+
+    if not conditions:
+        conditions = [StoredContent.user_id == user.id]
+
+    stmt = select(StoredContent).where(
+        StoredContent.disabled.is_(None),
+        or_(*conditions) if len(conditions) > 1 else conditions[0]
+    ).order_by(StoredContent.created.desc())
+
+    rows = (await db_session.execute(stmt)).scalars().all()
+
+    onchain_hashes = set()
+    local_items = []
+
+    icon_map = {
+        'ready': '✅',
+        'processing': '⏳',
+        'failed': '⚠️',
+        'uploaded': '📦',
+    }
+
+    for content in rows:
+        meta = content.meta or {}
+        encrypted_cid = meta.get('content_cid') or meta.get('encrypted_cid') or content.content_id
+        status_info = await _compute_content_status(db_session, encrypted_cid, meta.get('content_type'))
+        icon = icon_map.get(status_info['final_state'], '📦')
+
+        if content.type.startswith('onchain'):
             try:
-            metadata_content = StoredContent.from_cid(db_session, content.json_format()['metadata_cid'])
+                metadata_content = await StoredContent.from_cid_async(db_session, content.json_format()['metadata_cid'])
                 with open(metadata_content.filepath, 'r') as f:
                     metadata_content_json = json.loads(f.read())
             except BaseException as e:
                 make_log("OwnedContent", f"Can't get metadata content: {e}", level='warning')
                 continue
 
+            onchain_hashes.add(content.hash)
+            display_name = metadata_content_json.get('name') or content.cid.serialize_v2()
             content_list.append([
                 {
-                'text': metadata_content_json['name'],
+                    'text': f"{icon} {display_name}"[:64],
                     'callback_data': f'NC_{content.id}'
                 }
             ])
+        else:
+            local_items.append((content, status_info, icon))
+
+    for content, status_info, icon in local_items:
+        if content.hash in onchain_hashes:
+            continue
+        meta = content.meta or {}
+        encrypted_cid = meta.get('encrypted_cid') or content.content_id
+        display_name = status_info['title'] or content.filename or content.cid.serialize_v2()
+        button_text = f"{icon} {display_name}"
+        content_list.append([
+            {
+                'text': button_text[:64],
+                'callback_data': f'LC_{content.id}'
+            }
+        ])
 
     return await tg_process_template(
         chat_wrap, message_text,
@@ -59,10 +183,9 @@ async def t_callback_owned_content(query: types.CallbackQuery, memory=None, user
 
 async def t_callback_node_content(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
     content_oid = int(query.data.split('_')[1])
+    row = (await db_session.execute(select(StoredContent).where(StoredContent.id == content_oid))).scalars().first()
     return await chat_wrap.send_content(
-        db_session, db_session.query(StoredContent).filter_by(
+        db_session, row,
-            id=content_oid
-        ).first(),
         extra_buttons=[
             [{
                 'text': user.translated('back_button'),
@@ -76,3 +199,51 @@ async def t_callback_node_content(query: types.CallbackQuery, memory=None, user=
 
 router.callback_query.register(t_callback_owned_content, F.data == 'ownedContent')
 router.callback_query.register(t_callback_node_content, F.data.startswith('NC_'))
+
+
+async def t_callback_local_content(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
+    content_oid = int(query.data.split('_')[1])
+    content = (await db_session.execute(select(StoredContent).where(StoredContent.id == content_oid))).scalars().first()
+    if not content:
+        return await query.answer(user.translated('error_contentNotFound'), show_alert=True)
+
+    upload_id = (content.meta or {}).get('upload_id')
+    upload_session = await db_session.get(UploadSession, upload_id) if upload_id else None
+
+    encrypted_cid = (content.meta or {}).get('encrypted_cid') or content.content_id
+    status_info = await _compute_content_status(db_session, encrypted_cid, (content.meta or {}).get('content_type'))
+    display_name = status_info['title'] or content.filename or content.cid.serialize_v2()
+    state_label = {
+        'ready': 'Готов',
+        'processing': 'Обработка',
+        'failed': 'Ошибка',
+        'uploaded': 'Загружено',
+    }.get(status_info['final_state'], 'Статус неизвестен')
+
+    lines = [
+        f"<b>{display_name}</b>",
+        f"Состояние: {state_label}"
+    ]
+    if upload_session:
+        lines.append(f"Статус загрузки: {upload_session.state}")
+        if upload_session.error:
+            lines.append(f"Ошибка: {upload_session.error}")
+    if status_info['summary']:
+        lines.append("Конвертация:")
+        for status, count in status_info['summary'].items():
+            lines.append(f"• {status}: {count}")
+
+    await chat_wrap.send_message(
+        '\n'.join(lines),
+        message_type='notification',
+        message_meta={'content_id': content.id},
+        reply_markup=get_inline_keyboard([
+            [{
+                'text': user.translated('back_button'),
+                'callback_data': 'ownedContent'
+            }]
+        ])
+    )
+
+
+router.callback_query.register(t_callback_local_content, F.data.startswith('LC_'))

app/bot/routers/home.py

@@ -3,6 +3,7 @@ from aiogram.filters import Command
 from tonsdk.utils import Address
 
 from app.core._blockchain.ton.connect import TonConnect
+from sqlalchemy import select, and_
 from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.models.wallet_connection import WalletConnection
@@ -32,7 +33,13 @@ async def send_home_menu(chat_wrap, user, wallet_connection, **kwargs):
 
 
 async def send_connect_wallets_list(db_session, chat_wrap, user, **kwargs):
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
+    # Try to restore existing connection via DB
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+    if ton_connection:
         await ton_connect.restore_connection()
     wallets = ton_connect._sdk_client.get_wallets()
     message_text = user.translated("connectWalletsList_menu")
@@ -66,10 +73,9 @@ async def t_home_menu(__msg, **extra):
     else:
         message_id = None
 
-    wallet_connection = db_session.query(WalletConnection).filter(
+    wallet_connection = (await db_session.execute(select(WalletConnection).where(
-        WalletConnection.user_id == user.id,
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
-        WalletConnection.invalidated == False
+    ))).scalars().first()
-    ).first()
 
     # if not wallet_connection:
     #     return await send_connect_wallets_list(db_session, chat_wrap, user, message_id=message_id)

app/bot/routers/tonconnect.py

@@ -7,6 +7,7 @@ from aiogram.filters import Command
 
 from app.bot.routers.home import send_connect_wallets_list, send_home_menu
 from app.core._blockchain.ton.connect import TonConnect, unpack_wallet_info
+from sqlalchemy import select, and_
 from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.logger import make_log
@@ -33,15 +34,21 @@ async def t_tonconnect_dev_menu(message: types.Message, memory=None, user=None,
 
     keyboard = []
 
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
+    # Restore recent connection
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
     make_log("TonConnect_DevMenu", f"Available wallets: {ton_connect._sdk_client.get_wallets()}", level='debug')
+    if ton_connection:
         await ton_connect.restore_connection()
     make_log("TonConnect_DevMenu", f"SDK connected?: {ton_connect.connected}", level='info')
     if not ton_connect.connected:
         if ton_connection:
             make_log("TonConnect_DevMenu", f"Invalidating old connection", level='debug')
             ton_connection.invalidated = True
-            db_session.commit()
+            await db_session.commit()
 
         message_text = f"""<b>Wallet is not connected</b>
         
@@ -71,7 +78,12 @@ Use /dev_tonconnect <code>{wallet_app_name}</code> for connect to wallet."""
 
 async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
     wallet_app_name = query.data.split("_")[1]
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user)
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+    if ton_connection:
         await ton_connect.restore_connection()
     connection_link = await ton_connect.new_connection(wallet_app_name)
     ton_connect.connected
@@ -98,10 +110,9 @@ async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, us
 
     start_ts = datetime.now()
     while datetime.now() - start_ts < timedelta(seconds=180):
-        new_connection = db_session.query(WalletConnection).filter(
+        new_connection = (await db_session.execute(select(WalletConnection).where(
-            WalletConnection.user_id == user.id,
+            and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
-            WalletConnection.invalidated == False
+        ))).scalars().first()
-        ).first()
         if new_connection:
             await tg_process_template(
                 chat_wrap, user.translated('p_successConnectWallet')
@@ -115,14 +126,13 @@ async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, us
 
 
 async def t_callback_disconnect_wallet(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
-    wallet_connections = db_session.query(WalletConnection).filter(
+    wallet_connections = (await db_session.execute(select(WalletConnection).where(
-        WalletConnection.user_id == user.id,
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
-        WalletConnection.invalidated == False
+    ))).scalars().all()
-    ).all()
     for wallet_connection in wallet_connections:
         wallet_connection.invalidated = True
 
-    db_session.commit()
+    await db_session.commit()
 
     return await send_home_menu(chat_wrap, user, None, message_id=query.message.message_id)
 

app/client_bot/routers/content.py

@@ -6,6 +6,7 @@ from aiogram import types, Router, F
 
 from app.core._keyboards import get_inline_keyboard
 from app.core.models.node_storage import StoredContent
+from sqlalchemy import select, and_
 import json
 from app.core.logger import make_log
 from app.core.models.content.user_content import UserAction, UserContent
@@ -30,7 +31,7 @@ CACHE_CHAT_ID = -1002390124789
 async def t_callback_purchase_node_content(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
     content_oid = int(query.data.split('_')[1])
     is_cancel_request = query.data.split('_')[2] == 'cancel' if len(query.data.split('_')) > 2 else False
-    content = db_session.query(StoredContent).filter_by(id=content_oid).first()
+    content = (await db_session.execute(select(StoredContent).where(StoredContent.id == content_oid))).scalars().first()
     if not content:
         return await query.answer(user.translated('error_contentNotFound'), show_alert=True)
 
@@ -43,11 +44,16 @@ async def t_callback_purchase_node_content(query: types.CallbackQuery, memory=No
 
         make_log("Purchase", f"User {user.id} initiated purchase for content ID {content_oid}. License price: {license_price_num}.", level='info')
 
-        ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
+        result = await db_session.execute(select(WalletConnection).where(
+            and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+        ).order_by(WalletConnection.created.desc()))
+        ton_connection = result.scalars().first()
+        ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+        if ton_connection:
             await ton_connect.restore_connection()
         assert ton_connect.connected, "No connected wallet"
 
-        user_wallet_address = user.wallet_address(db_session)
+        user_wallet_address = await user.wallet_address_async(db_session)
 
         memory._app.add_task(ton_connect._sdk_client.send_transaction({
             'valid_until': int(datetime.now().timestamp() + 300),
@@ -76,18 +82,15 @@ async def t_callback_purchase_node_content(query: types.CallbackQuery, memory=No
     else:
         # Logging cancellation attempt with detailed information
         make_log("Purchase", f"User {user.id} cancelled purchase for content ID {content_oid}.", level='info')
-        action = db_session.query(UserAction).filter_by(
+        action = (await db_session.execute(select(UserAction).where(
-            type='purchase',
+            and_(UserAction.type == 'purchase', UserAction.content_id == content_oid, UserAction.user_id == user.id, UserAction.status == 'requested')
-            content_id=content_oid,
+        ))).scalars().first()
-            user_id=user.id,
-            status='requested'
-        ).first()
         if not action:
             return await query.answer()
 
         action.status = 'canceled'
 
-    db_session.commit()
+    await db_session.commit()
     await chat_wrap.send_content(db_session, content, message_id=query.message.message_id)
 
 
@@ -104,9 +107,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
         args = None
         if source_args_ext.startswith('Q'):
             license_onchain_address = source_args_ext[1:]
-            licensed_content = db_session.query(UserContent).filter_by(
+            licensed_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == license_onchain_address))).scalars().first().content
-                onchain_address=license_onchain_address,
-            ).first().content
             make_log("InlineSearch", f"Query '{query.query}' is a license query for content ID {licensed_content.id}.", level='info')
             args = licensed_content.cid.serialize_v2()
         else:
@@ -118,15 +119,15 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
         content_list = []
         search_query = {'hash': cid.content_hash_b58}
         make_log("InlineSearch", f"Searching with query '{search_query}'.", level='info')
-        content = db_session.query(StoredContent).filter_by(**search_query).first()
+        content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == cid.content_hash_b58))).scalars().first()
-        content_prod = content.open_content(db_session)
+        content_prod = await content.open_content_async(db_session)
         # Get both encrypted and decrypted content objects
         encrypted_content = content_prod['encrypted_content']
         decrypted_content = content_prod['decrypted_content']
         decrypted_content_meta = decrypted_content.json_format()
 
         try:
-            metadata_content = StoredContent.from_cid(db_session, content.json_format()['metadata_cid'])
+            metadata_content = await StoredContent.from_cid_async(db_session, content.json_format()['metadata_cid'])
             with open(metadata_content.filepath, 'r') as f:
                 metadata_content_json = json.loads(f.read())
         except BaseException as e:
@@ -144,7 +145,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
 
         result_kwargs = {}
         try:
-            cover_content = StoredContent.from_cid(db_session, decrypted_content_meta.get('cover_cid') or None)
+            cover_content = await StoredContent.from_cid_async(db_session, decrypted_content_meta.get('cover_cid') or None)
         except BaseException as e:
             cover_content = None
 
@@ -152,9 +153,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
             result_kwargs['thumb_url'] = cover_content.web_url
 
         content_type_declared = decrypted_content_meta.get('content_type', 'application/x-binary').split('/')[0]
-        preview_content = db_session.query(StoredContent).filter_by(
+        preview_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == content.meta.get('converted_content', {}).get('low_preview')))).scalars().first()
-            hash=content.meta.get('converted_content', {}).get('low_preview')
-        ).first()
         content_type_declared = {
             'mp3': 'audio',
             'flac': 'audio',
@@ -196,7 +195,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
                         **decrypted_content.meta,
                         'telegram_file_cache_preview': preview_file_id
                     }
-                    db_session.commit()
+                    await db_session.commit()
                 except Exception as e:
                     # Logging error during preview upload with detailed content type and query information
                     make_log("InlineSearch", f"Error uploading preview for content type '{content_type_declared}' during inline query '{query.query}': {e}", level='error')

app/client_bot/routers/home.py

@@ -3,6 +3,7 @@ from aiogram.filters import Command
 from tonsdk.utils import Address
 
 from app.core._blockchain.ton.connect import TonConnect
+from sqlalchemy import select, and_
 from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.logger import make_log
@@ -32,7 +33,12 @@ async def send_home_menu(chat_wrap, user, wallet_connection, **kwargs):
 
 
 async def send_connect_wallets_list(db_session, chat_wrap, user, **kwargs):
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+    if ton_connection:
         await ton_connect.restore_connection()
     wallets = ton_connect._sdk_client.get_wallets()
     message_text = user.translated("connectWalletsList_menu")
@@ -66,10 +72,9 @@ async def t_home_menu(__msg, **extra):
     else:
         message_id = None
 
-    wallet_connection = db_session.query(WalletConnection).filter(
+    wallet_connection = (await db_session.execute(select(WalletConnection).where(
-        WalletConnection.user_id == user.id,
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
-        WalletConnection.invalidated == False
+    ))).scalars().first()
-    ).first()
 
     # if not wallet_connection:
     #     return await send_connect_wallets_list(db_session, chat_wrap, user, message_id=message_id)
@@ -81,7 +86,10 @@ async def t_home_menu(__msg, **extra):
     make_log("Home", f"Home menu args: {args}", level='debug')
     if args:
         if args[0].startswith('C'):
-            content = StoredContent.from_cid(db_session, args[0][1:])
+            payload = args[0][1:]
+            if '!' in payload:
+                payload = payload.split('!', 1)[0]
+            content = StoredContent.from_cid(db_session, payload)
             return await chat_wrap.send_content(db_session, content, message_id=message_id)
 
     return await send_home_menu(chat_wrap, user, wallet_connection, message_id=message_id)

app/client_bot/routers/stars.py

@@ -1,4 +1,6 @@
 from aiogram import types, Router, F
+from sqlalchemy import select
+
 from app.core.logger import make_log
 from app.core.models import StarsInvoice
 
@@ -12,9 +14,10 @@ async def t_pre_checkout_query_stars_processing(pre_checkout_query: types.PreChe
 
     invoice_id = pre_checkout_query.invoice_payload
 
-    existing_invoice = db_session.query(StarsInvoice).filter(
+    result = await db_session.execute(
-        StarsInvoice.external_id == invoice_id
+        select(StarsInvoice).where(StarsInvoice.external_id == invoice_id)
-    ).first()
+    )
+    existing_invoice = result.scalars().first()
     if not existing_invoice:
         return await pre_checkout_query.answer(ok=False, error_message="Invoice not found")
     

app/client_bot/routers/tonconnect.py

@@ -7,6 +7,7 @@ from aiogram.filters import Command
 
 from app.client_bot.routers.home import send_connect_wallets_list, send_home_menu
 from app.core._blockchain.ton.connect import TonConnect, unpack_wallet_info
+from sqlalchemy import select, and_
 from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.logger import make_log
@@ -34,15 +35,20 @@ async def t_tonconnect_dev_menu(message: types.Message, memory=None, user=None,
 
     keyboard = []
 
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
     make_log("TonConnect_DevMenu", f"Available wallets: {ton_connect._sdk_client.get_wallets()}", level='debug')
+    if ton_connection:
         await ton_connect.restore_connection()
     make_log("TonConnect_DevMenu", f"SDK connected?: {ton_connect.connected}", level='info')
     if not ton_connect.connected:
         if ton_connection:
             make_log("TonConnect_DevMenu", f"Invalidating old connection", level='debug')
             ton_connection.invalidated = True
-            db_session.commit()
+            await db_session.commit()
 
         message_text = f"""<b>Wallet is not connected</b>
 
@@ -73,7 +79,12 @@ Use /dev_tonconnect <code>{wallet_app_name}</code> for connect to wallet."""
 async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, user=None, db_session=None,
                                      chat_wrap=None, **extra):
     wallet_app_name = query.data.split("_")[1]
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user)
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+    if ton_connection:
         await ton_connect.restore_connection()
     connection_link = await ton_connect.new_connection(wallet_app_name)
     ton_connect.connected
@@ -100,10 +111,9 @@ async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, us
 
     start_ts = datetime.now()
     while datetime.now() - start_ts < timedelta(seconds=180):
-        new_connection = db_session.query(WalletConnection).filter(
+        new_connection = (await db_session.execute(select(WalletConnection).where(
-            WalletConnection.user_id == user.id,
+            and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
-            WalletConnection.invalidated == False
+        ))).scalars().first()
-        ).first()
         if new_connection:
             await tg_process_template(
                 chat_wrap, user.translated('p_successConnectWallet')
@@ -118,14 +128,13 @@ async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, us
 
 async def t_callback_disconnect_wallet(query: types.CallbackQuery, memory=None, user=None, db_session=None,
                                        chat_wrap=None, **extra):
-    wallet_connections = db_session.query(WalletConnection).filter(
+    wallet_connections = (await db_session.execute(select(WalletConnection).where(
-        WalletConnection.user_id == user.id,
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
-        WalletConnection.invalidated == False
+    ))).scalars().all()
-    ).all()
     for wallet_connection in wallet_connections:
         wallet_connection.invalidated = True
 
-    db_session.commit()
+    await db_session.commit()
     return await send_home_menu(chat_wrap, user, None, message_id=query.message.message_id)
 
 

app/core/_blockchain/ton/contracts/blank.py

@@ -5,7 +5,7 @@ from app.core._secrets import service_wallet
 
 
 class Blank(Contract):
-    code = 'B5EE9C72010104010042000114FF00F4A413F4BCF2C80B010202CA0203004FD043A0E9AE43F48061DA89A1F480618E0BE5C323A803A1A843F60803A1DA3DDAA7A861DAA9E2026F0007A0DD7C12'
+    code = 'b5ee9c72410104010042000114ff00f4a413f4bcf2c80b010202ca03020007a0dd7c12004fd043a0e9ae43f48061da89a1f480618e0be5c323a803a1a843f60803a1da3ddaa7a861daa9e2026f102bdd33'
 
     def __init__(self, **kwargs):
         kwargs['code'] = Cell.one_from_boc(self.code)

app/core/_blockchain/ton/contracts/cop_nft.py

@@ -6,7 +6,7 @@ from app.core._config import MY_PLATFORM_CONTRACT
 
 
 class COP_NFT(Contract):
-    code = 'b5ee9c7241022f01000ac8000114ff00f4a413f4bcf2c80b010201620b020201200803020120070402016a0605004bae43b941781bb12d2f0dfa54dda7472bd6fbc06813c0f71615c0094ee3548791d80b2bfcecc0007dac1ff6a26869ff80fc317d2000fc31906ba4e1807c30fc20c70d698380fc327d2000fc32ea00fc336a00fc33ea00fc346a00fc34ef68fc21fc217c227c22c000cbb8fcfed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f8419ac8c9707f22d023d05503e1f849d0f846f00702987ff84504d4305e21e05b7ff842f843f84504d4301034413080201200a0900d5ba7a3ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f848d0d431d430f8287003c8cbffc94130c85003cf16cb07ccc97020c8cb0113f400f400cb00c9f9007074c8cb02ca07cbffc9d080093bb54ded44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded171f828f842f843f844f845f849f846f848f84780202c71e0c020148180d020272140e0101f40f01bac882f01bc04b5291c26a46d918139138b992d2de976d6851d0893b0476b85bfbdfc6e6228307f40e6fa130d3ff3001cbff82f03dbe3c57aae062079b4712b8a650cf2abe2d6f986e0ecf1785d2ba835974175d228307f40e6fa130cf1610015e82f0e3868b37bc801236c714f134200a4e3211860f3ad67d22be995321452723c516228307f40e6fa1923031e30dc91101c0d3073001cb0782f0862bd78e42e143bb51660c521752437648ef67967d5055093d11be1117c774b0228307f40e6fa130cf1682f0ade34d680d2df2ad88267395ea1c3b159c4df766416d69d2bbcc8e18e87bd2ba228307f40e6fa130d43001cc1201fe82f093354845030274cd4bf1686abd60ab28ec52e1a792fa6f7ccb9cbd0ddff53d12228307f40e6fa130d43001cc82f0852d0f3fd8bdef2aab5f91714d86638ddc57db97743a350fab3394bc9ebd9518228307f40e6fa130d43001cc82f089445ea08b55421faa49919a5fd272e9a520f701b479d6084847e161ca5b7711581300168307f40e6fa130d43001cc01bbf36fc216465ffc1780de025a948e135236c8c09c89c5cc9696f4bb6b428e8449d823b5c2dfdefe3732c4183fa21e47c21e78b41781edf1e2bd5703103cda3895c532867955f16b7cc3707678bc2e95d41acba0baeac4183fa21fc20f18041501fef844c8cb0782f0e3868b37bc801236c714f134200a4e3211860f3ad67d22be995321452723c516588307f443c8f845cf1682f0862bd78e42e143bb51660c521752437648ef67967d5055093d11be1117c774b0588307f443f846c8cc82f0ade34d680d2df2ad88267395ea1c3b159c4df766416d69d2bbcc8e18e87bd2ba581601bc8307f443f847c8cc82f093354845030274cd4bf1686abd60ab28ec52e1a792fa6f7ccb9cbd0ddff53d12588307f443f848c8cc82f0852d0f3fd8bdef2aab5f91714d86638ddc57db97743a350fab3394bc9ebd9518588307f443f849c8cc17004e82f089445ea08b55421faa49919a5fd272e9a520f701b479d6084847e161ca5b7711588307f4430202761b19017df1998e83a6b90fd201876a26869ff80fc317d2000fc31906ba4e1807c30fc20c70d698380fc327d2000fc32ea00fc336a00fc33ea00fc346a00fc34ef6880c1a01c8d3fffa4001f865f8435230c7058e5331fa405312c7058e416c12d30701f864f84271c8cb00cb3f58cf16c9f866d401f867d401f868d430f869f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54e05bf843c705f2e1afe30d2e0201201d1c00415f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54800694ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded180201cb201f0011f686900699ffd20184020148222100113e910c30003cb8536003f5007434c0c05c6c2497c1383e903e900c7e800c5c75c87e800c7e800c1cea6d003b513434ffc07e18be90007e18c835d270c03e187e106386b4c1c07e193e90007e1975007e19b5007e19f5007e1a35007e1a77b47e1078c0dc14c0f5d270882616c0b4c7f4cfd111378860840a8c6564eea3a116c4b6cf380d48202d272302da82105fcc3d14ba8e85304330db3ce031342382102fcb26a2ba8eb23132f846f007706d82108b771735c8cb1f16cb3f049702c8cbff01cf169b6c21f842c8cbfff843cf16e212cf17128040db3ce0320282102fa30f96ba9ff843c705f2e191d401fb04d430ed54e05b840ff2f0242c03e0f84514c705f2e191f844c000f844c003b1f2e191fa4021f001fa40d20031fa000682084c4b40a121945315a0a1de22d70b01c300209206a19136e220c2fff2e19223f865218e9d6d821005138d91c8cb1f5260cb3ff845cf165008cf1610341771db3c039410266c31e202925f03e30d2c26250040f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54012622f0016d8210d53276dbc8cb1f12cb3f71db3c2c01e6f847d0d403d30721c003f2e19af844c000f844c003b1f2e1a4d3fffa403020d70b01c000923025de05f4043003d022a59320c2009601fa003101a5e830fa0030208209312d005320bcf2e208f848d0d4d430f8287007c8cbffc9225088c85003cf16cb07ccc97020c8cb0113f400f400cb00c92803fef9007074c8cb02ca07cbffc9d01ac705965392bef2e208df5192a1f846f007228f5c50675f053333343434028107d0a8812710a904f8456d70c8cb1f8d04935648149959995c9c985b0814185e5bdd5d20cf16102372db3c82084c4b4070fb02706d732282102a319593c8cb1fcb3fcb0715cbff5003cf1613810082db3ce02c2c2901fe35f8287022c8cbffc91038c85003cf16cb07ccc97020c8cb0113f400f400cb00c920f9007074c8cb02ca07cbffc9d0f849d0c803d013cf16f849f8486d6dc870fa0270fa02500afa02c9c8cc19f40018f400c904d3ff3029c8cbfff843cf16c90fc8cc1fccc9c8cc1ecbff2ccf16f828cf1619cb0712cc14cc1acc10234a502a03b472db3ca405c8ca0015cb3f5005cf16c9f866f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed5422c2008e926d708210d53276dbc8cb1fcb3f102472db3c926c21e22282103b9aca00bc925f03e30d2c2c2b02e202821005f5e100a1208106a4a8812710a90466a1f8436d8210247a9ebac8cb1f1023102472db3c593121c2008ec08ebc78f4966fa531208eaf01d430d0fa40d30f305240a8812710a9046d70c8cb1f8d04535648149Line truncated
+    code = 'b5ee9c7241023201000bbc000114ff00f4a413f4bcf2c80b01020120040201f6f2ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f849d0d3ffd31f038308d71820f9015882f053b50ddfe5a9533f2e76ac054411db94432a1f7b7ae17fc64cf7aec5df8705d5f910f2e212fa40f82812c705f2e213d31f5213bc0301d4f2e21401d33f01f823bcf2e215f80082084c4b4070fb02d4308e3cd0d31f218210e3e30001ba8e1731f404216e91319301fb04e2f404216e91319301ed54e28e10018210e3e30002ba96d307d402fb00dee2f40430206ee63002d4d43002c8cbff13cb1f12ccccc9f869280201480e050201200b060201200a0702016a0908004bae43bac17829da86eff2d4a99f973b5602a208edca21950fbdbd70bfe3267bd762efc382eac0007dac1ff6a26869ff80fc317d2000fc31906ba4e1807c30fc20c70d698380fc327d2000fc32ea00fc336a00fc33ea00fc346a00fc34ef68fc21fc217c227c22c000cbb8fcfed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f8419ac8c9707f22d023d05503e1f849d0f846f00702987ff84504d4305e21e05b7ff842f843f84504d4301034413080201200d0c00d5ba7a3ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f848d0d431d430f8287003c8cbffc94130c85003cf16cb07ccc97020c8cb0113f400f400cb00c9f9007074c8cb02ca07cbffc9d080093bb54ded44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded171f828f842f843f844f845f849f846f848f84780202c7210f0201481b1002027217110101f41201bac882f01bc04b5291c26a46d918139138b992d2de976d6851d0893b0476b85bfbdfc6e6228307f40e6fa130d3ff3001cbff82f03dbe3c57aae062079b4712b8a650cf2abe2d6f986e0ecf1785d2ba835974175d228307f40e6fa130cf1613015e82f0e3868b37bc801236c714f134200a4e3211860f3ad67d22be995321452723c516228307f40e6fa1923031e30dc91401c0d3073001cb0782f0862bd78e42e143bb51660c521752437648ef67967d5055093d11be1117c774b0228307f40e6fa130cf1682f0ade34d680d2df2ad88267395ea1c3b159c4df766416d69d2bbcc8e18e87bd2ba228307f40e6fa130d43001cc1501fe82f093354845030274cd4bf1686abd60ab28ec52e1a792fa6f7ccb9cbd0ddff53d12228307f40e6fa130d43001cc82f0852d0f3fd8bdef2aab5f91714d86638ddc57db97743a350fab3394bc9ebd9518228307f40e6fa130d43001cc82f089445ea08b55421faa49919a5fd272e9a520f701b479d6084847e161ca5b7711581600168307f40e6fa130d43001cc01bbf36fc216465ffc1780de025a948e135236c8c09c89c5cc9696f4bb6b428e8449d823b5c2dfdefe3732c4183fa21e47c21e78b41781edf1e2bd5703103cda3895c532867955f16b7cc3707678bc2e95d41acba0baeac4183fa21fc20f18041801fef844c8cb0782f0e3868b37bc801236c714f134200a4e3211860f3ad67d22be995321452723c516588307f443c8f845cf1682f0862bd78e42e143bb51660c521752437648ef67967d5055093d11be1117c774b0588307f443f846c8cc82f0ade34d680d2df2ad88267395ea1c3b159c4df766416d69d2bbcc8e18e87bd2ba581901bc8307f443f847c8cc82f093354845030274cd4bf1686abd60ab28ec52e1a792fa6f7ccb9cbd0ddff53d12588307f443f848c8cc82f0852d0f3fd8bdef2aab5f91714d86638ddc57db97743a350fab3394bc9ebd9518588307f443f849c8cc1a004e82f089445ea08b55421faa49919a5fd272e9a520f701b479d6084847e161ca5b7711588307f4430202761e1c017df1998e83a6b90fd201876a26869ff80fc317d2000fc31906ba4e1807c30fc20c70d698380fc327d2000fc32ea00fc336a00fc33ea00fc346a00fc34ef6880c1d01c8d3fffa4001f865f8435230c7058e5331fa405312c7058e416c12d30701f864f84271c8cb00cb3f58cf16c9f866d401f867d401f868d430f869f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54e05bf843c705f2e1afe30d31020120201f00415f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54800694ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded180201cb23220011f686900699ffd20184020148252400113e910c30003cb8536003f5007434c0c05c6c2497c1383e903e900c7e800c5c75c87e800c7e800c1cea6d003b513434ffc07e18be90007e18c835d270c03e187e106386b4c1c07e193e90007e1975007e19b5007e19f5007e1a35007e1a77b47e1078c0dc14c0f5d270882616c0b4c7f4cfd111378860840a8c6564eea3a116c4b6cf380d4820302a2602da82105fcc3d14ba8e85304330db3ce031342382102fcb26a2ba8eb23132f846f007706d82108b771735c8cb1f16cb3f049702c8cbff01cf169b6c21f842c8cbfff843cf16e212cf17128040db3ce0320282102fa30f96ba9ff843c705f2e191d401fb04d430ed54e05b840ff2f0272f03e0f84514c705f2e191f844c000f844c003b1f2e191fa4021f001fa40d20031fa000682084c4b40a121945315a0a1de22d70b01c300209206a19136e220c2fff2e19223f865218e9d6d821005138d91c8cb1f5260cb3ff845cf165008cf1610341771db3c039410266c31e202925f03e30d2f29280040f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54012622f0016d8210d53276dbc8cb1f12cb3f71db3c2f01e6f847d0d403d30721c003f2e19af844c000f844c003b1f2e1a4d3fffa403020d70b01c000923025de05f4043003d022a59320c2009601fa003101a5e830fa0030208209312d005320bcf2e208f848d0d4d430f8287007c8cbffc9225088c85003cf16cb07ccc97020c8cb0113f400f400cb00c92b03fef9007074c8cb02ca07cbffc9d01ac705965392bef2e208df5192a1f846f007228f5c50675f053333343434028107d0a8812710a904f8456d70c8cb1f8d04935648149959995c9c985b0814185e5bdd5d20cf16102372db3c82084c4b4070fb02706d732282102a319593c8cb1fcb3fcb0715cbff5003cf1613810082db3ce02f2f2c01fe35f8287022c8cbffc91038c85003cf16cb07ccc97020c8cb0113f400f400cb00c920f9007074c8cb02ca07cbffc9d0f849d0c803d013cf16f849f8486d6dc870fa0270fa02500Line truncated
     codebase_version = 5
 
     def __init__(self, **kwargs):

app/core/_blockchain/ton/contracts/platform.py

@@ -3,7 +3,7 @@ from tonsdk.contract import Contract
 
 
 class Platform(Contract):
-    code = 'b5ee9c7241021601000310000114ff00f4a413f4bcf2c80b010201620d0202012006030201200504004bbac877282f037625a5e1bf4a9bb4e8e57adf780d02781ee2c2b80129dc6a90f23b01657f9d980057b905bed44d0fa4001f861d3ff01f862d401f863f843d0d431d430f864d401f865d1f845d0f84201d430f84180201200a07020120090800a1b4f47da89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba3f089f050e0079197ff92826190a0079e2d960f9992e04191960227e801e801960193f200e0e9919605940f97ff93a10000fb5daeeb00c9f05100201200c0b0059b6a9bda89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba2e1f051f085f087f089f08b00051b56ba63da89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba2e391960f999300202c70f0e0007a0dd7c120201cf111000113e910c30003cb8536002f30cf434c0c05c6c2497c0f83e90087c007e900c7e800c5c75c87e800c7e800c1cea6d0008f5d27048245c2540f4c7d411388830002497c1783b51343e90007e1874ffc07e18b5007e18fe10f4350c750c3e1935007e1974482084091ea7aeaea497c178082084152474232ea3a14c104c36cf380c4cbe1071c160131201dcf2e19120820833cc77ba9730d4d30730fb00e0208210b99cd03bba9701fa4001f86101de208210d81c632fba9601d401f86501de208210b5de5f9eba8e8b30fa40fa00306d6d71db3ce082102fa30f96ba98d401fb04d430ed54e030f845f843f842c8f841cf16cbffccccc9ed541502f082084c4b4001a013bef2e20801d3ffd4d430f844f82870f842c8cbffc9c85003cf16cb07ccc97020c8cb0113f400f400cb00c920f9007074c8cb02ca07cbffc9d0f843d070c804d014cf16f843f842c8cbfff828cf16c903d430c8cc13ccc9c8cc17cbff5007cf1614cc15cccc43308040db3cf842a4f86215140024f845f843f842c8f841cf16cbffccccc9ed540078708010c8cb055006cf165004fa0214cb68216e947032cb019bc858cf17c97158cb00f400e2226e95327058cb0099c85003cf17c958f400e2c901fb004e32cb65'
+    code = 'b5ee9c724102160100032e000114ff00f4a413f4bcf2c80b010201620d0202012006030201200504004bbac877582f053b50ddfe5a9533f2e76ac054411db94432a1f7b7ae17fc64cf7aec5df8705d580057b905bed44d0fa4001f861d3ff01f862d401f863f843d0d431d430f864d401f865d1f845d0f84201d430f84180201200a07020120090800a1b4f47da89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba3f089f050e0079197ff92826190a0079e2d960f9992e04191960227e801e801960193f200e0e9919605940f97ff93a10000fb5daeeb00c9f05100201200c0b0059b6a9bda89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba2e1f051f085f087f089f08b00051b56ba63da89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba2e391960f999300202c70f0e0007a0dd7c120201cf111000113e910c30003cb8536002f30cf434c0c05c6c2497c0f83e90087c007e900c7e800c5c75c87e800c7e800c1cea6d0008f5d27048245c2540f4c7d411388830002497c1783b51343e90007e1874ffc07e18b5007e18fe10f4350c750c3e1935007e1974482084091ea7aeaea497c178082084152474232ea3a14c104c36cf380c4cbe1071c160131201faf2e19120820833cc77ba9730d4d30730fb00e0208210b99cd03bba9701fa4001f86101de208210d81c632fba9601d401f86501de208210b5de5f9eba8e8b30fa40fa00306d6d71db3ce082102fa30f96ba8e16f404216e91319301fb04e2f40430206e913092ed54e2e030f845f843f842c8f841cf16cbffccccc9ed541501f682084c4b4001a013bef2e20801d3fffa4021d70b01c0009231029133e202d4d430f844f82870f842c8cbffc9c85003cf16cb07ccc97020c8cb0113f400f400cb00c920f9007074c8cb02ca07cbffc9d0f843d070c804d014cf16f843f842c8cbfff828cf16c903d430c8cc13ccc9c8cc17cbff5007cf1614cc15cc14013ccc43308040db3cf842a4f862f845f843f842c8f841cf16cbffccccc9ed54150078708010c8cb055006cf165004fa0214cb68216e947032cb009bc858cf17c97158cb00f400e2226e95327058cb0099c85003cf17c958f400e2c901fb003366cbbe'
     codebase_version = 5
 
     def __init__(self, **kwargs):

app/core/_blockchain/ton/platform.py

@@ -12,11 +12,34 @@ kwargs = {}
 if int(os.getenv('INIT_DEPLOY_PLATFORM_CONTRACT', 0)) == 0:
     kwargs['address'] = Address(MY_PLATFORM_CONTRACT)
 
-platform = Platform(
+def platform_with_salt(s: int = 0):
-    admin_address=Address('UQAjz4Kdqoo4_Obg-UrUmuhoUB2W00vngZoX0MnAAnetZuAk'),
+    return Platform(
+        admin_address=Address('UQD3XALhbETNo7ItrdPNFzMJtRHC5u6dIb39DCYa40jnWZdg'),
         blank_code=Cell.one_from_boc(Blank.code),
         cop_code=Cell.one_from_boc(COP_NFT.code),
-
+        collection_content_uri=f'{PROJECT_HOST}/api/platform-metadata.json' + f"?s={s}",
-    collection_content_uri=f'{PROJECT_HOST}/api/platform-metadata.json',
         **kwargs
     )
+
+platform = platform_with_salt()
+
+if int(os.getenv('INIT_DEPLOY_PLATFORM_CONTRACT', 0)) == 1:
+    def is_nice_address(address: Address):
+        bounceable_addr = address.to_string(True, True, True)
+        non_bounceable_addr = address.to_string(True, True, False)
+
+        if '-' in bounceable_addr or '-' in non_bounceable_addr:
+            return False
+
+        if '_' in bounceable_addr or '_' in non_bounceable_addr:
+            return False
+
+        if bounceable_addr[-1] != 'A':
+            return False
+        
+        return True
+
+    salt_value = 0
+    while not is_nice_address(platform.address):
+        platform = platform_with_salt(salt_value)
+        salt_value += 1

app/core/_config.py

@@ -7,7 +7,12 @@ load_dotenv(dotenv_path='.env')
 
 PROJECT_HOST = os.getenv('PROJECT_HOST', 'http://127.0.0.1:8080')
 SANIC_PORT = int(os.getenv('SANIC_PORT', '8080'))
+# Path inside the running backend container where content files are visible
 UPLOADS_DIR = os.getenv('UPLOADS_DIR', '/app/data')
+# Host path where the same content directory is mounted (used for docker -v from within container)
+BACKEND_DATA_DIR_HOST = os.getenv('BACKEND_DATA_DIR_HOST', '/Storage/storedContent')
+# Host path for converter logs (used for docker -v). Optional.
+BACKEND_LOGS_DIR_HOST = os.getenv('BACKEND_LOGS_DIR_HOST', '/Storage/logs/converter')
 if not os.path.exists(UPLOADS_DIR):
     os.makedirs(UPLOADS_DIR)
 
@@ -16,12 +21,31 @@ assert TELEGRAM_API_KEY, "Telegram API_KEY required"
 CLIENT_TELEGRAM_API_KEY = os.environ.get('CLIENT_TELEGRAM_API_KEY')
 assert CLIENT_TELEGRAM_API_KEY, "Client Telegram API_KEY required"
 import httpx
-TELEGRAM_BOT_USERNAME = httpx.get(f"https://api.telegram.org/bot{TELEGRAM_API_KEY}/getMe").json()['result']['username']
-CLIENT_TELEGRAM_BOT_USERNAME = httpx.get(f"https://api.telegram.org/bot{CLIENT_TELEGRAM_API_KEY}/getMe").json()['result']['username']
 
 
-MYSQL_URI = os.environ['MYSQL_URI']
+def _resolve_bot_username(token: str, label: str) -> str:
-MYSQL_DATABASE = os.environ['MYSQL_DATABASE']
+    try:
+        resp = httpx.get(f"https://api.telegram.org/bot{token}/getMe", timeout=10.0)
+        resp.raise_for_status()
+        payload = resp.json()
+    except Exception as exc:
+        raise RuntimeError(f"{label} Telegram token validation failed: {exc}") from exc
+
+    if not payload.get('ok'):
+        detail = payload.get('description') or 'unknown Telegram API error'
+        raise RuntimeError(f"{label} Telegram token validation failed: {detail}")
+
+    username = (payload.get('result') or {}).get('username')
+    if not username:
+        raise RuntimeError(f"{label} Telegram token validation failed: username missing in Telegram response")
+    return username
+
+
+TELEGRAM_BOT_USERNAME = _resolve_bot_username(TELEGRAM_API_KEY, 'Uploader bot')
+CLIENT_TELEGRAM_BOT_USERNAME = _resolve_bot_username(CLIENT_TELEGRAM_API_KEY, 'Client bot')
+
+# Unified database URL (PostgreSQL)
+DATABASE_URL = os.environ['DATABASE_URL']
 
 LOG_LEVEL = os.getenv('LOG_LEVEL', 'DEBUG')
 LOG_DIR = os.getenv('LOG_DIR', 'logs')
@@ -32,7 +56,7 @@ _now_str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
 LOG_FILEPATH = f"{LOG_DIR}/{_now_str}.log"
 
 WEB_APP_URLS = {
-    'uploadContent': f"https://web2-client.vercel.app/uploadContent"
+    'uploadContent': f"https://my-public-node-8.projscale.dev/uploadContent"
 }
 
 ALLOWED_CONTENT_TYPES = [
@@ -48,5 +72,5 @@ TONCENTER_HOST = os.getenv('TONCENTER_HOST', 'https://toncenter.com/api/v2/')
 TONCENTER_API_KEY = os.getenv('TONCENTER_API_KEY')
 TONCENTER_V3_HOST = os.getenv('TONCENTER_V3_HOST', 'https://toncenter.com/api/v3/')
 
-MY_PLATFORM_CONTRACT = 'EQDmWp6hbJlYUrXZKb9N88sOrTit630ZuRijfYdXEHLtheMY'
+MY_PLATFORM_CONTRACT = 'EQBVjuNuaIK87v9nm7mghgJ41ikqfx3GNBFz05GfmNbRQ9EA'
 MY_FUND_ADDRESS = 'UQDarChHFMOI2On9IdHJNeEKttqepgo0AY4bG1trw8OAAwMY'

app/core/_crypto/content.py

@@ -36,9 +36,10 @@ async def create_new_encryption_key(db_session, user_id: int = None) -> KnownKey
         meta={"I_user_id": user_id} if user_id else None,
         created=datetime.now()
     )
+    from sqlalchemy import select
     db_session.add(new_key)
-    db_session.commit()
+    await db_session.commit()
-    new_key = db_session.query(KnownKey).filter(KnownKey.seed_hash == new_seed_hash).first()
+    new_key = (await db_session.execute(select(KnownKey).where(KnownKey.seed_hash == new_seed_hash))).scalars().first()
     assert new_key, "Key not created"
     return new_key
 
@@ -46,42 +47,51 @@ async def create_new_encryption_key(db_session, user_id: int = None) -> KnownKey
 async def create_encrypted_content(
     db_session, decrypted_content: StoredContent,
 ) -> StoredContent:
-    encrypted_content = db_session.query(StoredContent).filter(
+    from sqlalchemy import select
-        StoredContent.id == decrypted_content.decrypted_content_id
+    # Try to find an already created encrypted counterpart for this decrypted content
-    ).first()
+    encrypted_content = (
+        await db_session.execute(
+            select(StoredContent).where(StoredContent.decrypted_content_id == decrypted_content.id)
+        )
+    ).scalars().first()
     if encrypted_content:
         make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): already exist (found by decrypted content)", level="debug")
         return encrypted_content
 
     encrypted_content = None
-    if decrypted_content.key is None:
+    # Avoid accessing relationship attributes in async context to prevent MissingGreenlet
+    if not decrypted_content.key_id:
         key = await create_new_encryption_key(db_session, user_id=decrypted_content.user_id)
         decrypted_content.key_id = key.id
-        db_session.commit()
+        await db_session.commit()
-        decrypted_content = db_session.query(StoredContent).filter(
-            StoredContent.id == decrypted_content.id
-        ).first()
         assert decrypted_content.key_id, "Key not assigned"
 
+    # Explicitly load the key to avoid lazy-loading via relationship in async mode
+    key = (
+        await db_session.execute(select(KnownKey).where(KnownKey.id == decrypted_content.key_id))
+    ).scalars().first()
+    # If the referenced key is missing or malformed, create a fresh one
+    if not key or not key.seed:
+        key = await create_new_encryption_key(db_session, user_id=decrypted_content.user_id)
+        decrypted_content.key_id = key.id
+        await db_session.commit()
+
     decrypted_path = os.path.join(UPLOADS_DIR, decrypted_content.hash)
     decrypted_bin = b58decode(decrypted_content.hash)
 
-    key = decrypted_content.key
     cipher = AESCipher(key.seed_bin)
 
     encrypted_bin = cipher.encrypt(decrypted_bin)
     encrypted_hash_bin = sha256(encrypted_bin).digest()
     encrypted_hash = b58encode(encrypted_hash_bin).decode()
-    encrypted_content = db_session.query(StoredContent).filter(
+    encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash))).scalars().first()
-        StoredContent.hash == encrypted_hash
-    ).first()
     if encrypted_content:
         make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): already exist (found by encrypted_hash)", level="debug")
         return encrypted_content
 
     encrypted_content = None
 
-    encrypted_meta = decrypted_content.meta
+    encrypted_meta = dict(decrypted_content.meta or {})
     encrypted_meta["encrypt_algo"] = "AES256"
 
     encrypted_content = StoredContent(
@@ -99,19 +109,15 @@ async def create_encrypted_content(
         created=datetime.now(),
     )
     db_session.add(encrypted_content)
-    db_session.commit()
+    await db_session.commit()
 
     encrypted_path = os.path.join(UPLOADS_DIR, encrypted_hash)
     async with aiofiles.open(encrypted_path, mode='wb') as file:
         await file.write(encrypted_bin)
 
-    encrypted_content = db_session.query(StoredContent).filter(
+    encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash))).scalars().first()
-        StoredContent.hash == encrypted_hash
-    ).first()
     assert encrypted_content, "Content not created"
     make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): created new content/bin", level="debug")
     return encrypted_content
 
 
-
-

app/core/_secrets.py

@@ -1,44 +1,115 @@
 from os import getenv, urandom
+import os
+import time
+import json
 
 from nacl.bindings import crypto_sign_seed_keypair
 from tonsdk.utils import Address
 
 from app.core._blockchain.ton.wallet_v3cr3 import WalletV3CR3
-from app.core.models._config import ServiceConfig
-from app.core.storage import db_session
 from app.core.logger import make_log
-import os
+from sqlalchemy import create_engine, inspect
+from sqlalchemy.orm import Session
+from typing import Optional
+from app.core.models._config import ServiceConfigValue
 
 
-def load_hot_pair():
+def _load_seed_from_env_or_generate() -> bytes:
-    with db_session() as session:
+    seed_hex = os.getenv("TON_INIT_HOT_SEED")
-        service_config = ServiceConfig(session)
+    if seed_hex:
-        hot_seed = service_config.get('private_key')
+        make_log("HotWallet", "Loaded seed from env")
-        if hot_seed is None:
+        return bytes.fromhex(seed_hex)
-            make_log("HotWallet", "No seed found, generating new one", level='info')
+    make_log("HotWallet", "No seed provided; generating ephemeral seed", level='info')
-            hot_seed = os.getenv("TON_INIT_HOT_SEED")
+    return urandom(32)
-            if not hot_seed:
+
-                hot_seed = urandom(32)
+
-                make_log("HotWallet", f"Generated random seed")
+def _init_seed_via_db() -> bytes:
+    """Store and read hot seed from PostgreSQL service_config (key='private_key').
+    Primary node writes it once; workers wait until it appears.
+    """
+    from app.core._config import DATABASE_URL
+
+    engine = create_engine(DATABASE_URL, pool_pre_ping=True)
+    role = os.getenv("NODE_ROLE", "worker").lower()
+
+    def db_ready(conn) -> bool:
+        try:
+            inspector = inspect(conn)
+            return inspector.has_table('service_config')
+        except Exception:
+            return False
+
+    # Wait for table to exist
+    start = time.time()
+    # Wait for table existence, reconnecting to avoid stale transactions
+    while True:
+        with engine.connect() as conn:
+            if db_ready(conn):
+                break
+        time.sleep(0.5)
+        if time.time() - start > 120:
+            raise TimeoutError("service_config table not available")
+
+    def read_seed() -> Optional[bytes]:
+        # Use a fresh connection/session per read to avoid snapshot staleness
+        try:
+            with engine.connect() as rconn:
+                with Session(bind=rconn) as s:
+                    row = s.query(ServiceConfigValue).filter(ServiceConfigValue.key == 'private_key').first()
+                    if not row:
+                        return None
+                    packed = row.packed_value or {}
+                    if isinstance(packed, str):
+                        packed = json.loads(packed)
+                    seed_hex = packed.get('value')
+                    return bytes.fromhex(seed_hex) if seed_hex else None
+        except Exception:
+            return None
+
+    seed = read_seed()
+    if seed:
+        return seed
+
+    if role == "primary":
+        seed = _load_seed_from_env_or_generate()
+        # Try insert; if another primary raced, ignore
+        try:
+            with engine.connect() as wconn:
+                with Session(bind=wconn) as s:
+                    s.add(ServiceConfigValue(key='private_key', packed_value={"value": seed.hex()}))
+                    s.commit()
+            make_log("HotWallet", "Seed saved in service_config by primary", level='info')
+            return seed
+        except Exception:
+            # Read again in case of race
+            seed2 = read_seed()
+            if seed2:
+                return seed2
+            raise
     else:
-                hot_seed = bytes.fromhex(hot_seed)
+        make_log("HotWallet", "Worker waiting for seed in service_config...", level='info')
-                make_log("HotWallet", f"Loaded seed from env")
+        while True:
-
+            seed = read_seed()
-            service_config.set('private_key', hot_seed.hex())
+            if seed:
-            return load_hot_pair()
+                return seed
-
+            time.sleep(0.5)
-        hot_seed = bytes.fromhex(hot_seed)
-        public_key, private_key = crypto_sign_seed_keypair(hot_seed)
-        return hot_seed, public_key, private_key
 
 
 _extra_ton_wallet_options = {}
 if getenv('TON_CUSTOM_WALLET_ADDRESS'):
     _extra_ton_wallet_options['address'] = Address(getenv('TON_CUSTOM_WALLET_ADDRESS'))
 
-hot_seed, hot_pubkey, hot_privkey = load_hot_pair()
+
-service_wallet = WalletV3CR3(
+def _init_wallet():
-    private_key=hot_privkey,
+    # Primary writes to DB; workers wait and read from DB
-    public_key=hot_pubkey,
+    hot_seed_bytes = _init_seed_via_db()
+    pub, priv = crypto_sign_seed_keypair(hot_seed_bytes)
+    wallet = WalletV3CR3(
+        private_key=priv,
+        public_key=pub,
         **_extra_ton_wallet_options
     )
+    return hot_seed_bytes, pub, priv, wallet
+
+
+hot_seed, hot_pubkey, hot_privkey, service_wallet = _init_wallet()

app/core/_utils/create_maria_tables.py

@@ -1,10 +1,12 @@
+from sqlalchemy.ext.asyncio import AsyncEngine
 from app.core.models import BlockchainTask
 from app.core.models.base import AlchemyBase
 
 
-def create_maria_tables(engine):
+async def create_db_tables(engine: AsyncEngine):
-    """Create all tables in the database."""
+    """Create all tables in the database (PostgreSQL, async)."""
+    # ensure model import side-effects initialize mappers
     BlockchainTask()
-    AlchemyBase.metadata.create_all(engine)
+    async with engine.begin() as conn:
-    
+        await conn.run_sync(AlchemyBase.metadata.create_all)
     

app/core/_utils/resolve_content.py

@@ -2,8 +2,9 @@ from app.core.content.content_id import ContentId
 
 
 def resolve_content(content_id) -> ContentId:  # -> [content, error]
+    if isinstance(content_id, ContentId):
+        return content_id, None
     try:
         return ContentId.deserialize(content_id), None
     except BaseException as e:
         return None, f"{e}"
-

app/core/_utils/send_status.py

@@ -5,7 +5,6 @@ from httpx import AsyncClient
 
 from app.core._config import PROJECT_HOST
 from app.core._crypto.signer import Signer
-from app.core._secrets import hot_seed
 from app.core.logger import make_log
 
 
@@ -17,6 +16,8 @@ async def send_status(service: str, status: str):
             'status': status,
         }
         message_bytes = dumps(message).encode()
+        # Lazy import to avoid triggering _secrets before DB is ready
+        from app.core._secrets import hot_seed
         signer = Signer(hot_seed)
         message_signature = signer.sign(message_bytes)
         async with AsyncClient() as client:

app/core/_utils/share_links.py

@@ -0,0 +1,20 @@
+from typing import Optional
+from urllib.parse import urlencode
+
+STARTAPP_LIMIT = 64
+
+
+def build_content_links(content_token: str, ref_id: Optional[str], *, project_host: str, bot_username: str):
+    """Return tuple of (startapp_payload, telegram_url, web_url)."""
+    payload = (content_token or '').strip()
+    if len(payload) > STARTAPP_LIMIT:
+        payload = payload[:STARTAPP_LIMIT]
+
+    telegram_url = f"https://t.me/{bot_username}/content?startapp={payload}"
+
+    query = [('content', content_token)]
+    if ref_id:
+        query.append(('ref', ref_id))
+    web_url = f"{project_host}/viewContent?{urlencode(query)}"
+
+    return payload, telegram_url, web_url

app/core/auth_v1.py

@@ -56,9 +56,10 @@ class AuthenticationMixin:
             },
             created=datetime.fromtimestamp(init_ts)
         )
+        from sqlalchemy import select
         db_session.add(new_key)
-        db_session.commit()
+        await db_session.commit()
-        new_key = db_session.query(KnownKey).filter(KnownKey.seed_hash == new_key.seed_hash).first()
+        new_key = (await db_session.execute(select(KnownKey).where(KnownKey.seed_hash == new_key.seed_hash))).scalars().first()
         assert new_key, "Key not created"
         make_log("auth", f"[new-K] User {user_id} created new {token_type} key {new_key.id}")
         return {

app/core/background/convert_service.py

@@ -6,7 +6,7 @@ import json
 import shutil
 import magic  # python-magic for MIME detection
 from base58 import b58decode, b58encode
-from sqlalchemy import and_, or_
+from sqlalchemy import and_, or_, select
 from app.core.models.node_storage import StoredContent
 from app.core.models._telegram import Wrapped_CBotChat
 from app.core._utils.send_status import send_status
@@ -14,14 +14,14 @@ from app.core.logger import make_log
 from app.core.models.user import User
 from app.core.models import WalletConnection
 from app.core.storage import db_session
-from app.core._config import UPLOADS_DIR
+from app.core._config import UPLOADS_DIR, BACKEND_DATA_DIR_HOST, BACKEND_LOGS_DIR_HOST
 from app.core.content.content_id import ContentId
 
 
 async def convert_loop(memory):
-    with db_session() as session:
+    async with db_session() as session:
         # Query for unprocessed encrypted content
-        unprocessed_encrypted_content = session.query(StoredContent).filter(
+        unprocessed_encrypted_content = (await session.execute(select(StoredContent).where(
             and_(
                 StoredContent.type == "onchain/content",
                 or_(
@@ -29,27 +29,30 @@ async def convert_loop(memory):
                     StoredContent.ipfs_cid == None,
                 )
             )
-        ).first()
+        ))).scalars().first()
         if not unprocessed_encrypted_content:
             make_log("ConvertProcess", "No content to convert", level="debug")
             return
 
         # Достаем расшифрованный файл
-        decrypted_content = session.query(StoredContent).filter(
+        decrypted_content = (await session.execute(select(StoredContent).where(
             StoredContent.id == unprocessed_encrypted_content.decrypted_content_id
-        ).first()
+        ))).scalars().first()
         if not decrypted_content:
             make_log("ConvertProcess", "Decrypted content not found", level="error")
             return
 
         # Определяем путь и расширение входного файла
-        input_file_path = f"/Storage/storedContent/{decrypted_content.hash}"
+        # Путь внутри текущего контейнера (доступен Python процессу)
+        input_file_container = os.path.join(UPLOADS_DIR, decrypted_content.hash)
+        # Хостовый путь (нужен для docker -v маппинга при запуске конвертера)
+        input_file_host = os.path.join(BACKEND_DATA_DIR_HOST, decrypted_content.hash)
         input_ext = (unprocessed_encrypted_content.filename.split('.')[-1]
                      if '.' in unprocessed_encrypted_content.filename else "mp4")
 
         # ==== Новая логика: определение MIME-тип через python-magic ====
         try:
-            mime_type = magic.from_file(input_file_path.replace("/Storage/storedContent", "/app/data"), mime=True)
+            mime_type = magic.from_file(input_file_container, mime=True)
         except Exception as e:
             make_log("ConvertProcess", f"magic probe failed: {e}", level="warning")
             mime_type = ""
@@ -78,7 +81,7 @@ async def convert_loop(memory):
                     option_name: decrypted_content.hash for option_name in ['high', 'low', 'low_preview']
                 }
             }
-            session.commit()
+            await session.commit()
             return
 
         # ==== Конвертация для видео или аудио: оригинальная логика ====
@@ -100,7 +103,8 @@ async def convert_loop(memory):
             REQUIRED_CONVERT_OPTIONS = ['high', 'low']  # no preview for audio
 
         converted_content = {}
-        logs_dir = "/Storage/logs/converter"
+        # Директория логов на хосте для docker-контейнера конвертера
+        logs_dir_host = BACKEND_LOGS_DIR_HOST
 
         for option in REQUIRED_CONVERT_OPTIONS:
             # Set quality parameter and trim option (only for preview)
@@ -113,22 +117,26 @@ async def convert_loop(memory):
 
             # Generate a unique output directory for docker container
             output_uuid = str(uuid.uuid4())
-            output_dir = f"/Storage/storedContent/converter-output/{output_uuid}"
+            # Директория вывода в текущем контейнере (та же что и в UPLOADS_DIR, смонтирована с хоста)
+            output_dir_container = os.path.join(UPLOADS_DIR, "converter-output", output_uuid)
+            os.makedirs(output_dir_container, exist_ok=True)
+            # Соответствующая директория на хосте — нужна для docker -v
+            output_dir_host = os.path.join(BACKEND_DATA_DIR_HOST, "converter-output", output_uuid)
 
             # Build the docker command
             cmd = [
                 "docker", "run", "--rm",
-                "-v", f"{input_file_path}:/app/input",
+                # Важно: источники - это ХОСТОВЫЕ пути, так как docker демону они нужны на хосте
-                "-v", f"{output_dir}:/app/output",
+                "-v", f"{input_file_host}:/app/input:ro",
-                "-v", f"{logs_dir}:/app/logs",
+                "-v", f"{output_dir_host}:/app/output",
+                "-v", f"{logs_dir_host}:/app/logs",
                 "media_converter",
                 "--ext", input_ext,
                 "--quality", quality
             ]
             if trim_value:
                 cmd.extend(["--trim", trim_value])
-            if content_kind == "audio":
+            # converter auto-detects audio/video, no explicit flag required
-                cmd.append("--audio-only")  # audio-only flag
 
             process = await asyncio.create_subprocess_exec(
                 *cmd,
@@ -142,7 +150,7 @@ async def convert_loop(memory):
 
             # List files in output dir
             try:
-                files = os.listdir(output_dir.replace("/Storage/storedContent", "/app/data"))
+                files = os.listdir(output_dir_container)
             except Exception as e:
                 make_log("ConvertProcess", f"Error reading output directory {output_dir}: {e}", level="error")
                 return
@@ -152,10 +160,7 @@ async def convert_loop(memory):
                 make_log("ConvertProcess", f"Expected one media file, found {len(media_files)} for option {option}", level="error")
                 return
 
-            output_file = os.path.join(
+            output_file = os.path.join(output_dir_container, media_files[0])
-                output_dir.replace("/Storage/storedContent", "/app/data"),
-                media_files[0]
-            )
 
             # Compute SHA256 hash of the output file
             hash_process = await asyncio.create_subprocess_exec(
@@ -171,9 +176,7 @@ async def convert_loop(memory):
             file_hash = b58encode(bytes.fromhex(file_hash)).decode()
 
             # Save new StoredContent if not exists
-            if not session.query(StoredContent).filter(
+            if not (await session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first():
-                StoredContent.hash == file_hash
-            ).first():
                 new_content = StoredContent(
                     type="local/content_bin",
                     hash=file_hash,
@@ -183,7 +186,7 @@ async def convert_loop(memory):
                     created=datetime.now(),
                 )
                 session.add(new_content)
-                session.commit()
+                await session.commit()
 
             save_path = os.path.join(UPLOADS_DIR, file_hash)
             try:
@@ -200,10 +203,7 @@ async def convert_loop(memory):
             converted_content[option] = file_hash
 
             # Process output.json for ffprobe_meta
-            output_json_path = os.path.join(
+            output_json_path = os.path.join(output_dir_container, "output.json")
-                output_dir.replace("/Storage/storedContent", "/app/data"),
-                "output.json"
-            )
             if os.path.exists(output_json_path) and unprocessed_encrypted_content.meta.get('ffprobe_meta') is None:
                 try:
                     with open(output_json_path, "r") as f:
@@ -217,7 +217,7 @@ async def convert_loop(memory):
 
             # Cleanup output directory
             try:
-                shutil.rmtree(output_dir.replace("/Storage/storedContent", "/app/data"))
+                shutil.rmtree(output_dir_container)
             except Exception as e:
                 make_log("ConvertProcess", f"Error removing output dir {output_dir}: {e}", level="warning")
 
@@ -233,13 +233,13 @@ async def convert_loop(memory):
             **unprocessed_encrypted_content.meta,
             'converted_content': converted_content
         }
-        session.commit()
+        await session.commit()
 
         # Notify user if needed
         if not unprocessed_encrypted_content.meta.get('upload_notify_msg_id'):
-            wallet_owner_connection = session.query(WalletConnection).filter(
+            wallet_owner_connection = (await session.execute(select(WalletConnection).where(
                 WalletConnection.wallet_address == unprocessed_encrypted_content.owner_address
-            ).order_by(WalletConnection.id.desc()).first()
+            ).order_by(WalletConnection.id.desc()))).scalars().first()
             if wallet_owner_connection:
                 wallet_owner_user = wallet_owner_connection.user
                 bot = Wrapped_CBotChat(
@@ -249,7 +249,7 @@ async def convert_loop(memory):
                     db_session=session
                 )
                 unprocessed_encrypted_content.meta['upload_notify_msg_id'] = await bot.send_content(session, unprocessed_encrypted_content)
-                session.commit()
+                await session.commit()
 
 
 async def main_fn(memory):
@@ -257,11 +257,23 @@ async def main_fn(memory):
     seqno = 0
     while True:
         try:
-            make_log("ConvertProcess", "Service running", level="debug")
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
+            make_log("ConvertProcess", "Service running", level="debug", rid=rid)
             await convert_loop(memory)
             await asyncio.sleep(5)
             await send_status("convert_service", f"working (seqno={seqno})")
             seqno += 1
         except BaseException as e:
-            make_log("ConvertProcess", f"Error: {e}", level="error")
+            make_log("ConvertProcess", f"Error: {e}", level="error", rid=locals().get('rid'))
             await asyncio.sleep(3)
+        finally:
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(None)
+            except BaseException:
+                pass

app/core/background/convert_v3_service.py

@@ -0,0 +1,479 @@
+import asyncio
+import os
+import json
+import shutil
+import tempfile
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core._config import UPLOADS_DIR, BACKEND_LOGS_DIR_HOST
+from app.core.models.content_v3 import (
+    EncryptedContent,
+    ContentKey,
+    ContentDerivative,
+    UploadSession,
+)
+from app.core.models.node_storage import StoredContent
+from app.core.ipfs_client import cat_stream
+from app.core.crypto.encf_stream import decrypt_encf_auto
+from app.core.crypto.keywrap import unwrap_dek, wrap_dek, KeyWrapError
+from app.core.network.key_client import request_key_from_peer
+from app.core.models.my_network import KnownNode
+from app.core._utils.resolve_content import resolve_content
+from app.core.content.content_id import ContentId
+
+
+CONCURRENCY = int(os.getenv("CONVERT_V3_MAX_CONCURRENCY", "3"))
+STAGING_SUBDIR = os.getenv("CONVERT_V3_STAGING_SUBDIR", "convert-staging")
+UPLOADS_PATH = Path(UPLOADS_DIR).resolve()
+_host_uploads_env = os.getenv("BACKEND_DATA_DIR_HOST")
+HOST_UPLOADS_PATH = Path(_host_uploads_env).resolve() if _host_uploads_env else None
+
+
+@dataclass
+class PlainStaging:
+    container_path: str
+    host_path: str
+
+
+def _container_to_host(path: str) -> str:
+    """Map a container path under UPLOADS_DIR to the host path for docker -v."""
+    if not HOST_UPLOADS_PATH:
+        raise RuntimeError("BACKEND_DATA_DIR_HOST is not configured for convert_v3")
+    real_path = Path(path).resolve()
+    try:
+        real_path.relative_to(UPLOADS_PATH)
+    except ValueError:
+        # Not under uploads; best effort fallback to original string
+        return str(real_path)
+    rel = real_path.relative_to(UPLOADS_PATH)
+    return str(HOST_UPLOADS_PATH / rel)
+
+
+MEDIA_CONVERTER_CPU_LIMIT = os.getenv("MEDIA_CONVERTER_CPU_LIMIT")
+MEDIA_CONVERTER_MEM_LIMIT = os.getenv("MEDIA_CONVERTER_MEM_LIMIT")
+MEDIA_CONVERTER_CPUSET = os.getenv("MEDIA_CONVERTER_CPUSET") or os.getenv("CONVERT_CPUSET")
+ERROR_TRUNCATE_LIMIT = 512
+
+
+def _ensure_dir(path: str):
+    try:
+        os.makedirs(path, exist_ok=True)
+    except Exception:
+        pass
+
+
+async def _sha256_b58(file_path: str) -> str:
+    import hashlib
+    import base58
+    h = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(2 * 1024 * 1024), b''):
+            h.update(chunk)
+    return base58.b58encode(h.digest()).decode()
+
+
+async def _save_derivative(file_path: str, filename: str) -> Tuple[str, int]:
+    """Move file into UPLOADS_DIR under sha256 b58 name; return (hash_b58, size)."""
+    file_hash = await _sha256_b58(file_path)
+    dst = os.path.join(UPLOADS_DIR, file_hash)
+    try:
+        os.remove(dst)
+    except FileNotFoundError:
+        pass
+    shutil.move(file_path, dst)
+    size = os.path.getsize(dst)
+    return file_hash, size
+
+
+async def _run_media_converter(staging: PlainStaging, input_ext: str, quality: str, trim_value: Optional[str], is_audio: bool):
+    if not os.path.exists(staging.container_path):
+        raise FileNotFoundError(f"Plain input missing at {staging.container_path}")
+
+    host_input_path = staging.host_path
+    if not host_input_path or not host_input_path.startswith('/'):
+        host_input_path = os.path.abspath(host_input_path)
+
+    rid = __import__('uuid').uuid4().hex[:8]
+    output_dir_container = UPLOADS_PATH / "convert-output" / f"conv_{rid}"
+    output_dir_host = _container_to_host(output_dir_container)
+    _ensure_dir(str(output_dir_container))
+
+    logs_dir_candidate = os.getenv("BACKEND_LOGS_DIR_HOST", "")
+    logs_dir_host = logs_dir_candidate if logs_dir_candidate else str(HOST_UPLOADS_PATH / "logs" / "converter") if HOST_UPLOADS_PATH else "/tmp/converter-logs"
+    if not logs_dir_host.startswith('/'):
+        logs_dir_host = os.path.join(os.getcwd(), logs_dir_host)
+    try:
+        os.makedirs(logs_dir_host, exist_ok=True)
+    except Exception:
+        fallback_logs = HOST_UPLOADS_PATH / "logs" / "converter" if HOST_UPLOADS_PATH else Path("/tmp/converter-logs")
+        logs_dir_host = str(fallback_logs)
+        os.makedirs(logs_dir_host, exist_ok=True)
+
+    cmd = [
+        "docker", "run", "--rm",
+        "-v", f"{host_input_path}:/app/input:ro",
+        "-v", f"{output_dir_host}:/app/output",
+        "-v", f"{logs_dir_host}:/app/logs",
+    ]
+    if MEDIA_CONVERTER_CPU_LIMIT:
+        cmd.extend(["--cpus", str(MEDIA_CONVERTER_CPU_LIMIT)])
+    if MEDIA_CONVERTER_MEM_LIMIT:
+        cmd.extend(["--memory", str(MEDIA_CONVERTER_MEM_LIMIT)])
+    if MEDIA_CONVERTER_CPUSET:
+        cmd.extend(["--cpuset-cpus", MEDIA_CONVERTER_CPUSET])
+
+    cmd.append("media_converter")
+    cmd.extend(["--ext", input_ext, "--quality", quality])
+    if trim_value:
+        cmd.extend(["--trim", trim_value])
+
+    make_log('convert_v3', f"Run media_converter cmd: {' '.join(cmd)}")
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        raise RuntimeError(f"media_converter failed: {stderr.decode()}")
+
+    # Find produced media file and optional output.json
+    try:
+        files = os.listdir(output_dir_container)
+    except Exception as e:
+        raise RuntimeError(f"Read output dir error: {e}")
+    media_files = [f for f in files if f != "output.json"]
+    if len(media_files) != 1:
+        raise RuntimeError(f"Expected one media file, found {len(media_files)}: {media_files}")
+    output_media = os.path.join(output_dir_container, media_files[0])
+    ffprobe_meta = {}
+    out_json = os.path.join(output_dir_container, "output.json")
+    if os.path.exists(out_json):
+        try:
+            with open(out_json, 'r') as f:
+                ffprobe_meta = json.load(f)
+        except Exception:
+            ffprobe_meta = {}
+    return output_media, ffprobe_meta
+
+
+async def _update_upload_session(ec: EncryptedContent, all_success: bool, errors: List[str]):
+    async with db_session() as session:
+        upload_row = (await session.execute(
+            select(UploadSession).where(UploadSession.encrypted_cid == ec.encrypted_cid)
+        )).scalars().first()
+        if upload_row:
+            if all_success:
+                upload_row.state = 'converted'
+                upload_row.error = None
+            elif upload_row.state != 'converted':
+                upload_row.state = 'conversion_failed'
+                if errors:
+                    upload_row.error = _short_error(errors[0])
+            await session.commit()
+
+
+async def _convert_content(ec: EncryptedContent, staging: PlainStaging):
+    content_kind = 'audio' if ec.content_type.startswith('audio/') else ('video' if ec.content_type.startswith('video/') else 'other')
+    input_ext = (ec.content_type.split('/')[-1] or 'bin')
+    is_audio = content_kind == 'audio'
+    encrypted_hash_b58 = ContentId.deserialize(ec.encrypted_cid).content_hash_b58
+
+    if content_kind == 'other':
+        errors: List[str] = []
+        all_success = True
+        try:
+            file_hash, size_bytes = await _save_derivative(staging.container_path, staging.container_path)
+            plain_path = os.path.join(UPLOADS_DIR, file_hash)
+            plain_filename = f"{ec.encrypted_cid}.{input_ext}" if input_ext else ec.encrypted_cid
+            async with db_session() as session:
+                existing = (await session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first()
+                if not existing:
+                    sc = StoredContent(
+                        type="local/content_bin",
+                        hash=file_hash,
+                        user_id=None,
+                        filename=plain_filename,
+                        meta={'encrypted_cid': ec.encrypted_cid, 'kind': 'original'},
+                        created=datetime.utcnow(),
+                    )
+                    session.add(sc)
+                    await session.flush()
+                derivative = ContentDerivative(
+                    content_id=ec.id,
+                    kind='decrypted_original',
+                    local_path=plain_path,
+                    content_type=ec.content_type,
+                    size_bytes=size_bytes,
+                    status='ready',
+                )
+                session.add(derivative)
+                await session.commit()
+            make_log('convert_v3', f"Stored original derivative for {ec.encrypted_cid}")
+        except Exception as e:
+            all_success = False
+            errors.append(str(e))
+            make_log('convert_v3', f"Convert error {ec.encrypted_cid} opt=original: {e}", level='error')
+        await _update_upload_session(ec, all_success, errors)
+        return
+
+    # audio/video path
+    required = ['high', 'low', 'low_preview']
+    conf = ec.preview_conf or {}
+    intervals = conf.get('intervals') or [[0, int(conf.get('duration_ms', 30000))]]
+    main_interval = intervals[0]
+    start_s = max(0, int(main_interval[0]) // 1000)
+    dur_s = max(1, int((main_interval[1] - main_interval[0]) // 1000) or 30)
+    trim_value = f"{start_s}-{start_s + dur_s}"
+
+    qualities = {
+        'high': 'high',
+        'low': 'low',
+        'low_preview': 'low',
+    }
+
+    all_success = True
+    errors: List[str] = []
+
+    for opt in required:
+        derivative_kind = f"decrypted_{opt if opt != 'low_preview' else 'preview'}"
+        derivative_id: Optional[int] = None
+        try:
+            async with db_session() as session:
+                cd = ContentDerivative(
+                    content_id=ec.id,
+                    kind=derivative_kind,
+                    interval_start_ms=main_interval[0] if opt == 'low_preview' else None,
+                    interval_end_ms=main_interval[1] if opt == 'low_preview' else None,
+                    local_path="",
+                    status='processing',
+                )
+                session.add(cd)
+                await session.flush()
+                derivative_id = cd.id
+                await session.commit()
+
+            out_path, ffprobe = await _run_media_converter(
+                staging=staging,
+                input_ext=input_ext,
+                quality=qualities[opt],
+                trim_value=trim_value if opt == 'low_preview' else None,
+                is_audio=is_audio,
+            )
+
+            file_hash, size_bytes = await _save_derivative(out_path, os.path.basename(out_path))
+
+            async with db_session() as session:
+                sc = (await session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first()
+                meta_payload = {'encrypted_cid': ec.encrypted_cid, 'kind': opt, 'ffprobe_meta': ffprobe}
+                if sc:
+                    sc.type = sc.type or "local/content_bin"
+                    sc.filename = os.path.basename(out_path)
+                    sc.meta = meta_payload
+                    sc.updated = datetime.utcnow()
+                else:
+                    sc = StoredContent(
+                        type="local/content_bin",
+                        hash=file_hash,
+                        user_id=None,
+                        filename=os.path.basename(out_path),
+                        meta=meta_payload,
+                        created=datetime.utcnow(),
+                    )
+                    session.add(sc)
+                    await session.flush()
+
+                encrypted_sc = (await session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash_b58))).scalars().first()
+                if encrypted_sc:
+                    meta = dict(encrypted_sc.meta or {})
+                    converted = dict(meta.get('converted_content') or {})
+                    converted[opt] = file_hash
+                    meta['converted_content'] = converted
+                    encrypted_sc.meta = meta
+                    if opt == 'high':
+                        encrypted_sc.decrypted_content_id = sc.id
+                    encrypted_sc.updated = datetime.utcnow()
+
+                cd = await session.get(ContentDerivative, derivative_id) if derivative_id else None
+                if cd:
+                    cd.local_path = os.path.join(UPLOADS_DIR, file_hash)
+                    cd.size_bytes = size_bytes
+                    if is_audio:
+                        cd.content_type = 'audio/flac' if opt == 'high' else 'audio/mpeg'
+                    else:
+                        cd.content_type = ec.content_type if opt == 'high' else 'video/mp4'
+                    cd.status = 'ready'
+                    cd.error = None
+                await session.commit()
+
+            output_parent = Path(out_path).parent
+            shutil.rmtree(output_parent, ignore_errors=True)
+            make_log('convert_v3', f"Converted {ec.encrypted_cid} opt={opt} -> {file_hash}")
+        except Exception as e:
+            make_log('convert_v3', f"Convert error {ec.encrypted_cid} opt={opt}: {e}", level='error')
+            all_success = False
+            errors.append(_short_error(e))
+            async with db_session() as session:
+                cd = await session.get(ContentDerivative, derivative_id) if derivative_id else None
+                if cd:
+                    cd.status = 'failed'
+                    cd.error = _short_error(e)
+                else:
+                    session.add(ContentDerivative(
+                        content_id=ec.id,
+                        kind=derivative_kind,
+                        status='failed',
+                        error=_short_error(e),
+                        local_path="",
+                    ))
+                await session.commit()
+
+    await _update_upload_session(ec, all_success, errors)
+
+
+async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, PlainStaging]]:
+    async with db_session() as session:
+        # Find A/V contents with preview_enabled and no ready low/low_preview derivatives yet
+        ecs = (await session.execute(select(EncryptedContent).where(
+            EncryptedContent.preview_enabled == True
+        ).order_by(EncryptedContent.created_at.desc()))).scalars().all()
+
+        picked: List[Tuple[EncryptedContent, PlainStaging]] = []
+        for ec in ecs:
+            try:
+                cid_obj, cid_err = resolve_content(ec.encrypted_cid)
+                if cid_err:
+                    make_log('convert_v3', f"Skip {ec.encrypted_cid}: resolve error {cid_err}", level='debug')
+                    continue
+                encrypted_hash_b58 = cid_obj.content_hash_b58
+            except Exception as exc:
+                make_log('convert_v3', f"Skip {ec.encrypted_cid}: resolve exception {exc}", level='warning')
+                continue
+
+            sc = (await session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash_b58))).scalars().first()
+            if not sc or sc.onchain_index is None:
+                continue
+
+            # Check if derivatives already ready
+            rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id))).scalars().all()
+            kinds_ready = {r.kind for r in rows if r.status == 'ready'}
+            required = {'decrypted_low', 'decrypted_high'} if ec.content_type.startswith('audio/') else {'decrypted_low', 'decrypted_high', 'decrypted_preview'}
+            if required.issubset(kinds_ready):
+                continue
+            # Always decrypt from IPFS using local or remote key
+            staging: Optional[PlainStaging] = None
+            ck = (await session.execute(select(ContentKey).where(ContentKey.content_id == ec.id))).scalars().first()
+            if ck:
+                staging = await stage_plain_from_ipfs(ec, ck.key_ciphertext_b64)
+            if not staging:
+                peers = (await session.execute(select(KnownNode))).scalars().all()
+                for peer in peers:
+                    base_url = f"http://{peer.ip}:{peer.port}"
+                    dek = await request_key_from_peer(base_url, ec.encrypted_cid)
+                    if not dek:
+                        continue
+                    try:
+                        dek_b64 = wrap_dek(dek)
+                    except KeyWrapError as exc:
+                        make_log('convert_v3', f"wrap failed for peer DEK: {exc}", level='error')
+                        continue
+                    session_ck = ContentKey(
+                        content_id=ec.id,
+                        key_ciphertext_b64=dek_b64,
+                        key_fingerprint=peer.public_key,
+                        issuer_node_id=peer.public_key,
+                        allow_auto_grant=True,
+                    )
+                    session.add(session_ck)
+                    await session.commit()
+                    staging = await stage_plain_from_ipfs(ec, dek_b64)
+                    if staging:
+                        break
+            if not staging or not os.path.exists(staging.container_path):
+                continue
+            picked.append((ec, staging))
+            if len(picked) >= limit:
+                break
+        return picked
+
+
+async def worker_loop():
+    sem = asyncio.Semaphore(CONCURRENCY)
+
+    async def _run_one(ec: EncryptedContent, staging: PlainStaging):
+        async with sem:
+            try:
+                await _convert_content(ec, staging)
+                # After successful conversion, attempt to remove staging file to avoid duplicates
+                try:
+                    if staging and staging.container_path and os.path.exists(staging.container_path):
+                        os.remove(staging.container_path)
+                except Exception:
+                    pass
+            except Exception as e:
+                make_log('convert_v3', f"job error {ec.encrypted_cid}: {e}", level='error')
+
+    while True:
+        try:
+            batch = await _pick_pending(limit=CONCURRENCY * 2)
+            if not batch:
+                await asyncio.sleep(3)
+                continue
+            tasks = [asyncio.create_task(_run_one(ec, staging)) for (ec, staging) in batch]
+            await asyncio.gather(*tasks)
+        except Exception as e:
+            make_log('convert_v3', f"loop error: {e}", level='error')
+            await asyncio.sleep(2)
+
+
+async def main_fn(memory):
+    make_log('convert_v3', f"Service started with concurrency={CONCURRENCY}", level='info')
+    await worker_loop()
+
+
+async def stage_plain_from_ipfs(ec: EncryptedContent, dek_wrapped: str) -> Optional[PlainStaging]:
+    """Download encrypted ENCF stream from IPFS and decrypt on the fly into shared staging."""
+    os.makedirs(UPLOADS_PATH / STAGING_SUBDIR, exist_ok=True)
+    try:
+        dek = unwrap_dek(dek_wrapped)
+    except KeyWrapError as exc:
+        make_log('convert_v3', f"unwrap failed for {ec.encrypted_cid}: {exc}", level='error')
+        return None
+
+    tmp = tempfile.NamedTemporaryFile(
+        prefix=f"dec_{ec.encrypted_cid[:8]}_",
+        dir=UPLOADS_PATH / STAGING_SUBDIR,
+        delete=False,
+    )
+    tmp_path = tmp.name
+    tmp.close()
+    try:
+        async def _aiter():
+            async for ch in cat_stream(ec.encrypted_cid):
+                yield ch
+        await decrypt_encf_auto(_aiter(), dek, tmp_path)
+        host_path = _container_to_host(tmp_path)
+        return PlainStaging(container_path=tmp_path, host_path=host_path)
+    except Exception as e:
+        make_log('convert_v3', f"decrypt from ipfs failed: {e}", level='error')
+        try:
+            os.remove(tmp_path)
+        except Exception:
+            pass
+        return None
+
+
+    
+def _short_error(message: str, limit: int = ERROR_TRUNCATE_LIMIT) -> str:
+    if not message:
+        return message
+    message = str(message)
+    return message if len(message) <= limit else message[: limit - 3] + '...'

app/core/background/derivative_cache_janitor.py

@@ -0,0 +1,97 @@
+import asyncio
+import os
+from datetime import datetime, timedelta
+
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core.models.content_v3 import ContentDerivative
+from app.core.models._config import ServiceConfig
+
+
+ENV_MAX_GB = float(os.getenv('DERIVATIVE_CACHE_MAX_GB', '50'))
+ENV_TTL_DAYS = int(os.getenv('DERIVATIVE_CACHE_TTL_DAYS', '0'))
+INTERVAL_SEC = int(os.getenv('DERIVATIVE_JANITOR_INTERVAL_SEC', '600'))
+
+
+async def _current_total_size() -> int:
+    async with db_session() as session:
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        return sum(int(r.size_bytes or 0) for r in rows)
+
+
+async def _evict_over_ttl(now: datetime) -> int:
+    removed = 0
+    # Pull TTL from ServiceConfig each time
+    async with db_session() as session:
+        ttl_days = int(await ServiceConfig(session).get('DERIVATIVE_CACHE_TTL_DAYS', ENV_TTL_DAYS))
+    if ttl_days <= 0:
+        return 0
+    async with db_session() as session:
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        for r in rows:
+            la = r.last_access_at or r.created_at
+            if la and (now - la) > timedelta(days=ttl_days):
+                try:
+                    if r.local_path and os.path.exists(r.local_path):
+                        os.remove(r.local_path)
+                except Exception:
+                    pass
+                r.status = 'pending'
+                r.local_path = None
+                r.size_bytes = None
+                r.last_access_at = None
+                removed += 1
+        await session.commit()
+    return removed
+
+
+async def _evict_to_fit():
+    async with db_session() as session:
+        max_gb = await ServiceConfig(session).get('DERIVATIVE_CACHE_MAX_GB', ENV_MAX_GB)
+    limit_bytes = int(float(max_gb) * (1024 ** 3))
+    total = await _current_total_size()
+    if total <= limit_bytes:
+        return 0
+    to_remove = total - limit_bytes
+    removed = 0
+    async with db_session() as session:
+        # Oldest first by last_access_at
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        rows.sort(key=lambda r: (r.last_access_at or r.created_at or datetime.utcfromtimestamp(0)))
+        for r in rows:
+            if to_remove <= 0:
+                break
+            size = int(r.size_bytes or 0)
+            try:
+                if r.local_path and os.path.exists(r.local_path):
+                    os.remove(r.local_path)
+            except Exception:
+                pass
+            r.status = 'pending'
+            r.local_path = None
+            r.last_access_at = None
+            r.size_bytes = None
+            await session.commit()
+            to_remove -= size
+            removed += 1
+    return removed
+
+
+async def main_fn(memory):
+    async with db_session() as session:
+        cfg = ServiceConfig(session)
+        runtime_max_gb = float(await cfg.get('DERIVATIVE_CACHE_MAX_GB', ENV_MAX_GB))
+        runtime_ttl_days = int(await cfg.get('DERIVATIVE_CACHE_TTL_DAYS', ENV_TTL_DAYS))
+    make_log('derivative_janitor', f"Started (MAX_GB={runtime_max_gb}, TTL_DAYS={runtime_ttl_days})", level='info')
+    while True:
+        try:
+            now = datetime.utcnow()
+            r1 = await _evict_over_ttl(now)
+            r2 = await _evict_to_fit()
+            if r1 or r2:
+                make_log('derivative_janitor', f"Evicted: ttl={r1}, fit={r2}")
+        except Exception as e:
+            make_log('derivative_janitor', f"Error: {e}", level='error')
+        await asyncio.sleep(INTERVAL_SEC)

app/core/background/index_scout_v3.py

@@ -0,0 +1,185 @@
+import asyncio
+import os
+from typing import List, Optional
+
+import httpx
+import random
+import shutil
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core.models.my_network import KnownNode
+from app.core.models.content_v3 import EncryptedContent, ContentDerivative
+from app.core.ipfs_client import pin_add, find_providers, swarm_connect
+
+
+INTERVAL_SEC = 60
+ENV_PIN_CONCURRENCY = int(os.getenv('SYNC_MAX_CONCURRENT_PINS', '4'))
+ENV_DISK_WATERMARK_PCT = int(os.getenv('SYNC_DISK_LOW_WATERMARK_PCT', '90'))
+
+
+async def fetch_index(base_url: str, etag: Optional[str], since: Optional[str]) -> tuple[List[dict], Optional[str]]:
+    try:
+        headers = {}
+        params = {}
+        if since:
+            params['since'] = since
+        url = f"{base_url.rstrip('/')}/api/v1/content.delta" if since else f"{base_url.rstrip('/')}/api/v1/content.index"
+        if etag:
+            headers['If-None-Match'] = etag
+        async with httpx.AsyncClient(timeout=20) as client:
+            r = await client.get(url, headers=headers, params=params)
+            if r.status_code != 200:
+                if r.status_code == 304:
+                    return [], etag
+                return [], etag
+            j = r.json()
+            new_etag = r.headers.get('ETag') or etag
+            return j.get('items') or [], (j.get('next_since') or new_etag or etag)
+    except Exception:
+        return [], etag
+
+
+async def upsert_content(item: dict):
+    cid = item.get('encrypted_cid')
+    if not cid:
+        return
+    async with db_session() as session:
+        row = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+        if not row:
+            row = EncryptedContent(
+                encrypted_cid=cid,
+                title=item.get('title') or cid,
+                description=item.get('description') or '',
+                content_type=item.get('content_type') or 'application/octet-stream',
+                enc_size_bytes=item.get('size_bytes'),
+                preview_enabled=bool(item.get('preview_enabled')),
+                preview_conf=item.get('preview_conf') or {},
+                salt_b64=item.get('salt_b64'),
+            )
+            session.add(row)
+        else:
+            row.title = item.get('title') or row.title
+            row.description = item.get('description') or row.description
+            row.content_type = item.get('content_type') or row.content_type
+            row.enc_size_bytes = item.get('size_bytes') or row.enc_size_bytes
+            row.preview_enabled = bool(item.get('preview_enabled')) if item.get('preview_enabled') is not None else row.preview_enabled
+            if item.get('preview_conf'):
+                row.preview_conf = item['preview_conf']
+            if item.get('salt_b64'):
+                row.salt_b64 = item['salt_b64']
+        await session.commit()
+
+    # Fetch thumbnail via HTTP if provided and not present locally
+    cover_url = item.get('cover_url')
+    if cover_url:
+        try:
+            async with db_session() as session:
+                ec = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+                have_thumb = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id, ContentDerivative.kind == 'decrypted_thumbnail', ContentDerivative.status == 'ready'))).scalars().first()
+                if not have_thumb:
+                    import httpx, tempfile, os
+                    async with httpx.AsyncClient(timeout=30) as client:
+                        r = await client.get(cover_url)
+                        r.raise_for_status()
+                        tmp = tempfile.NamedTemporaryFile(delete=False)
+                        tmp.write(r.content)
+                        tmp.close()
+                    # Save into store
+                    from app.core.background.convert_v3_service import _save_derivative
+                    h, size = await _save_derivative(tmp.name, os.path.basename(cover_url) or 'thumb.jpg')
+                    cd = ContentDerivative(
+                        content_id=ec.id,
+                        kind='decrypted_thumbnail',
+                        local_path=os.path.join(os.getenv('UPLOADS_DIR', '/app/data'), h),
+                        content_type=r.headers.get('Content-Type') or 'image/jpeg',
+                        size_bytes=size,
+                        status='ready',
+                    )
+                    session.add(cd)
+                    await session.commit()
+        except Exception as e:
+            make_log('index_scout_v3', f"thumbnail fetch failed for {cid}: {e}", level='warning')
+
+
+async def main_fn(memory):
+    make_log('index_scout_v3', 'Service started', level='info')
+    sem = None
+    while True:
+        try:
+            # Read runtime config from ServiceConfig (fallback to env)
+            from app.core.models._config import ServiceConfig
+            async with db_session() as session:
+                max_pins = int(await ServiceConfig(session).get('SYNC_MAX_CONCURRENT_PINS', ENV_PIN_CONCURRENCY))
+                disk_pct = int(await ServiceConfig(session).get('SYNC_DISK_LOW_WATERMARK_PCT', ENV_DISK_WATERMARK_PCT))
+            if sem is None or sem._value != max_pins:
+                sem = asyncio.Semaphore(max_pins)
+            async with db_session() as session:
+                nodes = (await session.execute(select(KnownNode))).scalars().all()
+            for n in nodes:
+                base = f"http://{n.ip}:{n.port}"
+                # jitter 0..30s per node to reduce stampede
+                await asyncio.sleep(random.uniform(0, 30))
+                etag = (n.meta or {}).get('index_etag')
+                since = (n.meta or {}).get('index_since')
+                items, marker = await fetch_index(base, etag, since)
+                if not items and marker == etag:
+                    continue
+                # update node markers
+                try:
+                    async with db_session() as session:
+                        row = (await session.execute(select(KnownNode).where(KnownNode.id == n.id))).scalars().first()
+                        if row:
+                            meta = row.meta or {}
+                            meta['index_etag'] = marker
+                            meta['index_since'] = marker if (marker and 'T' in str(marker)) else meta.get('index_since')
+                            row.meta = meta
+                            await session.commit()
+                except Exception:
+                    pass
+
+                if not items:
+                    continue
+                make_log('index_scout_v3', f"Fetched {len(items)} from {base}")
+
+                # Check disk watermark
+                try:
+                    from app.core._config import UPLOADS_DIR
+                    du = shutil.disk_usage(UPLOADS_DIR)
+                    used_pct = int(100 * (1 - du.free / du.total))
+                    if used_pct >= disk_pct:
+                        make_log('index_scout_v3', f"Disk watermark reached ({used_pct}%), skipping pins")
+                        continue
+                except Exception:
+                    pass
+
+                async def _pin_one(cid: str):
+                    async with sem:
+                        try:
+                            # Try to pre-connect to discovered providers
+                            try:
+                                provs = await find_providers(cid, max_results=5)
+                                for p in provs:
+                                    for addr in (p.get('addrs') or [])[:2]:
+                                        try:
+                                            await swarm_connect(addr)
+                                        except Exception:
+                                            pass
+                            except Exception:
+                                pass
+                            await pin_add(cid, recursive=True)
+                        except Exception as e:
+                            make_log('index_scout_v3', f"pin {cid} failed: {e}", level='warning')
+
+                tasks = []
+                for it in items:
+                    await upsert_content(it)
+                    cid = it.get('encrypted_cid')
+                    if cid:
+                        tasks.append(asyncio.create_task(_pin_one(cid)))
+                if tasks:
+                    await asyncio.gather(*tasks)
+        except Exception as e:
+            make_log('index_scout_v3', f"loop error: {e}", level='error')
+        await asyncio.sleep(INTERVAL_SEC)

app/core/background/indexer_service.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 from base64 import b64decode
 from datetime import datetime
 
@@ -6,17 +7,21 @@ from base58 import b58encode
 from sqlalchemy import String, and_, desc, cast
 from tonsdk.boc import Cell
 from tonsdk.utils import Address
-from app.core._config import CLIENT_TELEGRAM_BOT_USERNAME
+from app.core._config import CLIENT_TELEGRAM_BOT_USERNAME, PROJECT_HOST
 from app.core._blockchain.ton.platform import platform
 from app.core._blockchain.ton.toncenter import toncenter
 from app.core._utils.send_status import send_status
 from app.core.logger import make_log
 from app.core.models import UserContent, KnownTelegramMessage, ServiceConfig
+from app.core.models.user import User
 from app.core.models.node_storage import StoredContent
 from app.core._utils.resolve_content import resolve_content
 from app.core.models.wallet_connection import WalletConnection
-from app.core._keyboards import get_inline_keyboard
 from app.core.models._telegram import Wrapped_CBotChat
+
+
+MIN_ONCHAIN_INDEX = int(os.getenv("MIN_ONCHAIN_INDEX", "8"))
+from sqlalchemy import select, and_, desc
 from app.core.storage import db_session
 import os
 import traceback
@@ -33,7 +38,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
             platform_found = True
 
     make_log("Indexer", "Service running", level="debug")
-    with db_session() as session:
+    async with db_session() as session:
         try:
             result = await toncenter.run_get_method('EQD8TJ8xEWB1SpnRE4d89YO3jl0W0EiBnNS4IBaHaUmdfizE', 'get_pool_data')
             assert result['exit_code'] == 0, f"Error in get-method: {result}"
@@ -41,40 +46,44 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
             assert result['stack'][1][0] == 'num', f"get second element is not num"
             usdt_per_ton = (int(result['stack'][0][1], 16) * 1e3) / int(result['stack'][1][1], 16)
             ton_per_star = 0.014 / usdt_per_ton
-            ServiceConfig(session).set('live_tonPerStar', [ton_per_star, datetime.utcnow().timestamp()])
+            await ServiceConfig(session).set('live_tonPerStar', [ton_per_star, datetime.utcnow().timestamp()])
             make_log("TON_Daemon", f"TON per STAR price: {ton_per_star}", level="DEBUG")
         except BaseException as e:
             make_log("TON_Daemon", f"Error while saving TON per STAR price: {e}" + '\n' + traceback.format_exc(), level="ERROR")
 
-        new_licenses = session.query(UserContent).filter(
+        from sqlalchemy import cast
+        from sqlalchemy.dialects.postgresql import JSONB
+        new_licenses = (await session.execute(select(UserContent).where(
             and_(
-                ~UserContent.meta.contains({'notification_sent': True}),
+                ~(cast(UserContent.meta, JSONB).contains({'notification_sent': True})),
                 UserContent.type == 'nft/listen'
             )
-        ).all()
+        ))).scalars().all()
         for new_license in new_licenses:
-            licensed_content = session.query(StoredContent).filter(
+            licensed_content = (await session.execute(select(StoredContent).where(
                 StoredContent.id == new_license.content_id
-            ).first()
+            ))).scalars().first()
             if not licensed_content:
                 make_log("Indexer", f"Licensed content not found: {new_license.content_id}", level="error")
 
-            content_metadata = licensed_content.metadata_json(session)
+            content_metadata = await licensed_content.metadata_json_async(session)
             assert content_metadata, "No content metadata found"
 
             if not (licensed_content.owner_address == new_license.owner_address):
                 try:
-                    user = new_license.user
+                    user = await session.get(User, new_license.user_id)
                     if user.telegram_id and licensed_content:
                         await (Wrapped_CBotChat(memory._client_telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)).send_content(
                             session, licensed_content
                         )
 
-                    wallet_owner_connection = session.query(WalletConnection).filter_by(
+                    wallet_owner_connection = (await session.execute(
-                        wallet_address=licensed_content.owner_address,
+                        select(WalletConnection).where(
-                        invalidated=False
+                            WalletConnection.wallet_address == licensed_content.owner_address,
-                    ).order_by(desc(WalletConnection.id)).first()
+                            WalletConnection.invalidated == False
-                    wallet_owner_user = wallet_owner_connection.user
+                        ).order_by(desc(WalletConnection.id))
+                    )).scalars().first()
+                    wallet_owner_user = await session.get(User, wallet_owner_connection.user_id) if wallet_owner_connection else None
                     if wallet_owner_user.telegram_id:
                         wallet_owner_bot = Wrapped_CBotChat(memory._telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
                         await wallet_owner_bot.send_message(
@@ -89,28 +98,30 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
                     make_log("IndexerSendNewLicense", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
 
             new_license.meta = {**new_license.meta, 'notification_sent': True}
-            session.commit()
+            await session.commit()
 
-        content_without_cid = session.query(StoredContent).filter(
+        content_without_cid = (await session.execute(select(StoredContent).where(StoredContent.content_id == None))).scalars().all()
-            StoredContent.content_id == None
-        )
         for target_content in content_without_cid:
             target_cid = target_content.cid.serialize_v2()
             make_log("Indexer", f"Content without CID: {target_content.hash}, setting CID: {target_cid}", level="debug")
             target_content.content_id = target_cid
 
-        session.commit()
+        await session.commit()
 
-        last_known_index_ = session.query(StoredContent).filter(
+        last_known_index_ = (await session.execute(
-            StoredContent.onchain_index != None
+            select(StoredContent).where(StoredContent.onchain_index != None).order_by(StoredContent.onchain_index.desc())
-        ).order_by(StoredContent.onchain_index.desc()).first()
+        )).scalars().first()
         last_known_index = last_known_index_.onchain_index if last_known_index_ else 0
         last_known_index = max(last_known_index, 0)
+        if last_known_index < (MIN_ONCHAIN_INDEX - 1):
+            make_log(
+                "Indexer",
+                f"Adjusting last_known_index from {last_known_index} to {MIN_ONCHAIN_INDEX - 1} (MIN_ONCHAIN_INDEX)",
+                level="debug"
+            )
+            last_known_index = MIN_ONCHAIN_INDEX - 1
         make_log("Indexer", f"Last known index: {last_known_index}", level="debug")
-        if last_known_index_:
         next_item_index = last_known_index + 1
-        else:
-            next_item_index = 0
 
         resolve_item_result = await toncenter.run_get_method(platform.address.to_string(1, 1, 1), 'get_nft_address_by_index', [['num', next_item_index]])
         make_log("Indexer", f"Resolve item result: {resolve_item_result}", level="debug")
@@ -137,6 +148,13 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
 
         assert item_get_data_result['stack'][2][0] == 'num', "Item index is not a number"
         item_index = int(item_get_data_result['stack'][2][1], 16)
+        if item_index < MIN_ONCHAIN_INDEX:
+            make_log(
+                "Indexer",
+                f"Skip on-chain item {item_index}: below MIN_ONCHAIN_INDEX={MIN_ONCHAIN_INDEX}",
+                level="info"
+            )
+            return platform_found, seqno
         assert item_index == next_item_index, "Item index mismatch"
 
         item_platform_address = Cell.one_from_boc(b64decode(item_get_data_result['stack'][3][1]['bytes'])).begin_parse().read_msg_addr()
@@ -196,14 +214,13 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
 
         user_wallet_connection = None
         if item_owner_address:
-            user_wallet_connection = session.query(WalletConnection).filter(
+            user_wallet_connection = (await session.execute(select(WalletConnection).where(
                 WalletConnection.wallet_address == item_owner_address.to_string(1, 1, 1)
-            ).first()
+            ))).scalars().first()
 
-        encrypted_stored_content = session.query(StoredContent).filter(
+        encrypted_stored_content = (await session.execute(select(StoredContent).where(
-            StoredContent.hash == item_content_hash_str,
+            StoredContent.hash == item_content_hash_str
-            # StoredContent.type.like("local%")
+        ))).scalars().first()
-        ).first()
         if encrypted_stored_content:
             is_duplicate = encrypted_stored_content.type.startswith("onchain") \
                            and encrypted_stored_content.onchain_index != item_index
@@ -215,33 +232,40 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
                     user = None
                     if user_wallet_connection:
                         encrypted_stored_content.user_id = user_wallet_connection.user_id
-                        user = user_wallet_connection.user
+                        user = await session.get(User, user_wallet_connection.user_id)
 
                     if user:
                         user_uploader_wrapper = Wrapped_CBotChat(memory._telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)
-                        await user_uploader_wrapper.send_message(
+                        ref_id = (user.meta or {}).get('ref_id')
-                            user.translated('p_contentWasIndexed').format(
+                        if not ref_id:
+                            ref_id = user.ensure_ref_id()
+                            await session.commit()
+
+                        message_text = user.translated('p_contentWasIndexed').format(
                             item_address=item_address.to_string(1, 1, 1),
                             item_index=item_index,
-                            ),
+                        )
-                            message_type='notification',
+
-                            reply_markup=get_inline_keyboard([
+                        await user_uploader_wrapper.send_message(
-                                [{
+                            message_text,
-                                    'text': user.translated('viewTrackAsClient_button'),
+                            message_type='notification'
-                                    'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}?start=C{encrypted_stored_content.cid.serialize_v2()}"
+                        )
-                                }],
+
-                            ])
+                        await user_uploader_wrapper.send_content(
+                            session,
+                            encrypted_stored_content
                         )
 
                         try:
-                            for hint_message in session.query(KnownTelegramMessage).filter(
+                            result = await session.execute(select(KnownTelegramMessage).where(
                                 and_(
                                     KnownTelegramMessage.chat_id == user.telegram_id,
                                     KnownTelegramMessage.type == 'hint',
                                     cast(KnownTelegramMessage.meta['encrypted_content_hash'], String) == encrypted_stored_content.hash,
                                     KnownTelegramMessage.deleted == False
                                 )
-                            ).all():
+                            ))
+                            for hint_message in result.scalars().all():
                                 await user_uploader_wrapper.delete_message(hint_message.message_id)
                         except BaseException as e:
                             make_log("Indexer", f"Error while deleting hint messages: {e}" + '\n' + traceback.format_exc(), level="error")
@@ -259,8 +283,9 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
                     **encrypted_stored_content.meta,
                     **item_metadata_packed
                 }
+                encrypted_stored_content.content_id = item_content_cid_str
 
-                session.commit()
+                await session.commit()
                 return platform_found, seqno
             else:
                 item_metadata_packed['copied_from'] = encrypted_stored_content.id
@@ -279,10 +304,11 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
             encrypted=True,
             decrypted_content_id=None,
             key_id=None,
+            content_id=item_content_cid_str,
             updated=datetime.now()
         )
         session.add(onchain_stored_content)
-        session.commit()
+        await session.commit()
         make_log("Indexer", f"Item indexed: {item_content_hash_str}", level="info")
         last_known_index += 1
 
@@ -295,15 +321,27 @@ async def main_fn(memory, ):
     seqno = 0
     while True:
         try:
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
+            make_log("Indexer", f"Loop start", level="debug", rid=rid)
             platform_found, seqno = await indexer_loop(memory, platform_found, seqno)
         except BaseException as e:
-            make_log("Indexer", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
+            make_log("Indexer", f"Error: {e}" + '\n' + traceback.format_exc(), level="error", rid=locals().get('rid'))
 
         if platform_found:
             await send_status("indexer", f"working (seqno={seqno})")
 
         await asyncio.sleep(5)
         seqno += 1
+        try:
+            from app.core.log_context import ctx_rid
+            ctx_rid.set(None)
+        except BaseException:
+            pass
 
 
 

app/core/background/license_service.py

@@ -3,7 +3,7 @@ from base64 import b64decode
 from datetime import datetime, timedelta
 
 from base58 import b58encode
-from sqlalchemy import and_, or_
+from sqlalchemy import and_, or_, select, desc
 from tonsdk.boc import Cell
 from tonsdk.utils import Address
 
@@ -27,7 +27,7 @@ import traceback
 
 async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
     make_log("LicenseIndex", "Service running", level="debug")
-    with db_session() as session:
+    async with db_session() as session:
         async def check_telegram_stars_transactions():
             # Проверка звездных telegram транзакций, обновление paid
             offset = {'desc': 'Статичное число заранее известного количества транзакций, которое даже не знает наш бот', 'value': 1}['value'] + \
@@ -45,19 +45,19 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
                         continue
 
                     try:
-                        existing_invoice = session.query(StarsInvoice).filter(
+                        existing_invoice = (await session.execute(select(StarsInvoice).where(
                             StarsInvoice.external_id == star_payment.source.invoice_payload
-                        ).first()
+                        ))).scalars().first()
                         if not existing_invoice:
                             continue
 
                         if star_payment.amount == existing_invoice.amount:
                             if not existing_invoice.paid:
                                 existing_invoice.paid = True
-                                session.commit()
+                                await session.commit()
 
-                                licensed_content = session.query(StoredContent).filter(StoredContent.hash == existing_invoice.content_hash).first()
+                                licensed_content = (await session.execute(select(StoredContent).where(StoredContent.hash == existing_invoice.content_hash))).scalars().first()
-                                user = session.query(User).filter(User.id == existing_invoice.user_id).first()
+                                user = (await session.execute(select(User).where(User.id == existing_invoice.user_id))).scalars().first()
 
                                 await (Wrapped_CBotChat(memory._client_telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)).send_content(
                                     session, licensed_content
@@ -73,35 +73,37 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
             make_log("StarsProcessing", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
 
         # Проверка кошельков пользователей на появление новых NFT, добавление их в базу как неопознанные
-        for user in session.query(User).filter(
+        users = (await session.execute(select(User).where(
             User.last_use > datetime.now() - timedelta(hours=4)
-        ).order_by(User.updated.asc()).all():
+        ).order_by(User.updated.asc()))).scalars().all()
-            user_wallet_address = user.wallet_address(session)
+        for user in users:
+            user_wallet_address = await user.wallet_address_async(session)
             if not user_wallet_address:
-                make_log("LicenseIndex", f"User {user.id} has no wallet address", level="info")
+                make_log("LicenseIndex", f"User {user.id} has no wallet address", level="debug")
                 continue
 
-            make_log("LicenseIndex", f"User {user.id} has wallet address {user_wallet_address}", level="info")
+            make_log("LicenseIndex", f"User {user.id} has wallet address {user_wallet_address}", level="debug")
             last_updated_licenses = user.meta.get('last_updated_licenses')
             must_skip = last_updated_licenses and (datetime.now() - datetime.fromisoformat(last_updated_licenses)) < timedelta(minutes=1)
-            make_log("LicenseIndex", f"User: {user.id}, last_updated_licenses: {last_updated_licenses}, must_skip: {must_skip}", level="info")
+            make_log("LicenseIndex", f"User: {user.id}, last_updated_licenses: {last_updated_licenses}, must_skip: {must_skip}", level="debug")
             if must_skip:
                 continue
 
             try:
                 await user.scan_owned_user_content(session)
                 user.meta = {**user.meta, 'last_updated_licenses': datetime.now().isoformat()}
-                session.commit()
+                await session.commit()
             except BaseException as e:
                 make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
         
         # Проверка NFT на актуальность данных, в том числе уже проверенные
-        process_content = session.query(UserContent).filter(
+        process_content = (await session.execute(select(UserContent).where(
             and_(
                 UserContent.type.startswith('nft/'),
+                UserContent.type != 'nft/ignored',
                 UserContent.updated < (datetime.now() - timedelta(minutes=60)),
             )
-        ).order_by(UserContent.updated.asc()).first()
+        ).order_by(UserContent.updated.asc()))).scalars().first()
         if process_content:
             make_log("LicenseIndex", f"Syncing content with blockchain: {process_content.id}", level="info")
             try:
@@ -110,7 +112,7 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
                 make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
             finally:
                 process_content.updated = datetime.now()
-                session.commit()
+                await session.commit()
 
     return platform_found, seqno
 
@@ -121,14 +123,26 @@ async def main_fn(memory, ):
     seqno = 0
     while True:
         try:
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
+            make_log("LicenseIndex", f"Loop start", level="debug", rid=rid)
             platform_found, seqno = await license_index_loop(memory, platform_found, seqno)
             if platform_found:
                 await send_status("licenses", f"working (seqno={seqno})")
         except BaseException as e:
-            make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
+            make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error", rid=locals().get('rid'))
 
         await asyncio.sleep(1)
         seqno += 1
+        try:
+            from app.core.log_context import ctx_rid
+            ctx_rid.set(None)
+        except BaseException:
+            pass
 
 # if __name__ == '__main__':
 #     loop = asyncio.get_event_loop()

app/core/background/ton_service.py

@@ -97,7 +97,7 @@ async def main_fn(memory):
             service_wallet.create_transfer_message(
                 [{
                     'address': highload_wallet.address.to_string(1, 1, 0),
-                    'amount': int(0.08 * 10 ** 9),
+                    'amount': int(0.02 * 10 ** 9),
                     'send_mode': 1,
                     'payload': begin_cell().store_uint(0, 32).end_cell()
                 }], sw_seqno_value
@@ -122,10 +122,16 @@ async def main_fn(memory):
 
     while True:
         try:
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
             sw_seqno_value = await get_sw_seqno()
-            make_log("TON", f"Service running ({sw_seqno_value})", level="debug")
+            make_log("TON", f"Service running ({sw_seqno_value})", level="debug", rid=rid)
 
-            with db_session() as session:
+            async with db_session() as session:
                 # Проверка отправленных сообщений
                 await send_status("ton_daemon", f"working: processing in-txs (seqno={sw_seqno_value})")
                 async def process_incoming_transaction(transaction: dict):
@@ -142,14 +148,17 @@ async def main_fn(memory):
                         in_msg_created_at = in_msg_slice.read_uint(64)
                         in_msg_epoch = int(in_msg_created_at // (60 * 60))
                         in_msg_seqno = HighloadQueryId.from_query_id(in_msg_query_id).to_seqno()
+                        from sqlalchemy import select
                         in_msg_blockchain_task = (
-                            session.query(BlockchainTask).filter(
+                            await session.execute(
+                                select(BlockchainTask).where(
                                     and_(
                                         BlockchainTask.seqno == in_msg_seqno,
                                         BlockchainTask.epoch == in_msg_epoch,
                                     )
                                 )
-                        ).first()
+                            )
+                        ).scalars().first()
                         if not in_msg_blockchain_task:
                             return
 
@@ -157,13 +166,13 @@ async def main_fn(memory):
                             in_msg_blockchain_task.status = 'done'
                             in_msg_blockchain_task.transaction_hash = transaction_hash
                             in_msg_blockchain_task.transaction_lt = transaction_lt
-                            session.commit()
+                            await session.commit()
 
                     for blockchain_message in [transaction['in_msg']]:
                         try:
                             await process_incoming_message(blockchain_message)
                         except BaseException as e:
-                            pass # make_log("TON_Daemon", f"Error while processing incoming message: {e}" + '\n' + traceback.format_exc(), level='debug')
+                            pass # make_log("TON_Daemon", f"Error while processing incoming message: {e}" + '\n' + traceback.format_exc(), level='debug', rid=rid)
 
                 try:
                     sw_transactions = await toncenter.get_transactions(highload_wallet.address.to_string(1, 1, 1), limit=100)
@@ -171,18 +180,18 @@ async def main_fn(memory):
                         try:
                             await process_incoming_transaction(sw_transaction)
                         except BaseException as e:
-                            make_log("TON_Daemon", f"Error while processing incoming transaction: {e}", level="debug")
+                            make_log("TON_Daemon", f"Error while processing incoming transaction: {e}", level="debug", rid=rid)
                 except BaseException as e:
-                    make_log("TON_Daemon", f"Error while getting service wallet transactions: {e}", level="ERROR")
+                    make_log("TON_Daemon", f"Error while getting service wallet transactions: {e}", level="ERROR", rid=rid)
 
                 await send_status("ton_daemon", f"working: processing out-txs (seqno={sw_seqno_value})")
                 # Отправка подписанных сообщений
-                for blockchain_task in (
+                from sqlalchemy import select
-                    session.query(BlockchainTask).filter(
+                _processing = (await session.execute(select(BlockchainTask).where(
-                        BlockchainTask.status == 'processing',
+                    BlockchainTask.status == 'processing'
-                    ).order_by(BlockchainTask.updated.asc()).all()
+                ).order_by(BlockchainTask.updated.asc()))).scalars().all()
-                ):
+                for blockchain_task in _processing:
-                    make_log("TON_Daemon", f"Processing task (processing) {blockchain_task.id}")
+                    make_log("TON_Daemon", f"Processing task (processing) {blockchain_task.id}", rid=rid)
                     query_boc = bytes.fromhex(blockchain_task.meta['signed_message'])
                     errors_list = []
 
@@ -210,23 +219,22 @@ async def main_fn(memory):
                             # or sum([int("terminating vm with exit code 36" in e) for e in errors_list]) > 0:
                         make_log("TON_Daemon", f"Task {blockchain_task.id} done", level="DEBUG")
                         blockchain_task.status = 'done'
-                        session.commit()
+                        await session.commit()
                         continue
 
                     await asyncio.sleep(0.5)
 
                 await send_status("ton_daemon", f"working: creating new messages (seqno={sw_seqno_value})")
                 # Создание новых подписей
-                for blockchain_task in (
+                _waiting = (await session.execute(select(BlockchainTask).where(BlockchainTask.status == 'wait'))).scalars().all()
-                    session.query(BlockchainTask).filter(BlockchainTask.status == 'wait').all()
+                for blockchain_task in _waiting:
-                ):
                     try:
                         # Check processing tasks in current epoch < 3_000_000
-                        if (
+                        from sqlalchemy import func
-                            session.query(BlockchainTask).filter(
+                        _cnt = (await session.execute(select(func.count()).select_from(BlockchainTask).where(
-                                BlockchainTask.epoch == blockchain_task.epoch,
+                            BlockchainTask.epoch == blockchain_task.epoch
-                            ).count() > 3_000_000
+                        ))).scalar() or 0
-                        ):
+                        if _cnt > 3_000_000:
                             make_log("TON", f"Too many processing tasks in epoch {blockchain_task.epoch}", level="error")
                             await send_status("ton_daemon", f"working: too many tasks in epoch {blockchain_task.epoch}")
                             await asyncio.sleep(5)
@@ -235,10 +243,11 @@ async def main_fn(memory):
                         sign_created = int(datetime.utcnow().timestamp()) - 60
                         try:
                             current_epoch = int(datetime.utcnow().timestamp() // (60 * 60))
+                            from sqlalchemy import func
                             max_epoch_seqno = (
-                                session.query(func.max(BlockchainTask.seqno)).filter(
+                                (await session.execute(select(func.max(BlockchainTask.seqno)).where(
                                     BlockchainTask.epoch == current_epoch
-                                ).scalar() or 0
+                                ))).scalar() or 0
                             )
                             current_epoch_shift = 3_000_000 if current_epoch % 2 == 0 else 0
                             current_seqno = max_epoch_seqno + 1 + (current_epoch_shift if max_epoch_seqno == 0 else 0)
@@ -258,7 +267,7 @@ async def main_fn(memory):
                             )
                             query_boc = query['message'].to_boc(False)
                         except BaseException as e:
-                            make_log("TON", f"Error creating transfer message: {e}", level="error")
+                            make_log("TON", f"Error creating transfer message: {e}", level="error", rid=rid)
                             query_boc = begin_cell().end_cell().to_boc(False)
                         
                         blockchain_task.meta = {
@@ -266,10 +275,10 @@ async def main_fn(memory):
                             'sign_created': sign_created,
                             'signed_message': query_boc.hex(),
                         }
-                        session.commit()
+                        await session.commit()
-                        make_log("TON", f"Created signed message for task {blockchain_task.id}" + '\n' + traceback.format_exc(), level="info")
+                        make_log("TON", f"Created signed message for task {blockchain_task.id}" + '\n' + traceback.format_exc(), level="info", rid=rid)
                     except BaseException as e:
-                        make_log("TON", f"Error processing task {blockchain_task.id}: {e}" + '\n' + traceback.format_exc(), level="error")
+                        make_log("TON", f"Error processing task {blockchain_task.id}: {e}" + '\n' + traceback.format_exc(), level="error", rid=rid)
                         continue
 
                     await asyncio.sleep(1)
@@ -277,14 +286,17 @@ async def main_fn(memory):
             await asyncio.sleep(1)
             await send_status("ton_daemon", f"working (seqno={sw_seqno_value})")
         except BaseException as e:
-            make_log("TON", f"Error: {e}", level="error")
+            make_log("TON", f"Error: {e}", level="error", rid=locals().get('rid'))
             await asyncio.sleep(3)
+        finally:
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(None)
+            except BaseException:
+                pass
 
 # if __name__ == '__main__':
 #     loop = asyncio.get_event_loop()
 #     loop.run_until_complete(main())
 #     loop.close()
 
-
-
-

app/core/background/uploader_service.py

@@ -13,14 +13,26 @@ async def main_fn(memory):
     seqno = 0
     while True:
         try:
-            make_log("Uploader", "Service running", level="debug")
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
+            make_log("Uploader", f"Service running", level="debug", rid=rid)
             await uploader_loop()
             await asyncio.sleep(5)
             await send_status("uploader_daemon", f"working (seqno={seqno})")
             seqno += 1
         except BaseException as e:
-            make_log("Uploader", f"Error: {e}", level="error")
+            make_log("Uploader", f"Error: {e}", level="error", rid=locals().get('rid'))
             await asyncio.sleep(3)
+        finally:
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(None)
+            except BaseException:
+                pass
 
 # if __name__ == '__main__':
 #     loop = asyncio.get_event_loop()
@@ -28,5 +40,3 @@ async def main_fn(memory):
 #     loop.close()
 
 
-
-

app/core/content/content_id.py

@@ -1,3 +1,6 @@
+from base64 import b32decode
+from typing import Optional, Tuple
+
 from base58 import b58encode, b58decode
 
 from tonsdk.boc import begin_cell
@@ -12,25 +15,50 @@ from app.core._utils.string_binary import string_to_bytes_fixed_size, bytes_to_s
 # cid_v2#_ cid_version:uint8 content_sha256:uint256 *[Param]s = CIDv2;
 
 class ContentId:
+    """Unified abstraction for legacy ContentID and ENCF/IPFS CID strings."""
+
     def __init__(
         self,
-        version: int = None,
+        version: Optional[int] = None,
-        content_hash: bytes = None,  # only SHA256
+        content_hash: Optional[bytes] = None,  # only SHA256
-        onchain_index: int = None,
+        onchain_index: Optional[int] = None,
-        accept_type: str = None,
+        accept_type: Optional[str] = None,
-        encryption_key_sha256: bytes = None,
+        encryption_key_sha256: Optional[bytes] = None,
+        *,
+        raw_value: Optional[str] = None,
+        cid_format: Optional[str] = None,
+        multibase_prefix: Optional[str] = None,
+        multicodec: Optional[int] = None,
+        multihash_code: Optional[int] = 0x12,
+        multihash_length: Optional[int] = 32,
     ):
         self.version = version
         self.content_hash = content_hash
 
-        self.onchain_index = onchain_index or -1
+        self.onchain_index = onchain_index if onchain_index is not None else -1
         self.accept_type = accept_type
         self.encryption_key_sha256 = encryption_key_sha256
         if self.encryption_key_sha256:
             assert len(self.encryption_key_sha256) == 32, "Invalid encryption key length"
 
+        self._raw_value = raw_value
+        if cid_format:
+            self.cid_format = cid_format
+        else:
+            if self.version == 1:
+                self.cid_format = 'content_id_v1'
+            elif self.version == 2:
+                self.cid_format = 'content_id_v2'
+            else:
+                self.cid_format = 'content_id_v2'
+        self.multibase_prefix = multibase_prefix
+        self.multicodec = multicodec
+        self.multihash_code = multihash_code
+        self.multihash_length = multihash_length
+
     @property
     def content_hash_b58(self) -> str:
+        assert self.content_hash, "Content hash is not set"
         return b58encode(self.content_hash).decode()
 
     @property
@@ -38,6 +66,11 @@ class ContentId:
         return self.onchain_index if (not (self.onchain_index is None) and self.onchain_index >= 0) else None
 
     def serialize_v2(self, include_accept_type=False) -> str:
+        if self.cid_format == 'ipfs':
+            if self._raw_value:
+                return self._raw_value
+            return self._serialize_ipfs()
+
         cid_bin = (
             (2).to_bytes(1, 'big')  # cid version
             + self.content_hash
@@ -60,6 +93,8 @@ class ContentId:
         return b58encode(cid_bin).decode()
 
     def serialize_v1(self) -> str:
+        if self.cid_format == 'ipfs':
+            raise ValueError("Cannot serialize IPFS CID as ContentId v1")
         at_bin = string_to_bytes_fixed_size(self.accept_type, 15)
         assert len(self.content_hash) == 32, "Invalid hash length"
         if self.onchain_index < 0:
@@ -133,13 +168,31 @@ class ContentId:
 
     @classmethod
     def deserialize(cls, cid: str):
+        if not cid:
+            raise ValueError("Empty content id provided")
+
+        first_char = cid[0]
+        if first_char in ('b', 'B', 'z', 'Z'):
+            return cls.from_ipfs(cid)
+
+        try:
             cid_version = int.from_bytes(b58decode(cid)[0:1], 'big')
+        except Exception:
+            cid_version = None
+
         if cid_version == 1:
-            return cls.from_v1(cid)
+            obj = cls.from_v1(cid)
-        elif cid_version == 2:
+            obj._raw_value = cid
-            return cls.from_v2(cid)
+            return obj
-        else:
+        if cid_version == 2:
-            raise ValueError("Invalid cid version")
+            obj = cls.from_v2(cid)
+            obj._raw_value = cid
+            return obj
+
+        try:
+            return cls.from_ipfs(cid)
+        except Exception as exc:
+            raise ValueError(f"Invalid cid format: {exc}") from exc
 
     def json_format(self):
         return {
@@ -147,7 +200,130 @@ class ContentId:
             "content_hash": self.content_hash_b58,
             "onchain_index": self.safe_onchain_index,
             "accept_type": self.accept_type,
-            "encryption_key_sha256": b58encode(self.encryption_key_sha256).decode() if self.encryption_key_sha256 else None
+            "encryption_key_sha256": b58encode(self.encryption_key_sha256).decode() if self.encryption_key_sha256 else None,
+            "format": self.cid_format,
+            "raw": self.serialize_v2() if self.cid_format == 'ipfs' else None,
         }
 
+    # --- helpers for IPFS/ENCF CID handling ---------------------------------
 
+    @staticmethod
+    def _decode_multibase(cid_str: str) -> Tuple[bytes, Optional[str]]:
+        prefix = cid_str[0]
+        if prefix in ('b', 'B'):
+            payload = cid_str[1:]
+            padding = (-len(payload)) % 8
+            decoded = b32decode(payload.upper() + ('=' * padding), casefold=True)
+            return decoded, prefix.lower()
+        if prefix in ('z', 'Z'):
+            return b58decode(cid_str[1:]), prefix.lower()
+        # CIDv0 without explicit prefix
+        return b58decode(cid_str), None
+
+    @staticmethod
+    def _read_varint(data: bytes, offset: int) -> Tuple[int, int]:
+        result = 0
+        shift = 0
+        while True:
+            if offset >= len(data):
+                raise ValueError("truncated varint")
+            byte = data[offset]
+            offset += 1
+            result |= (byte & 0x7F) << shift
+            if not (byte & 0x80):
+                break
+            shift += 7
+            if shift > 63:
+                raise ValueError("varint overflow")
+        return result, offset
+
+    @classmethod
+    def from_ipfs(cls, cid: str):
+        cid = cid.strip()
+        payload, multibase_prefix = cls._decode_multibase(cid)
+
+        idx = 0
+        version: Optional[int] = None
+        codec: Optional[int] = None
+
+        if multibase_prefix is not None:
+            version, idx = cls._read_varint(payload, idx)
+            if version not in (0, 1):
+                raise ValueError(f"unsupported CID version: {version}")
+            if version == 1:
+                codec, idx = cls._read_varint(payload, idx)
+            else:
+                codec = 0x70  # dag-pb default for CIDv0
+        else:
+            # CIDv0 without explicit version/codec
+            version = 0
+            codec = 0x70
+
+        multihash_code, idx = cls._read_varint(payload, idx)
+        multihash_length, idx = cls._read_varint(payload, idx)
+        digest = payload[idx:idx + multihash_length]
+        if len(digest) != multihash_length:
+            raise ValueError("truncated multihash digest")
+        if multihash_length != 32:
+            raise ValueError("unsupported multihash length (expected 32 bytes)")
+        if multihash_code != 0x12:
+            raise ValueError(f"unsupported multihash code: {hex(multihash_code)}")
+
+        return cls(
+            version=version,
+            content_hash=digest,
+            onchain_index=None,
+            accept_type=None,
+            encryption_key_sha256=None,
+            raw_value=cid,
+            cid_format='ipfs',
+            multibase_prefix=multibase_prefix,
+            multicodec=codec,
+            multihash_code=multihash_code,
+            multihash_length=multihash_length,
+        )
+
+    def _serialize_ipfs(self) -> str:
+        if not self.content_hash:
+            raise ValueError("Cannot serialize IPFS CID without content hash")
+        if self.multibase_prefix is None:
+            # default to CIDv0 (base58btc) dag-pb
+            multihash = self._encode_varint(self.multihash_code or 0x12) + self._encode_varint(self.multihash_length or len(self.content_hash)) + self.content_hash
+            return b58encode(multihash).decode()
+
+        version_bytes = self._encode_varint(self.version or 1)
+        codec_bytes = b''
+        if (self.version or 1) == 1:
+            codec_bytes = self._encode_varint(self.multicodec or 0x70)
+
+        multihash = (
+            version_bytes +
+            codec_bytes +
+            self._encode_varint(self.multihash_code or 0x12) +
+            self._encode_varint(self.multihash_length or len(self.content_hash)) +
+            self.content_hash
+        )
+
+        if self.multibase_prefix == 'z':
+            return 'z' + b58encode(multihash).decode()
+        if self.multibase_prefix == 'b':
+            from base64 import b32encode
+            encoded = b32encode(multihash).decode().rstrip('=').lower()
+            return 'b' + encoded
+        # Fallback to base58btc without prefix
+        return b58encode(multihash).decode()
+
+    @staticmethod
+    def _encode_varint(value: int) -> bytes:
+        if value < 0:
+            raise ValueError("varint cannot encode negative values")
+        out = bytearray()
+        while True:
+            to_write = value & 0x7F
+            value >>= 7
+            if value:
+                out.append(to_write | 0x80)
+            else:
+                out.append(to_write)
+                break
+        return bytes(out)

app/core/content/utils.py

@@ -26,7 +26,9 @@ async def create_new_content(
     content_hash_bin = sha256(content_bin).digest()
     content_hash_b58 = b58encode(content_hash_bin).decode()
 
-    new_content = db_session.query(StoredContent).filter(StoredContent.hash == content_hash_b58).first()
+    from sqlalchemy import select
+    result = await db_session.execute(select(StoredContent).where(StoredContent.hash == content_hash_b58))
+    new_content = result.scalars().first()
     if new_content:
         return new_content, False
 
@@ -38,8 +40,9 @@ async def create_new_content(
 
     )
     db_session.add(new_content)
-    db_session.commit()
+    await db_session.commit()
-    new_content = db_session.query(StoredContent).filter(StoredContent.hash == content_hash_b58).first()
+    result = await db_session.execute(select(StoredContent).where(StoredContent.hash == content_hash_b58))
+    new_content = result.scalars().first()
     assert new_content, "Content not created (through utils)"
     content_filepath = os.path.join(UPLOADS_DIR, content_hash_b58)
     async with aiofiles.open(content_filepath, 'wb') as file:

app/core/crypto/aes_gcm_siv_stream.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import hmac
+import hashlib
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+from cryptography.hazmat.primitives.ciphers.aead import AESGCMSIV
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+SCHEME_AES_GCM_SIV = 0x01
+
+
+def _derive_nonce(salt: bytes, idx: int) -> bytes:
+    b = idx.to_bytes(8, 'big')
+    return hmac.new(salt, b, hashlib.sha256).digest()[:12]
+
+
+def build_header(chunk_bytes: int, salt: bytes) -> bytes:
+    assert 0 < chunk_bytes <= (1 << 31)
+    assert 1 <= len(salt) <= 255
+    # MAGIC(4) | ver(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt | reserved(5)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr += bytes([VERSION])
+    hdr += bytes([SCHEME_AES_GCM_SIV])
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr += bytes([len(salt)])
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """
+    Yield ENCF v1 stream using AES-GCM-SIV per chunk with deterministic nonces.
+    Frame: [p_len:4][cipher][tag(16)].
+    """
+    yield build_header(chunk_bytes, salt)
+    idx = 0
+    cipher = AESGCMSIV(key)
+
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        ct_and_tag = cipher.encrypt(nonce, block, associated_data=None)
+        # Split tag
+        tag = ct_and_tag[-16:]
+        ct = ct_and_tag[:-16]
+        yield struct.pack(">I", len(block))
+        yield ct
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """Parse ENCF v1 (AES-GCM-SIV) and write plaintext to out_path."""
+    import aiofiles
+    buf = bytearray()
+
+    async def _fill(n: int):
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # header minimal
+    await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    if version != 1 or scheme != SCHEME_AES_GCM_SIV:
+        raise ValueError("unsupported encf header")
+    chunk_bytes = struct.unpack(">I", bytes(buf[6:10]))[0]
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    salt = bytes(buf[11:11 + salt_len])
+    del buf[:hdr_len]
+
+    cipher = AESGCMSIV(key)
+
+    async with aiofiles.open(out_path, 'wb') as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            await _fill(4)
+            if len(buf) == 0:
+                break
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            ct = bytes(buf[:p_len])
+            tag = bytes(buf[p_len:p_len+TAG_LEN])
+            del buf[:p_len+TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            pt = cipher.decrypt(nonce, ct + tag, associated_data=None)
+            await out.write(pt)
+            idx += 1

app/core/crypto/aes_gcm_stream.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import hmac
+import hashlib
+import os
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+import aiofiles
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+SCHEME_AES_GCM = 0x03
+
+CHUNK_BYTES = int(os.getenv("CRYPTO_CHUNK_BYTES", "1048576"))
+
+
+def _derive_nonce(salt: bytes, idx: int) -> bytes:
+    """Derive a deterministic 12-byte nonce from salt and chunk index."""
+    if len(salt) < 12:
+        raise ValueError("salt must be at least 12 bytes")
+    idx_bytes = idx.to_bytes(8, "big")
+    return hmac.new(salt, idx_bytes, hashlib.sha256).digest()[:12]
+
+
+def build_header(chunk_bytes: int, salt: bytes) -> bytes:
+    if not (0 < chunk_bytes <= (1 << 31)):
+        raise ValueError("chunk_bytes must be between 1 and 2^31")
+    if not (1 <= len(salt) <= 255):
+        raise ValueError("salt length must be 1..255 bytes")
+    # MAGIC(4) | ver(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt | reserved(5 zeros)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr.append(VERSION)
+    hdr.append(SCHEME_AES_GCM)
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr.append(len(salt))
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """Yield ENCF v1 frames encrypted with AES-GCM."""
+    if len(key) not in (16, 24, 32):
+        raise ValueError("AES-GCM key must be 128, 192 or 256 bits long")
+    cipher = AESGCM(key)
+    yield build_header(chunk_bytes, salt)
+    idx = 0
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        ct = cipher.encrypt(nonce, block, associated_data=None)
+        tag = ct[-16:]
+        data = ct[:-16]
+        yield struct.pack(">I", len(block))
+        yield data
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """Parse ENCF v1 (AES-GCM) stream and write plaintext to `out_path`."""
+    if len(key) not in (16, 24, 32):
+        raise ValueError("AES-GCM key must be 128, 192 or 256 bits long")
+    cipher = AESGCM(key)
+    buf = bytearray()
+
+    async def _fill(n: int) -> None:
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # Parse header
+    await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    if version != VERSION or scheme != SCHEME_AES_GCM:
+        raise ValueError("unsupported ENCF header")
+    chunk_bytes = struct.unpack(">I", bytes(buf[6:10]))[0]
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    salt = bytes(buf[11:11 + salt_len])
+    del buf[:hdr_len]
+
+    async with aiofiles.open(out_path, "wb") as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            await _fill(4)
+            if len(buf) == 0:
+                break
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            ct = bytes(buf[:p_len])
+            tag = bytes(buf[p_len:p_len + TAG_LEN])
+            del buf[:p_len + TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            pt = cipher.decrypt(nonce, ct + tag, associated_data=None)
+            await out.write(pt)
+            idx += 1

app/core/crypto/aes_siv_stream.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import os
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+from Crypto.Cipher import AES
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+
+# Scheme codes
+SCHEME_AES_SIV = 0x02  # RFC5297 AES-SIV (CMAC-based)
+
+
+def build_header(chunk_bytes: int, salt: bytes, scheme: int = SCHEME_AES_SIV) -> bytes:
+    assert 0 < chunk_bytes <= (1 << 31)
+    assert 1 <= len(salt) <= 255
+    # Layout: MAGIC(4) | version(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt(N) | reserved(5 zeros)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr += bytes([VERSION])
+    hdr += bytes([scheme])
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr += bytes([len(salt)])
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def parse_header(buf: bytes) -> tuple[int, int, int, bytes, int]:
+    if len(buf) < 4 + 1 + 1 + 4 + 1:
+        raise ValueError("header too short")
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    chunk_bytes = struct.unpack(">I", buf[6:10])[0]
+    salt_len = buf[10]
+    needed = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    if len(buf) < needed:
+        raise ValueError("incomplete header")
+    salt = buf[11:11 + salt_len]
+    # reserved 5 bytes at the end ignored
+    return version, scheme, chunk_bytes, salt, needed
+
+
+def _ad(salt: bytes, idx: int) -> bytes:
+    return salt + struct.pack(">Q", idx)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """
+    Yield ENCF v1 stream bytes: [header] then for each chunk: [p_len:4][cipher][tag(16)].
+    Uses AES-SIV (RFC5297) with per-chunk associated data salt||index.
+    """
+    yield build_header(chunk_bytes, salt, SCHEME_AES_SIV)
+    idx = 0
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        siv = AES.new(key, AES.MODE_SIV)  # new object per message
+        siv.update(_ad(salt, idx))
+        ciph, tag = siv.encrypt_and_digest(block)
+        yield struct.pack(">I", len(block))
+        yield ciph
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """
+    Parse ENCF v1 stream from async byte iterator and write plaintext to out_path.
+    """
+    import aiofiles
+    from Crypto.Cipher import AES as _AES
+
+    buf = bytearray()
+
+    async def _fill(n: int):
+        """Ensure at least n bytes in buffer (or EOF)."""
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # Read and parse header
+    await _fill(4 + 1 + 1 + 4 + 1)  # minimal header
+    # Might still be incomplete if salt_len > 0; keep filling progressively
+    # First, get preliminary to know salt_len
+    if len(buf) < 11:
+        await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    version, scheme, chunk_bytes, salt, consumed = parse_header(bytes(buf))
+    del buf[:consumed]
+    if version != 1:
+        raise ValueError("unsupported ENCF version")
+    if scheme != SCHEME_AES_SIV:
+        raise ValueError("unsupported scheme")
+
+    async with aiofiles.open(out_path, 'wb') as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            # Need at least 4 bytes for p_len
+            await _fill(4)
+            if len(buf) == 0:
+                break  # EOF exactly on boundary
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            # Now need p_len + 16 bytes
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            c = bytes(buf[:p_len])
+            t = bytes(buf[p_len:p_len+TAG_LEN])
+            del buf[:p_len+TAG_LEN]
+            siv = _AES.new(key, _AES.MODE_SIV)
+            siv.update(_ad(salt, idx))
+            p = siv.decrypt_and_verify(c, t)
+            await out.write(p)
+            idx += 1
+

app/core/crypto/aesgcm_stream.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import os
+import hmac
+import hashlib
+from typing import BinaryIO, Iterator
+
+from Crypto.Cipher import AES
+
+
+CHUNK_BYTES = int(os.getenv("CRYPTO_CHUNK_BYTES", "1048576"))  # 1 MiB
+
+
+def _derive_nonce(salt: bytes, chunk_index: int) -> bytes:
+    """Derive a 12-byte GCM nonce deterministically from per-file salt and chunk index."""
+    idx = chunk_index.to_bytes(8, 'big')
+    digest = hmac.new(salt, idx, hashlib.sha256).digest()
+    return digest[:12]
+
+
+def encrypt_stream_aesgcm(src: BinaryIO, key: bytes, salt: bytes) -> Iterator[bytes]:
+    """
+    Read plaintext from src by CHUNK_BYTES, encrypt each chunk with AES-GCM using a
+    deterministic nonce derived from (salt, index). Yields bytes in framing: [C_i][TAG_i]...
+    Ciphertext length equals plaintext chunk length. Tag is 16 bytes.
+    """
+    assert len(key) in (16, 24, 32)
+    assert len(salt) >= 12
+    idx = 0
+    while True:
+        block = src.read(CHUNK_BYTES)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+        ciphertext, tag = cipher.encrypt_and_digest(block)
+        yield ciphertext
+        yield tag
+        idx += 1
+
+
+def decrypt_stream_aesgcm_iter(byte_iter: Iterator[bytes], key: bytes, salt: bytes) -> Iterator[bytes]:
+    """
+    Decrypt a stream that was produced by encrypt_stream_aesgcm.
+    Frame format: concatenation of [C_i][TAG_i] for each i, where |C_i| = CHUNK_BYTES and |TAG_i|=16.
+    We accept arbitrary chunking from the underlying iterator and reframe accordingly.
+    """
+    assert len(key) in (16, 24, 32)
+    buf = bytearray()
+    idx = 0
+    TAG_LEN = 16
+    def _try_yield():
+        nonlocal idx
+        out = []
+        while len(buf) >= CHUNK_BYTES + TAG_LEN:
+            c = bytes(buf[:CHUNK_BYTES])
+            t = bytes(buf[CHUNK_BYTES:CHUNK_BYTES+TAG_LEN])
+            del buf[:CHUNK_BYTES+TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+            try:
+                p = cipher.decrypt_and_verify(c, t)
+            except Exception as e:
+                raise ValueError(f"Decrypt failed at chunk {idx}: {e}")
+            out.append(p)
+            idx += 1
+        return out
+    for chunk in byte_iter:
+        if not chunk:
+            continue
+        buf.extend(chunk)
+        for p in _try_yield():
+            yield p
+    # At end, buffer must be empty
+    if len(buf) != 0:
+        raise ValueError("Trailing bytes in encrypted stream (incomplete frame)")
+

app/core/crypto/cli.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+import sys
+from typing import Optional
+
+from .aes_gcm_stream import CHUNK_BYTES, encrypt_file_to_encf
+from .encf_stream import decrypt_encf_auto
+from .keywrap import unwrap_dek, KeyWrapError
+
+
+def _normalize_base64(value: str) -> str:
+    padding = (-len(value)) % 4
+    if padding:
+        return value + "=" * padding
+    return value
+
+
+def _decode_key(value: str, fmt: str) -> bytes:
+    if fmt == "base64":
+        return base64.b64decode(_normalize_base64(value))
+    if fmt == "hex":
+        cleaned = value[2:] if value.lower().startswith("0x") else value
+        return bytes.fromhex(cleaned)
+    if fmt == "raw":
+        return value.encode()
+    raise ValueError(f"unsupported key format: {fmt}")
+
+
+def _decode_salt(value: str, fmt: str) -> bytes:
+    if fmt == "base64":
+        return base64.b64decode(_normalize_base64(value))
+    if fmt == "hex":
+        cleaned = value[2:] if value.lower().startswith("0x") else value
+        return bytes.fromhex(cleaned)
+    raise ValueError(f"unsupported salt format: {fmt}")
+
+
+async def _decrypt_file(input_path: str, key: bytes, output_path: str) -> None:
+    async def _aiter():
+        with open(input_path, "rb") as src:
+            while True:
+                chunk = src.read(65536)
+                if not chunk:
+                    break
+                yield chunk
+
+    await decrypt_encf_auto(_aiter(), key, output_path)
+
+
+def cmd_encrypt(args: argparse.Namespace) -> int:
+    key = _decode_key(args.key, args.key_format)
+    salt = _decode_salt(args.salt, args.salt_format) if args.salt else os.urandom(args.salt_bytes)
+    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+    with open(args.input, "rb") as src, open(args.output, "wb") as dst:
+        for chunk in encrypt_file_to_encf(src, key, args.chunk_bytes, salt):
+            dst.write(chunk)
+    # Emit JSON metadata with salt for convenience
+    meta = {
+        "salt_b64": base64.b64encode(salt).decode(),
+        "chunk_bytes": args.chunk_bytes,
+        "aead_scheme": "AES_GCM",
+    }
+    print(json.dumps(meta), file=sys.stdout)
+    return 0
+
+
+def cmd_decrypt(args: argparse.Namespace) -> int:
+    if bool(args.key) == bool(args.wrapped_key):
+        raise SystemExit("Provide exactly one of --key or --wrapped-key")
+    if args.wrapped_key:
+        try:
+            key = unwrap_dek(args.wrapped_key)
+        except KeyWrapError as exc:
+            raise SystemExit(f"Failed to unwrap key: {exc}") from exc
+    else:
+        key = _decode_key(args.key, args.key_format)
+    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+    asyncio.run(_decrypt_file(args.input, key, args.output))
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="python -m app.core.crypto.cli", description="ENCF AES-GCM helper")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    enc = sub.add_parser("encrypt", help="Encrypt file into ENCF v1 stream (AES-256-GCM)")
+    enc.add_argument("--input", required=True, help="Path to plaintext input file")
+    enc.add_argument("--output", required=True, help="Destination path for ENCF output")
+    enc.add_argument("--key", required=True, help="Encryption key")
+    enc.add_argument("--key-format", choices=["base64", "hex", "raw"], default="base64")
+    enc.add_argument("--salt", help="Salt in specified format; generates random if omitted")
+    enc.add_argument("--salt-format", choices=["base64", "hex"], default="base64")
+    enc.add_argument("--salt-bytes", type=int, default=16, help="Salt length when generated (default: 16)")
+    enc.add_argument("--chunk-bytes", type=int, default=CHUNK_BYTES, help="Plaintext chunk size (default from env)")
+    enc.set_defaults(func=cmd_encrypt)
+
+    dec = sub.add_parser("decrypt", help="Decrypt ENCF stream to plaintext")
+    dec.add_argument("--input", required=True, help="Path to ENCF input file")
+    dec.add_argument("--output", required=True, help="Destination path for decrypted file")
+    dec.add_argument("--key", help="Plaintext key")
+    dec.add_argument("--wrapped-key", help="Wrapped key produced by the backend")
+    dec.add_argument("--key-format", choices=["base64", "hex", "raw"], default="base64")
+    dec.set_defaults(func=cmd_decrypt)
+
+    return parser
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

app/core/crypto/encf_stream.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from typing import AsyncIterator
+
+from .aes_gcm_siv_stream import MAGIC as _MAGIC, VERSION as _VER, SCHEME_AES_GCM_SIV
+from .aes_gcm_siv_stream import decrypt_encf_to_file as _dec_gcmsiv
+from .aes_gcm_stream import SCHEME_AES_GCM, decrypt_encf_to_file as _dec_gcm
+from .aes_siv_stream import decrypt_encf_to_file as _dec_siv
+
+
+async def decrypt_encf_auto(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """
+    Detect scheme by peeking header, then delegate to proper decrypter.
+    Re-feeds the peeked bytes back to the chosen decoder.
+    """
+    buf = bytearray()
+
+    async def _fill(n: int):
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                ch = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if ch:
+                buf.extend(ch)
+
+    await _fill(11)
+    if buf[:4] != _MAGIC:
+        raise ValueError("bad magic")
+    scheme = buf[5]
+
+    async def _prepend_iter():
+        nonlocal buf
+        if buf:
+            yield bytes(buf)
+        async for ch in byte_iter:
+            yield ch
+
+    if scheme == SCHEME_AES_GCM_SIV:
+        await _dec_gcmsiv(_prepend_iter(), key, out_path)
+    elif scheme == SCHEME_AES_GCM:
+        await _dec_gcm(_prepend_iter(), key, out_path)
+    else:
+        await _dec_siv(_prepend_iter(), key, out_path)

app/core/crypto/keywrap.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import base64
+import os
+import threading
+from typing import Optional
+
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+
+_VERSION = 1
+_PREFIX_LEN = 1  # version byte
+_NONCE_LEN = 12
+_TAG_LEN = 16
+_valid_key_lengths = {16, 24, 32}
+_kek_lock = threading.Lock()
+_cached_kek: Optional[bytes] = None
+
+
+class KeyWrapError(RuntimeError):
+    """Raised when KEK configuration or unwrap operations fail."""
+
+
+def _normalize_base64(value: str) -> str:
+    v = value.strip()
+    missing = (-len(v)) % 4
+    if missing:
+        v += "=" * missing
+    return v
+
+
+def _decode_key_material(value: str) -> bytes:
+    v = value.strip()
+    if v.startswith("0x") or v.startswith("0X"):
+        v = v[2:]
+    try:
+        raw = bytes.fromhex(v)
+        if len(raw) in _valid_key_lengths:
+            return raw
+    except ValueError:
+        pass
+    try:
+        raw = base64.b64decode(_normalize_base64(value), validate=False)
+        if len(raw) in _valid_key_lengths:
+            return raw
+    except Exception as exc:  # noqa: BLE001 - we want to re-raise as KeyWrapError
+        raise KeyWrapError(f"invalid KEK encoding: {exc}") from exc
+    raise KeyWrapError("KEK must decode to 16/24/32 bytes")
+
+
+def _load_kek() -> bytes:
+    global _cached_kek
+    if _cached_kek is not None:
+        return _cached_kek
+    with _kek_lock:
+        if _cached_kek is not None:
+            return _cached_kek
+        env = os.getenv("CONTENT_KEY_KEK_B64") or os.getenv("CONTENT_KEY_KEK_HEX")
+        if not env:
+            raise KeyWrapError("CONTENT_KEY_KEK_B64 or CONTENT_KEY_KEK_HEX must be set")
+        kek = _decode_key_material(env)
+        if len(kek) != 32:
+            # Force 256-bit KEK for uniform security properties
+            raise KeyWrapError("KEK must be 32 bytes (256-bit) for AES-256-GCM")
+        _cached_kek = kek
+        return _cached_kek
+
+
+def wrap_dek(plaintext: bytes) -> str:
+    """Wrap a DEK (plaintext bytes) with AES-256-GCM; return base64 string."""
+    if not isinstance(plaintext, (bytes, bytearray)):
+        raise TypeError("plaintext must be bytes")
+    kek = _load_kek()
+    nonce = os.urandom(_NONCE_LEN)
+    cipher = AESGCM(kek)
+    ct = cipher.encrypt(nonce, bytes(plaintext), associated_data=None)
+    blob = bytes([_VERSION]) + nonce + ct
+    return base64.b64encode(blob).decode()
+
+
+def unwrap_dek(encoded: str) -> bytes:
+    """Unwrap DEK from base64 string. Supports legacy (raw base64 key) values."""
+    if not encoded:
+        raise KeyWrapError("empty key payload")
+    try:
+        raw = base64.b64decode(_normalize_base64(encoded), validate=False)
+    except Exception as exc:  # noqa: BLE001
+        raise KeyWrapError(f"invalid base64 payload: {exc}") from exc
+    if not raw:
+        raise KeyWrapError("decoded payload is empty")
+    version = raw[0]
+    if version == _VERSION:
+        if len(raw) < _PREFIX_LEN + _NONCE_LEN + _TAG_LEN + 1:
+            raise KeyWrapError("wrapped payload too short")
+        nonce = raw[_PREFIX_LEN:_PREFIX_LEN + _NONCE_LEN]
+        ciphertext = raw[_PREFIX_LEN + _NONCE_LEN:]
+        kek = _load_kek()
+        cipher = AESGCM(kek)
+        try:
+            return cipher.decrypt(nonce, ciphertext, associated_data=None)
+        except Exception as exc:  # noqa: BLE001
+            raise KeyWrapError(f"unwrap failed: {exc}") from exc
+    # Legacy fallback: value is raw DEK (no version prefix)
+    if len(raw) in {16, 24, 32}:
+        return raw
+    raise KeyWrapError("unknown key payload format")

app/core/crypto/x25519.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import base64
+from typing import Tuple
+
+from nacl import public, signing, bindings
+
+
+def ed25519_to_x25519(ed_seed: bytes) -> Tuple[public.PrivateKey, public.PublicKey]:
+    """Convert Ed25519 seed (32 bytes) to X25519 key pair using libsodium conversion."""
+    if len(ed_seed) != 32:
+        raise ValueError("ed25519 seed must be 32 bytes")
+    sk_ed = signing.SigningKey(ed_seed)
+    sk_ed_bytes = sk_ed._seed + sk_ed.verify_key._key  # 64-byte expanded sk (seed||pub)
+    sk_x_bytes = bindings.crypto_sign_ed25519_sk_to_curve25519(sk_ed_bytes)
+    pk_x_bytes = bindings.crypto_sign_ed25519_pk_to_curve25519(bytes(sk_ed.verify_key))
+    sk_x = public.PrivateKey(sk_x_bytes)
+    pk_x = public.PublicKey(pk_x_bytes)
+    return sk_x, pk_x
+
+
+def x25519_pub_b64_from_ed_seed(ed_seed: bytes) -> str:
+    _, pk = ed25519_to_x25519(ed_seed)
+    return base64.b64encode(bytes(pk)).decode()
+

app/core/ipfs_client.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import os
+from typing import AsyncIterator, Dict, Any, Iterable, Optional
+
+import httpx
+
+IPFS_API_URL = os.getenv("IPFS_API_URL", "http://ipfs:5001")
+IPFS_GATEWAY_URL = os.getenv("IPFS_GATEWAY_URL", "http://ipfs:8080")
+
+
+async def add_streamed_file(stream_iter: Iterable[bytes], filename: str = "file.bin", params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """
+    Stream-encrypt pipeline can pass a generator of bytes here. We stream to /api/v0/add as multipart.
+    Returns dict with fields from IPFS: { Name, Hash, Size }.
+    """
+    params = params or {}
+    # Ensure deterministic chunking and CIDv1
+    default_params = {
+        "cid-version": 1,
+        "raw-leaves": "true",
+        "chunker": f"size-{int(os.getenv('CRYPTO_CHUNK_BYTES', '1048576'))}",
+        "pin": "true",
+        "wrap-with-directory": "false",
+        "progress": "true",
+    }
+    q = {**default_params, **params}
+
+    class _StreamAdapter:
+        def __init__(self, iterable):
+            self._iter = iter(iterable)
+
+        def read(self, size=-1):
+            try:
+                return next(self._iter)
+            except StopIteration:
+                return b''
+
+    stream = _StreamAdapter(stream_iter)
+    async with httpx.AsyncClient(timeout=None) as client:
+        files = {"file": (filename, stream, "application/octet-stream")}
+        r = await client.post(f"{IPFS_API_URL}/api/v0/add", params=q, files=files)
+        r.raise_for_status()
+        # /add may emit NDJSON lines; most often single JSON
+        try:
+            data = r.json()
+        except Exception:
+            # Fallback: last non-empty line
+            last = [ln for ln in r.text.splitlines() if ln.strip()][-1]
+            import json as _json
+            data = _json.loads(last)
+        return data
+
+
+async def pin_add(cid: str, recursive: bool = True) -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=None) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/pin/add", params={"arg": cid, "recursive": str(recursive).lower(), "progress": "true"})
+        r.raise_for_status()
+        return r.json()
+
+
+async def pin_ls(cid: str) -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=30) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/pin/ls", params={"arg": cid})
+        r.raise_for_status()
+        return r.json()
+
+
+async def swarm_connect(multiaddr: str) -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=10) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/swarm/connect", params={"arg": multiaddr})
+        r.raise_for_status()
+        return r.json()
+
+
+async def cat_stream(cid: str):
+    client = httpx.AsyncClient(timeout=None)
+    try:
+        async with client.stream("POST", f"{IPFS_API_URL}/api/v0/cat", params={"arg": cid}) as r:
+            r.raise_for_status()
+            async for chunk in r.aiter_bytes():
+                if chunk:
+                    yield chunk
+    finally:
+        await client.aclose()
+
+
+async def find_providers(cid: str, max_results: int = 8):
+    """Query DHT for providers of a CID and return a list of {peer, addrs[]}.
+    Uses /api/v0/dht/findprovs and parses NDJSON stream.
+    """
+    out = []
+    async with httpx.AsyncClient(timeout=30) as client:
+        async with client.stream("POST", f"{IPFS_API_URL}/api/v0/dht/findprovs", params={"arg": cid}) as r:
+            r.raise_for_status()
+            async for line in r.aiter_lines():
+                if not line:
+                    continue
+                try:
+                    j = httpx.Response(200, text=line).json()
+                except Exception:
+                    import json as _json
+                    try:
+                        j = _json.loads(line)
+                    except Exception:
+                        continue
+                # Entries can include 'Extra' or 'Responses'
+                resps = j.get('Responses') or []
+                for resp in resps:
+                    peer = resp.get('ID') or resp.get('ID', '')
+                    addrs = resp.get('Addrs') or []
+                    if peer:
+                        out.append({"peer": peer, "addrs": addrs})
+                        if len(out) >= max_results:
+                            return out
+    return out
+
+
+async def bitswap_stat() -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=10) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/bitswap/stat")
+        r.raise_for_status()
+        return r.json()
+
+
+async def repo_stat() -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=10) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/repo/stat")
+        r.raise_for_status()
+        return r.json()
+
+
+async def id_info() -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=10) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/id")
+        r.raise_for_status()
+        return r.json()

app/core/log_context.py

@@ -0,0 +1,11 @@
+from contextvars import ContextVar
+
+# Correlation for HTTP requests
+ctx_session_id = ContextVar('ctx_session_id', default=None)
+ctx_user_id = ContextVar('ctx_user_id', default=None)
+ctx_method = ContextVar('ctx_method', default=None)
+ctx_path = ContextVar('ctx_path', default=None)
+ctx_remote = ContextVar('ctx_remote', default=None)
+
+# Correlation for background loop iterations
+ctx_rid = ContextVar('ctx_rid', default=None)

app/core/logger.py

@@ -1,5 +1,6 @@
 import os
 from app.core.projscale_logger import logger
+from app.core.log_context import ctx_session_id, ctx_user_id, ctx_method, ctx_path, ctx_remote, ctx_rid
 import logging
 
 LOG_LEVELS = {
@@ -17,6 +18,23 @@ def make_log(issuer, message, *args, level='INFO', **kwargs):
 
     assert level.upper() in LOG_LEVELS.keys(), f"Unknown log level"
     _log = getattr(logger, level.lower())
+    # Merge context variables if not explicitly provided
+    context_fields = {
+        'sid': kwargs.get('sid') or kwargs.get('session_id') or ctx_session_id.get(),
+        'user_id': kwargs.get('user_id') or ctx_user_id.get(),
+        'method': kwargs.get('method') or ctx_method.get(),
+        'path': kwargs.get('path') or ctx_path.get(),
+        'remote': kwargs.get('remote') or ctx_remote.get(),
+        'rid': kwargs.get('rid') or ctx_rid.get(),
+    }
+    # Only include non-empty context
+    for k, v in list(context_fields.items()):
+        if v is None:
+            context_fields.pop(k)
+    # Do not override provided kwargs; merge missing only
+    for k, v in context_fields.items():
+        kwargs.setdefault(k, v)
+
     log_buffer = f"[{issuer if not (issuer is None) else 'System'}] {message}"
     if args:
         log_buffer += f" | {args}"

app/core/models/__init__.py

@@ -13,3 +13,11 @@ from app.core.models.asset import Asset
 from app.core.models.my_network import KnownNode, KnownNodeIncident, RemoteContentIndex
 from app.core.models.promo import PromoAction
 from app.core.models.tasks import BlockchainTask
+from app.core.models.content_v3 import (
+    EncryptedContent,
+    ContentKey,
+    IpfsSync,
+    ContentDerivative,
+    ContentIndexItem,
+    KeyGrant,
+)

app/core/models/_config.py

@@ -1,6 +1,6 @@
 
 from app.core.models.base import AlchemyBase
-from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
+from sqlalchemy import Column, Integer, String, JSON, select
 
 
 class ServiceConfigValue(AlchemyBase):
@@ -8,7 +8,7 @@ class ServiceConfigValue(AlchemyBase):
 
     id = Column(Integer, autoincrement=True, primary_key=True)
     key = Column(String(128), nullable=False, unique=True)
-    packed_value = Column(JSON, nullable=False, default={})
+    packed_value = Column(JSON, nullable=False, default=dict)
 
     @property
     def value(self):
@@ -19,20 +19,18 @@ class ServiceConfig:
     def __init__(self, session):
         self.session = session
 
-    def get(self, key, default=None):
+    async def get(self, key, default=None):
-        result = self.session.query(ServiceConfigValue).filter(ServiceConfigValue.key == key).first()
+        result = (await self.session.execute(select(ServiceConfigValue).where(ServiceConfigValue.key == key))).scalars().first()
         return (result.value if result else None) or default
 
-    def set(self, key, value):
+    async def set(self, key, value):
-        config_value = self.session.query(ServiceConfigValue).filter(
+        result = (await self.session.execute(select(ServiceConfigValue).where(ServiceConfigValue.key == key))).scalars().first()
-            ServiceConfigValue.key == key
+        if not result:
-        ).first()
+            result = ServiceConfigValue(key=key)
-        if not config_value:
+            self.session.add(result)
-            config_value = ServiceConfigValue(key=key)
+            await self.session.commit()
-            self.session.add(config_value)
+            return await self.set(key, value)
-            self.session.commit()
-            return self.set(key, value)
 
-        config_value.packed_value = {'value': value}
+        result.packed_value = {'value': value}
-        self.session.commit()
+        await self.session.commit()
         return

app/core/models/_telegram/templates/player.py

@@ -1,4 +1,4 @@
-from sqlalchemy import and_
+from sqlalchemy import and_, select
 from app.core.models.node_storage import StoredContent
 from app.core.models.content.user_content import UserContent, UserAction
 from app.core.logger import make_log
@@ -11,6 +11,7 @@ import json
 import urllib
 
 from app.core.models.transaction import StarsInvoice
+from app.core._utils.share_links import build_content_links
 
 
 class PlayerTemplates:
@@ -21,79 +22,100 @@ class PlayerTemplates:
         template_kwargs = {}
         inline_keyboard_array = []
         text = ""
+        content_metadata_json = {}
+        description_block = ""
+        status_hint = ""
         if content:
             assert content.type.startswith('onchain/content'), "Invalid nodeStorage content type"
             cd_log = f"Content (SHA256: {content.hash}), Encrypted: {content.encrypted}, TelegramCID: {content.telegram_cid}. "
             if not content.encrypted:
                 local_content = content
             else:
-                local_content = db_session.query(StoredContent).filter_by(
+                local_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == content.decrypted_content_id))).scalars().first()
-                    id=content.decrypted_content_id
-                ).first()
                 # TODO: add check decrypted_content by .format_json()['content_cid']
                 if local_content:
                     cd_log += f"Decrypted: {local_content.hash}. "
                 else:
                     cd_log += "Can't decrypt content. "
+        user_wallet_address = await self.user.wallet_address_async(self.db_session)
+        user_existing_license = (await self.db_session.execute(select(UserContent).where(
+            and_(UserContent.user_id == self.user.id, UserContent.content_id == content.id)
+        ))).scalars().first()
 
-        user_wallet_address = self.user.wallet_address(self.db_session)
+        content_meta = content.json_format() if content else {}
-        user_existing_license = self.db_session.query(UserContent).filter_by(user_id=self.user.id, content_id=content.id).first()
-
         if local_content:
-            content_meta = content.json_format()
+            make_log("TG-Player", f"Content meta: {content_meta}. Local content meta: {local_content.json_format()}. ")
-            local_content_meta = local_content.json_format()
-            make_log("TG-Player", f"Content meta: {content_meta}. Local content meta: {local_content_meta}. ")
             try:
                 content_type, content_encoding = "audio", "aac"
             except:
                 content_type, content_encoding = 'application', 'x-binary'
 
-            content_metadata = StoredContent.from_cid(db_session, content_meta.get('metadata_cid') or None)
+        metadata_cid = content_meta.get('metadata_cid') if content_meta else None
+        if metadata_cid:
+            try:
+                content_metadata = await StoredContent.from_cid_async(db_session, metadata_cid)
                 with open(content_metadata.filepath, 'r') as f:
                     content_metadata_json = json.loads(f.read())
+            except BaseException as e:
+                make_log("TG-Player", f"Can't get metadata content: {e}", level='warning')
 
         try:
-                cover_content = StoredContent.from_cid(self.db_session, content_meta.get('cover_cid') or None)
+            cover_content = await StoredContent.from_cid_async(self.db_session, content_meta.get('cover_cid') if content_meta else None)
             cd_log += f"Cover content: {cover_content.cid.serialize_v2()}. "
         except BaseException as e:
             cd_log += f"Can't get cover content: {e}. "
             cover_content = None
 
+        share_target = user_existing_license.onchain_address if user_existing_license else content.cid.serialize_v2()
+        ref_id = (self.user.meta or {}).get('ref_id')
+        if not ref_id:
+            ref_id = self.user.ensure_ref_id()
+            if self.db_session:
+                await self.db_session.commit()
+
+        _, startapp_url, web_app_url = build_content_links(
+            share_target,
+            ref_id,
+            project_host=PROJECT_HOST,
+            bot_username=CLIENT_TELEGRAM_BOT_USERNAME
+        )
+
         content_share_link = {
             'text': self.user.translated('p_shareLinkContext').format(title=content_metadata_json.get('name', "")),
-                'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}/content?startapp={content.cid.serialize_v2()}"
+            'url': startapp_url,
+            'web_url': web_app_url,
+            'ref_id': ref_id
         }
-            if user_existing_license:
-                content_share_link['url'] = f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}/content?startapp={user_existing_license.onchain_address}"
 
         if cover_content:
             template_kwargs['photo'] = URLInputFile(cover_content.web_url)
 
         if not local_content:
-            text = self.user.translated('p_playerContext_unsupportedContent').format(
+            status_hint = self.user.translated('p_playerContext_contentNotReady')
-                content_type=content_type,
-                content_encoding=content_encoding
-            )
-            inline_keyboard_array = []
-            extra_buttons = []
-        else:
-            content_hashtags = content_metadata_json.get('description').strip()
-            if content_hashtags:
-                content_hashtags += '\n'
 
-            text = f"""<b>{content_metadata_json.get('name', 'Unnamed')}</b>
+        description = (content_metadata_json.get('description') or '').strip()
-{content_hashtags}
+        if description:
-Этот контент был загружен в MY
+            description_block = f"{description}\n"
+
+        title = content_metadata_json.get('name') or (local_content.filename if local_content else None) or (content.filename if content else None) or content.cid.serialize_v2()
+
+        status_block = f"{status_hint}\n" if status_hint else ""
+
+        text = f"""<b>{title}</b>
+{description_block}{status_block}Этот контент был загружен в MY
 \t/ p2p content market /
 <blockquote><a href="{content_share_link['url']}">🔴 «открыть в MY»</a></blockquote>"""
 
         make_log("TG-Player", f"Send content {content_type} ({content_encoding}) to chat {self._chat_id}. {cd_log}")
-        for kmsg in self.db_session.query(KnownTelegramMessage).filter_by(
+        kmsgs = (await self.db_session.execute(select(KnownTelegramMessage).where(
-            content_id=content.id,
+            and_(
-            chat_id=self._chat_id,
+                KnownTelegramMessage.content_id == content.id,
-            type=f'content/{content_type}',
+                KnownTelegramMessage.chat_id == self._chat_id,
-            deleted=False
+                KnownTelegramMessage.type == f'content/{content_type}',
-        ).all():
+                KnownTelegramMessage.deleted == False
+            )
+        ))).scalars().all()
+        for kmsg in kmsgs:
             await self.delete_message(kmsg.message_id)
 
         r = await tg_process_template(

app/core/models/_telegram/wrapped_bot.py

@@ -1,7 +1,7 @@
 from aiogram import Bot, types
 from datetime import datetime, timedelta
 
-from sqlalchemy import and_
+from sqlalchemy import and_, select
 
 from app.core.logger import make_log
 from app.core.models.messages import KnownTelegramMessage
@@ -46,14 +46,15 @@ class Wrapped_CBotChat(T, PlayerTemplates):
         if self.db_session:
             if message_type == 'common':
                 ci = 0
-                for oc_msg in self.db_session.query(KnownTelegramMessage).filter(
+                result = await self.db_session.execute(select(KnownTelegramMessage).where(
                     and_(
                         KnownTelegramMessage.type == 'common',
                         KnownTelegramMessage.bot_id == self.bot_id,
                         KnownTelegramMessage.chat_id == self._chat_id,
                         KnownTelegramMessage.deleted == False
                     )
-                ).all():
+                ))
+                for oc_msg in result.scalars().all():
                     make_log(self, f"Delete old message {oc_msg.message_id} {oc_msg.type} {oc_msg.bot_id} {oc_msg.chat_id}")
                     await self.delete_message(oc_msg.message_id)
                     ci += 1
@@ -75,7 +76,7 @@ class Wrapped_CBotChat(T, PlayerTemplates):
                         content_id=content_id
                     )
                 )
-                self.db_session.commit()
+                await self.db_session.commit()
             else:
                 make_log(self, f"Unknown result type: {type(result)}", level='warning')
 
@@ -127,14 +128,16 @@ class Wrapped_CBotChat(T, PlayerTemplates):
                 message_id
             )):
                 if self.db_session:
-                    known_message = self.db_session.query(KnownTelegramMessage).filter(
+                    known_message = (await self.db_session.execute(select(KnownTelegramMessage).where(
+                        and_(
                             KnownTelegramMessage.bot_id == self.bot_id,
                             KnownTelegramMessage.chat_id == self._chat_id,
                             KnownTelegramMessage.message_id == message_id
-                    ).first()
+                        )
+                    ))).scalars().first()
                     if known_message:
                         known_message.deleted = True
-                        self.db_session.commit()
+                        await self.db_session.commit()
         except Exception as e:
             make_log(self, f"Error deleting message {self._chat_id}/{message_id}. Error: {e}", level='warning')
             return None

app/core/models/asset.py

@@ -1,5 +1,6 @@
 from sqlalchemy import Column, Integer, String, DateTime, JSON, Boolean
 from sqlalchemy.orm import relationship
+from datetime import datetime
 
 from app.core._defaults import DEFAULT_ASSET_INITOBJ
 from app.core.models.base import AlchemyBase
@@ -15,10 +16,10 @@ class Asset(AlchemyBase):
 
     network = Column(String(32), nullable=True)
     address = Column(String(1024), nullable=True)
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
-    rates = Column(JSON, nullable=False, default={})
+    rates = Column(JSON, nullable=False, default=dict)
 
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
     is_active = Column(Boolean, nullable=False, default=True)
 
     balances = relationship('UserBalance', back_populates='asset')
@@ -29,22 +30,21 @@ class Asset(AlchemyBase):
         AlchemyBase.metadata.create_all(engine)
 
     @classmethod
-    def find(cls, session, **kwargs):
+    async def find_async(cls, session, **kwargs):
+        from sqlalchemy import select, func
         if 'symbol' in kwargs:
             kwargs['symbol'] = kwargs['symbol'].upper()
 
-        result = session.query(cls).filter_by(**kwargs)
+        result = await session.execute(select(cls).filter_by(**kwargs))
-        results_count = result.count()
+        row = result.scalars().first()
-        if results_count == 0:
+        if row:
-            any_count = session.query(cls).count()
+            return row
+
+        any_count = (await session.execute(select(func.count()).select_from(cls))).scalar() or 0
         if any_count == 0:
             init_asset = cls(**DEFAULT_ASSET_INITOBJ)
             session.add(init_asset)
-                session.commit()
+            await session.commit()
-                return cls.find(session, **kwargs)
+            return await cls.find_async(session, **kwargs)
 
         raise Exception(f"Asset not found: {kwargs}")
-        elif results_count == 1:
-            return result.first()
-        else:
-            raise Exception(f"Multiple assets found: {results_count}")

app/core/models/content/indexation_mixins.py

@@ -1,7 +1,9 @@
+import os
 import traceback
 
 import base58
-from sqlalchemy import and_
+from sqlalchemy import select
+from datetime import datetime
 
 from app.core.logger import make_log
 from app.core.models import StoredContent
@@ -42,6 +44,9 @@ class NodeStorageIndexationMixin:
     pass # async def fetch_onchain_metadata(self):
 
 
+MIN_ONCHAIN_INDEX = int(os.getenv("MIN_ONCHAIN_INDEX", "8"))
+
+
 class UserContentIndexationMixin:
     async def sync_with_chain(self, db_session):
         errored = False
@@ -54,16 +59,30 @@ class UserContentIndexationMixin:
                 cc_indexator_data = unpack_item_indexator_data(cc_indexator_result)
                 assert cc_indexator_data['type'] == 1, "Type is not a content"
                 assert cc_indexator_data['address'] == self.onchain_address, "Address is not equal"
+                license_type = cc_indexator_data.get('license_type')
+                if cc_indexator_data['index'] < MIN_ONCHAIN_INDEX and (license_type is None or license_type == 0):
+                    make_log(
+                        "UserContent",
+                        f"Skip license {self.onchain_address} with index {cc_indexator_data['index']} < MIN_ONCHAIN_INDEX={MIN_ONCHAIN_INDEX}",
+                        level="info"
+                    )
+                    self.type = 'nft/ignored'
+                    self.content_id = None
+                    self.updated = datetime.utcnow()
+                    await db_session.commit()
+                    return
                 values_slice = cc_indexator_data['values'].begin_parse()
                 content_hash_b58 = base58.b58encode(bytes.fromhex(hex(values_slice.read_uint(256))[2:])).decode()
                 make_log("UserContent", f"License ({self.onchain_address}) content hash: {content_hash_b58}", level="info")
-                stored_content = db_session.query(StoredContent).filter(
+                stored_content = (await db_session.execute(select(StoredContent).where(
-                    and_(
+                    StoredContent.hash == content_hash_b58
-                        StoredContent.type == 'onchain/content',
+                ))).scalars().first()
-                        StoredContent.hash == content_hash_b58,
+                if not stored_content:
-
+                    raise AssertionError(f"Stored content not found for hash={content_hash_b58}")
-                    )
+                if not (stored_content.type or '').startswith('onchain/content'):
-                ).first()
+                    stored_content.type = 'onchain/content' if stored_content.key_id else 'onchain/content_unknown'
+                    stored_content.onchain_index = stored_content.onchain_index or cc_indexator_data['index']
+                    stored_content.owner_address = stored_content.owner_address or cc_indexator_data['owner_address']
                 trusted_cop_address_result = await toncenter.run_get_method(stored_content.meta['item_address'], 'get_nft_address_by_index', [['num', cc_indexator_data['index']]])
                 assert trusted_cop_address_result.get('exit_code', -1) == 0, "Trusted cop address error"
                 trusted_cop_address = Cell.one_from_boc(b64decode(trusted_cop_address_result['stack'][0][1]['bytes'])).begin_parse().read_msg_addr().to_string(1, 1, 1)
@@ -72,7 +91,8 @@ class UserContentIndexationMixin:
                 self.owner_address = cc_indexator_data['owner_address']
                 self.type = 'nft/listen'
                 self.content_id = stored_content.id
-                db_session.commit()
+                self.meta = {**(self.meta or {}), 'license_type': license_type}
+                await db_session.commit()
             except BaseException as e:
                 errored = True
                 make_log("UserContent", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
@@ -80,7 +100,4 @@ class UserContentIndexationMixin:
         if errored is True:
             self.type = 'nft/unknown'
             self.content_id = None
-            db_session.commit()
+            await db_session.commit()
-
-
-

app/core/models/content/user_content.py

@@ -3,6 +3,7 @@ from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime
 from sqlalchemy.orm import relationship
 from app.core.models.base import AlchemyBase
 from app.core.models.content.indexation_mixins import UserContentIndexationMixin
+from datetime import datetime
 
 
 class UserContent(AlchemyBase, UserContentIndexationMixin):
@@ -14,12 +15,12 @@ class UserContent(AlchemyBase, UserContentIndexationMixin):
     owner_address = Column(String(1024), nullable=True)
     code_hash = Column(String(128), nullable=True)
     data_hash = Column(String(128), nullable=True)
-    updated = Column(DateTime, nullable=False, default=0)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
 
     content_id = Column(Integer, ForeignKey('node_storage.id'), nullable=True)
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
     user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
     wallet_connection_id = Column(Integer, ForeignKey('wallet_connections.id'), nullable=True)
     status = Column(String(64), nullable=False, default='active')  # 'transaction_requested'
@@ -41,9 +42,8 @@ class UserAction(AlchemyBase):
     to_address = Column(String(1024), nullable=True)
     from_address = Column(String(1024), nullable=True)
     status = Column(String(128), nullable=True)
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     user = relationship('User', uselist=False, foreign_keys=[user_id])
     content = relationship('StoredContent', uselist=False, foreign_keys=[content_id])
-

app/core/models/content_v3.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+from datetime import datetime
+from sqlalchemy import Column, BigInteger, Integer, String, DateTime, JSON, Boolean, ForeignKey
+from sqlalchemy.orm import relationship
+
+from .base import AlchemyBase
+
+
+class EncryptedContent(AlchemyBase):
+    __tablename__ = 'encrypted_contents'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    # CID of encrypted source stored in IPFS (CIDv1 base32)
+    encrypted_cid = Column(String(128), nullable=False, unique=True)
+
+    # Public metadata
+    title = Column(String(512), nullable=False)
+    description = Column(String(4096), nullable=True)
+    content_type = Column(String(64), nullable=False)  # e.g. audio/flac, video/mp4, application/octet-stream
+
+    # Sizes
+    enc_size_bytes = Column(BigInteger, nullable=True)
+    plain_size_bytes = Column(BigInteger, nullable=True)
+
+    # Preview flags and config (all preview params live here, not in derivatives)
+    preview_enabled = Column(Boolean, nullable=False, default=False)
+    preview_conf = Column(JSON, nullable=False, default=dict)
+
+    # Crypto parameters (fixed per network)
+    aead_scheme = Column(String(32), nullable=False, default='AES_GCM')
+    chunk_bytes = Column(Integer, nullable=False, default=1048576)
+    salt_b64 = Column(String(64), nullable=True)  # per-content salt used for nonce derivation
+
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class ContentKey(AlchemyBase):
+    __tablename__ = 'content_keys'
+
+    content_id = Column(Integer, ForeignKey('encrypted_contents.id'), primary_key=True)
+    key_ciphertext_b64 = Column(String(512), nullable=False)
+    key_fingerprint = Column(String(128), nullable=False)
+    issuer_node_id = Column(String(128), nullable=False)
+    allow_auto_grant = Column(Boolean, nullable=False, default=True)
+    lease_expires_at = Column(DateTime, nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+
+    content = relationship('EncryptedContent', uselist=False, foreign_keys=[content_id])
+
+
+class IpfsSync(AlchemyBase):
+    __tablename__ = 'ipfs_sync'
+
+    content_id = Column(Integer, ForeignKey('encrypted_contents.id'), primary_key=True)
+    pin_state = Column(String(32), nullable=False, default='pinned')  # not_pinned|queued|pinning|pinned|failed
+    pin_error = Column(String(1024), nullable=True)
+    bytes_total = Column(BigInteger, nullable=True)
+    bytes_fetched = Column(BigInteger, nullable=True)
+    providers_cache = Column(JSON, nullable=False, default=list)
+    first_seen_at = Column(DateTime, nullable=True)
+    pinned_at = Column(DateTime, nullable=True)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    content = relationship('EncryptedContent', uselist=False, foreign_keys=[content_id])
+
+
+class ContentDerivative(AlchemyBase):
+    __tablename__ = 'content_derivatives'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    content_id = Column(Integer, ForeignKey('encrypted_contents.id'), nullable=False)
+    kind = Column(String(64), nullable=False)  # decrypted_high|decrypted_low|decrypted_thumbnail|decrypted_preview
+    interval_start_ms = Column(Integer, nullable=True)
+    interval_end_ms = Column(Integer, nullable=True)
+    local_path = Column(String(1024), nullable=False)
+    content_type = Column(String(64), nullable=True)
+    size_bytes = Column(BigInteger, nullable=True)
+    status = Column(String(32), nullable=False, default='pending')  # pending|processing|ready|failed
+    error = Column(String(1024), nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    last_access_at = Column(DateTime, nullable=True)
+
+    content = relationship('EncryptedContent', uselist=False, foreign_keys=[content_id])
+
+
+class ContentIndexItem(AlchemyBase):
+    __tablename__ = 'content_index_items'
+
+    encrypted_cid = Column(String(128), primary_key=True)
+    payload = Column(JSON, nullable=False, default=dict)
+    sig = Column(String(512), nullable=False)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class KeyGrant(AlchemyBase):
+    __tablename__ = 'key_grants'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    encrypted_cid = Column(String(128), nullable=False)
+    issuer_node_id = Column(String(128), nullable=False)
+    to_node_id = Column(String(128), nullable=False)
+    sealed_key_b64 = Column(String(1024), nullable=False)
+    aead_scheme = Column(String(32), nullable=False)
+    chunk_bytes = Column(Integer, nullable=False)
+    constraints = Column(JSON, nullable=False, default=dict)
+    issued_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    sig = Column(String(512), nullable=False)
+
+
+class UploadSession(AlchemyBase):
+    __tablename__ = 'upload_sessions'
+
+    id = Column(String(128), primary_key=True)  # tus Upload.ID
+    filename = Column(String(512), nullable=True)
+    size_bytes = Column(BigInteger, nullable=True)
+    state = Column(String(32), nullable=False, default='uploading')  # uploading|processing|pinned|failed
+    encrypted_cid = Column(String(128), nullable=True)
+    storage_path = Column(String(1024), nullable=True)
+    error = Column(String(1024), nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)

app/core/models/keys.py

@@ -1,5 +1,6 @@
 from base58 import b58decode
 from sqlalchemy import Column, Integer, String, DateTime, JSON
+from datetime import datetime
 
 from .base import AlchemyBase
 
@@ -15,12 +16,12 @@ class KnownKey(AlchemyBase):
     public_key_hash = Column(String(64), nullable=False, unique=True)  # base58
 
     algo = Column(String(32), nullable=True, default=None)
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
     # {
     #   "I_user_id": TRUSTED_USER_ID,
     # }
 
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     # stored_content = relationship('StoredContent', back_populates='key')
 

app/core/models/memory.py

@@ -42,6 +42,10 @@ class Memory:
         self._telegram_bot = Bot(TELEGRAM_API_KEY)
         self._client_telegram_bot = Bot(CLIENT_TELEGRAM_API_KEY)
 
+        # Network handshake guards
+        self._handshake_rl = {"minute": 0, "counts": {}}
+        self._handshake_nonces = {}
+
     @asynccontextmanager
     async def transaction(self, desc=""):
         make_log("Memory.transaction", f"Starting transaction; {desc}", level='debug')
@@ -77,4 +81,3 @@ class Memory:
 
             self._execute_queue.append([_fn, args, kwargs])
 
-

app/core/models/my_network.py

@@ -12,9 +12,9 @@ class KnownNode(AlchemyBase):
     public_key = Column(String(256), nullable=False)
     codebase_hash = Column(String(512), nullable=True)  # Node software version
     reputation = Column(Integer, nullable=False, default=0)
-    last_sync = Column(DateTime, nullable=False, default=datetime.now)
+    last_sync = Column(DateTime, nullable=False, default=datetime.utcnow)
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
-    located_at = Column(DateTime, nullable=False, default=datetime.now)
+    located_at = Column(DateTime, nullable=False, default=datetime.utcnow)
 
 
 class KnownNodeIncident(AlchemyBase):
@@ -28,7 +28,7 @@ class KnownNodeIncident(AlchemyBase):
     severity = Column(Integer, nullable=False, default=1)  # Severity level (1-low to 5-critical)
     resolved = Column(Boolean, nullable=False, default=False)  # Whether the incident has been resolved
     resolved_at = Column(DateTime, nullable=True)  # Timestamp when the incident was resolved
-    meta = Column(JSON, nullable=False, default={})  # Additional metadata if needed
+    meta = Column(JSON, nullable=False, default=dict)  # Additional metadata if needed
 
 
 class RemoteContentIndex(AlchemyBase):
@@ -41,7 +41,6 @@ class RemoteContentIndex(AlchemyBase):
     decrypted_hash = Column(String(128), nullable=True)  # Decrypted content hash, available once permission is granted
     ton_address = Column(String(128), nullable=True)  # TON network address for the content
     onchain_index = Column(Integer, nullable=True)  # Onchain index or reference on a blockchain
-    meta = Column(JSON, nullable=False, default={})  # Additional metadata for flexible content description
+    meta = Column(JSON, nullable=False, default=dict)  # Additional metadata for flexible content description
     last_updated = Column(DateTime, nullable=False, default=datetime.utcnow)  # Timestamp of the last update
     created_at = Column(DateTime, nullable=False, default=datetime.utcnow)  # Record creation timestamp
-

app/core/models/node_storage.py

@@ -25,7 +25,8 @@ class StoredContent(AlchemyBase, AudioContentMixin):
 
     status = Column(String(32), nullable=True)
     filename = Column(String(1024), nullable=False)
-    meta = Column(JSON, nullable=False, default={})
+    # Use a factory for JSON default to avoid shared mutable dict
+    meta = Column(JSON, nullable=False, default=dict)
 
     user_id = Column(Integer, ForeignKey('users.id'), nullable=True)
     owner_address = Column(String(1024), nullable=True)
@@ -35,9 +36,11 @@ class StoredContent(AlchemyBase, AudioContentMixin):
     telegram_cid = Column(String(1024), nullable=True)
 
     codebase_version = Column(Integer, nullable=True)
-    created = Column(DateTime, nullable=False, default=0)
+    # Use proper datetime defaults; updated also auto-updates on change
-    updated = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
-    disabled = Column(DateTime, nullable=False, default=0)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+    # Timestamp of when content was disabled; None means active
+    disabled = Column(DateTime, nullable=True, default=None)
     disabled_by = Column(Integer, ForeignKey('users.id'), nullable=True, default=None)
 
     encrypted = Column(Boolean, nullable=False, default=False)
@@ -50,6 +53,11 @@ class StoredContent(AlchemyBase, AudioContentMixin):
 
     @property
     def cid(self) -> ContentId:
+        if self.content_id:
+            try:
+                return ContentId.deserialize(self.content_id)
+            except Exception as exc:
+                make_log("StoredContent", f"Failed to deserialize stored content_id '{self.content_id}': {exc}", level='warning')
         return ContentId(
             content_hash=b58decode(self.hash),
             onchain_index=self.onchain_index,
@@ -96,6 +104,30 @@ class StoredContent(AlchemyBase, AudioContentMixin):
             make_log("NodeStorage.open_content", f"Can't open content: {self.id} {e}", level='warning')
             raise e
 
+    async def open_content_async(self, db_session, content_type=None):
+        from sqlalchemy import select
+        try:
+            decrypted_content = self if not self.encrypted else None
+            encrypted_content = self if self.encrypted else None
+            if not decrypted_content:
+                decrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == self.decrypted_content_id))).scalars().first()
+            else:
+                encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.decrypted_content_id == self.id))).scalars().first()
+
+            assert decrypted_content, "Can't get decrypted content"
+            assert encrypted_content, "Can't get encrypted content"
+            _ct = content_type or decrypted_content.json_format()['content_type']
+            content_type = _ct.split('/')[0] if _ct else 'application'
+
+            return {
+                'encrypted_content': encrypted_content,
+                'decrypted_content': decrypted_content,
+                'content_type': content_type or 'application/x-binary'
+            }
+        except BaseException as e:
+            make_log("NodeStorage.open_content_async", f"Can't open content: {self.id} {e}", level='warning')
+            raise e
+
     def json_format(self):
         extra_fields = {}
         if self.type.startswith('local'):
@@ -144,6 +176,16 @@ class StoredContent(AlchemyBase, AudioContentMixin):
             with open(metadata_content.filepath, 'r') as f:
                 return json.loads(f.read())
 
+    async def metadata_json_async(self, db_session):
+        metadata_cid = self.meta.get('metadata_cid')
+        if not metadata_cid:
+            return None
+        metadata_content = await StoredContent.from_cid_async(db_session, metadata_cid)
+        import aiofiles
+        async with aiofiles.open(metadata_content.filepath, 'r') as f:
+            data = await f.read()
+        return json.loads(data)
+
     @classmethod
     def from_cid(cls, db_session, content_id):
         if isinstance(content_id, str):
@@ -155,3 +197,15 @@ class StoredContent(AlchemyBase, AudioContentMixin):
         assert content, "Content not found"
         return content
 
+    @classmethod
+    async def from_cid_async(cls, db_session, content_id):
+        from sqlalchemy import select
+        if isinstance(content_id, str):
+            cid = ContentId.deserialize(content_id)
+        else:
+            cid = content_id
+
+        result = await db_session.execute(select(StoredContent).where(StoredContent.hash == cid.content_hash_b58))
+        content = result.scalars().first()
+        assert content, "Content not found"
+        return content

app/core/models/promo.py

@@ -13,4 +13,4 @@ class PromoAction(AlchemyBase):
     action_type = Column(String(64), nullable=False)  # Type of action, e.g., 'referral', 'discount'
     action_ref = Column(String(512), nullable=False)  # Reference to the action, e.g., promo code
 
-    created = Column(DateTime, nullable=False, default=datetime.now)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)

app/core/models/tasks.py

@@ -15,11 +15,11 @@ class BlockchainTask(AlchemyBase):
     epoch = Column(Integer, nullable=True)
     seqno = Column(Integer, nullable=True)
 
-    created = Column(DateTime, nullable=False, default=datetime.now)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
-    updated = Column(DateTime, nullable=False, default=datetime.now)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
 
     user_id = Column(Integer, ForeignKey('users.id'), nullable=True)
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
     status = Column(String(256), nullable=False)
 
     transaction_hash = Column(String(1024), nullable=True)

app/core/models/transaction.py

@@ -13,8 +13,8 @@ class UserBalance(AlchemyBase):
     asset_id = Column(Integer, ForeignKey('assets.id'), nullable=False)
     balance = Column(Float, nullable=False, default=0)
 
-    updated = Column(DateTime, nullable=False, default=0)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     user = relationship('User', uselist=False, foreign_keys=[user_id], back_populates='balances')
     asset = relationship('Asset', uselist=False, foreign_keys=[asset_id], back_populates='balances')
@@ -32,7 +32,7 @@ class InternalTransaction(AlchemyBase):
     spent_transaction_id = Column(Integer, ForeignKey('internal_transactions.id'), nullable=True)
     type = Column(String(256), nullable=False, default="NOT_SPECIFIED")
 
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     user = relationship('User', uselist=False, back_populates='internal_transactions', foreign_keys=[user_id])
     asset = relationship('Asset', uselist=False, foreign_keys=[asset_id])

app/core/models/user/__init__.py

@@ -9,6 +9,10 @@ from app.core.translation import TranslationCore
 from ..base import AlchemyBase
 
 
+_BASE62_ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+_BASE62 = len(_BASE62_ALPHABET)
+
+
 class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1, WalletMixin):
     LOCALE_DOMAIN = 'sanic_telegram_bot'
 
@@ -18,7 +22,7 @@ class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1, W
 
     username = Column(String(512), nullable=True)
     lang_code = Column(String(8), nullable=False, default="en")
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
 
     last_use = Column(DateTime, nullable=False, default=datetime.utcnow)
     updated = Column(DateTime, nullable=False, default=datetime.utcnow)
@@ -32,4 +36,26 @@ class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1, W
     def __str__(self):
         return f"User, {self.id}_{self.telegram_id} | Username: {self.username} " + '\\'
 
+    def ensure_ref_id(self):
+        """Return a short referral identifier, generating it if missing."""
+        meta = self.meta or {}
+        ref_id = meta.get('ref_id')
+        if isinstance(ref_id, str) and ref_id:
+            return ref_id
 
+        ref_id = self._generate_ref_id()
+        self.meta = {**meta, 'ref_id': ref_id}
+        return ref_id
+
+    def _generate_ref_id(self):
+        user_id = int(self.id or 0)
+        if user_id <= 0:
+            return '000'
+
+        value = user_id % (_BASE62 ** 3)
+        chars = []
+        for _ in range(3):
+            chars.append(_BASE62_ALPHABET[value % _BASE62])
+            value //= _BASE62
+
+        return ''.join(reversed(chars)) or '000'

app/core/models/user/wallet_mixin.py

@@ -7,21 +7,29 @@ from tonsdk.utils import Address
 from datetime import datetime, timedelta
 from app.core.logger import make_log
 from httpx import AsyncClient
+from app.core.models.content.indexation_mixins import unpack_item_indexator_data, MIN_ONCHAIN_INDEX
+
+def _platform_address_str() -> str:
+    from app.core._blockchain.ton.platform import platform
+    return platform.address.to_string(1, 1, 1)
 
 
 class WalletMixin:
-    def wallet_connection(self, db_session):
+    async def wallet_connection_async(self, db_session):
-        return db_session.query(WalletConnection).filter(
+        from sqlalchemy import select, and_, desc
-            WalletConnection.user_id == self.id,
+        result = await db_session.execute(
-            WalletConnection.invalidated == False
+            select(WalletConnection)
-        ).order_by(WalletConnection.created.desc()).first()
+            .where(and_(WalletConnection.user_id == self.id, WalletConnection.invalidated == False))
+            .order_by(WalletConnection.created.desc())
+        )
+        return result.scalars().first()
 
-    def wallet_address(self, db_session):
+    async def wallet_address_async(self, db_session):
-        wallet_connection = self.wallet_connection(db_session)
+        wc = await self.wallet_connection_async(db_session)
-        return wallet_connection.wallet_address if wallet_connection else None
+        return wc.wallet_address if wc else None
 
     async def scan_owned_user_content(self, db_session):
-        user_wallet_address = self.wallet_address(db_session)
+        user_wallet_address = await self.wallet_address_async(db_session)
 
         async def get_nft_items_list():
             try:
@@ -40,10 +48,69 @@ class WalletMixin:
             item_address = Address(nft_item['address']).to_string(1, 1, 1)
             owner_address = Address(nft_item['owner']['address']).to_string(1, 1, 1)
 
-            user_content = db_session.query(UserContent).filter(
+            platform_address = _platform_address_str()
-                UserContent.onchain_address == item_address
+            collection_address = None
-            ).first()
+            if isinstance(nft_item, dict):
+                collection_data = nft_item.get('collection')
+                if isinstance(collection_data, dict):
+                    collection_address = collection_data.get('address')
+                collection_address = collection_address or nft_item.get('collection_address')
+            if collection_address:
+                try:
+                    collection_address = Address(collection_address).to_string(1, 1, 1)
+                except Exception:
+                    pass
+
+            item_index = None
+            license_type = None
+            # Prefer index from tonapi payload if available
+            raw_index = nft_item.get('index') if isinstance(nft_item, dict) else None
+            if isinstance(raw_index, int):
+                item_index = raw_index
+
+            need_chain_probe = item_index is None or item_index < MIN_ONCHAIN_INDEX
+            platform_address_onchain = None
+            if need_chain_probe:
+                try:
+                    indexator_raw = await toncenter.run_get_method(item_address, 'indexator_data')
+                    if indexator_raw.get('exit_code', -1) == 0:
+                        indexator_data = unpack_item_indexator_data(indexator_raw)
+                        item_index = indexator_data['index']
+                        license_type = indexator_data.get('license_type')
+                        platform_address_onchain = indexator_data.get('platform_address')
+                except BaseException as err:
+                    make_log(self, f"Failed to fetch indexator data for {item_address}: {err}", level='warning')
+
+            if item_index is None:
+                make_log(self, f"Skip NFT {item_address}: unable to resolve on-chain index", level='warning')
+                continue
+
+            if platform_address_onchain and platform_address_onchain != platform_address:
+                make_log(
+                    self,
+                    f"Skip foreign NFT {item_address}: platform mismatch {platform_address_onchain} != {platform_address}",
+                    level='debug'
+                )
+                continue
+
+            if item_index < MIN_ONCHAIN_INDEX and (license_type is None or license_type == 0):
+                make_log(
+                    self,
+                    f"Ignore NFT {item_address} with index {item_index} < MIN_ONCHAIN_INDEX={MIN_ONCHAIN_INDEX} (license_type={license_type})",
+                    level='debug'
+                )
+                continue
+
+            from sqlalchemy import select
+            user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
             if user_content:
+                if license_type is not None and license_type != 0 and user_content.type == 'nft/ignored':
+                    user_content.type = 'nft/unknown'
+                    user_content.meta = {**(user_content.meta or {}), 'license_type': license_type}
+                    user_content.owner_address = owner_address
+                    user_content.status = 'active'
+                    user_content.updated = datetime.fromtimestamp(0)
+                    await db_session.commit()
                 continue
 
             user_content = UserContent(
@@ -55,20 +122,20 @@ class WalletMixin:
                 updated=datetime.fromtimestamp(0),
                 content_id=None,  # not resolved yet
                 created=datetime.now(),
-                meta={},
+                meta={'license_type': license_type} if license_type is not None else {},
                 user_id=self.id,
-                wallet_connection_id=self.wallet_connection(db_session).id,
+                wallet_connection_id=(await self.wallet_connection_async(db_session)).id,
                 status="active"
             )
             db_session.add(user_content)
-            db_session.commit()
+            await db_session.commit()
             make_log(self, f"New onchain NFT found: {item_address}", level='info')
 
     async def ____scan_owned_user_content(self, db_session):
         page_id = -1
         page_size = 100
         have_next_page = True
-        user_wallet_address = self.wallet_address(db_session)
+        user_wallet_address = await self.wallet_address_async(db_session)
         while have_next_page:
             page_id += 1
             nfts_list = await toncenter.get_nft_items(limit=100, offset=page_id * page_size, owner_address=user_wallet_address)
@@ -81,9 +148,35 @@ class WalletMixin:
                     item_address = Address(nft_item['address']).to_string(1, 1, 1)
                     owner_address = Address(nft_item['owner_address']).to_string(1, 1, 1)
 
-                    user_content = db_session.query(UserContent).filter(
+                    platform_address = _platform_address_str()
-                        UserContent.onchain_address == item_address
+                    collection_address = nft_item.get('collection_address') if isinstance(nft_item, dict) else None
-                    ).first()
+                    if collection_address:
+                        try:
+                            normalized_collection = Address(collection_address).to_string(1, 1, 1)
+                        except Exception:
+                            normalized_collection = collection_address
+                        if normalized_collection != platform_address:
+                            make_log(self, f"Skip foreign NFT {item_address} from collection {normalized_collection}", level='debug')
+                            continue
+
+                    item_index = None
+                    try:
+                        indexator_raw = await toncenter.run_get_method(item_address, 'indexator_data')
+                        if indexator_raw.get('exit_code', -1) == 0:
+                            item_index = unpack_item_indexator_data(indexator_raw)['index']
+                    except BaseException as err:
+                        make_log(self, f"Failed to fetch indexator data for {item_address}: {err}", level='warning')
+
+                    if item_index is None:
+                        make_log(self, f"Skip NFT {item_address}: unable to resolve on-chain index", level='warning')
+                        continue
+
+                    if item_index is not None and item_index < MIN_ONCHAIN_INDEX:
+                        make_log(self, f"Ignore NFT {item_address} with index {item_index} < MIN_ONCHAIN_INDEX={MIN_ONCHAIN_INDEX}", level='debug')
+                        continue
+
+                    from sqlalchemy import select
+                    user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
                     if user_content:
                         continue
 
@@ -105,11 +198,11 @@ class WalletMixin:
                             'metadata_uri': nft_content,
                         },
                         user_id=self.id,
-                        wallet_connection_id=self.wallet_connection(db_session).id,
+                        wallet_connection_id=(await self.wallet_connection_async(db_session)).id,
                         status="active"
                     )
                     db_session.add(user_content)
-                    db_session.commit()
+                    await db_session.commit()
 
                     make_log(self, f"New onchain NFT found: {item_address}", level='info')
                 except BaseException as e:
@@ -122,6 +215,6 @@ class WalletMixin:
         except BaseException as e:
             make_log(self, f"Error while scanning user content: {e}", level='error')
 
-        return self.db_session.query(UserContent).filter(
+        from sqlalchemy import select
-            UserContent.user_id == self.id
+        result = await db_session.execute(select(UserContent).where(UserContent.user_id == self.id).offset(offset).limit(limit))
-        ).offset(offset).limit(limit).all()
+        return result.scalars().all()

app/core/models/user_activity.py

@@ -2,6 +2,7 @@
 from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
 from sqlalchemy.orm import relationship
 from .base import AlchemyBase
+from datetime import datetime
 
 
 class UserActivity(AlchemyBase):
@@ -9,10 +10,10 @@ class UserActivity(AlchemyBase):
 
     id = Column(Integer, autoincrement=True, primary_key=True)
     type = Column(String(64), nullable=False)
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
 
     user_id = Column(Integer, ForeignKey('users.id'), nullable=True)
     user_ip = Column(String(64), nullable=True)
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     user = relationship('User', uselist=False, foreign_keys=[user_id])

app/core/models/wallet_connection.py

@@ -1,5 +1,6 @@
 from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, JSON, Boolean
 from sqlalchemy.orm import relationship
+from datetime import datetime
 
 from .base import AlchemyBase
 
@@ -15,11 +16,11 @@ class WalletConnection(AlchemyBase):
 
     wallet_address = Column(String(1024), nullable=False)
 
-    keys = Column(JSON, nullable=False, default={})
+    keys = Column(JSON, nullable=False, default=dict)
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
 
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
-    updated = Column(DateTime, nullable=False, default=0)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
     invalidated = Column(Boolean, nullable=False, default=True)
     without_pk = Column(Boolean, nullable=False, default=False)
 

app/core/network/__init__.py

@@ -0,0 +1,2 @@
+# Network package for MY nodes
+

app/core/network/config.py

@@ -0,0 +1,38 @@
+import os
+from typing import List
+
+from app.core._config import PROJECT_HOST
+from .constants import NODE_TYPE_PUBLIC, NODE_TYPE_PRIVATE
+
+
+def _csv_list(val: str) -> List[str]:
+    return [x.strip() for x in (val or "").split(",") if x.strip()]
+
+
+# Handshake / network config driven by env
+NODE_PRIVACY = os.getenv("NODE_PRIVACY", NODE_TYPE_PUBLIC).strip().lower()
+if NODE_PRIVACY not in (NODE_TYPE_PUBLIC, NODE_TYPE_PRIVATE):
+    NODE_PRIVACY = NODE_TYPE_PUBLIC
+
+# Public endpoint for network (can be empty for private nodes)
+_env_public_host = os.getenv("PUBLIC_HOST")
+PUBLIC_HOST = _env_public_host if (_env_public_host is not None and _env_public_host.strip() != "") else None
+
+HANDSHAKE_INTERVAL_SEC = int(os.getenv("HANDSHAKE_INTERVAL_SEC", "5"))
+UNSUPPORTED_RECHECK_INTERVAL_SEC = int(os.getenv("UNSUPPORTED_RECHECK_INTERVAL_SEC", str(24 * 3600)))
+
+BOOTSTRAP_SEEDS = _csv_list(os.getenv("BOOTSTRAP_SEEDS", ""))
+BOOTSTRAP_REQUIRED = int(os.getenv("BOOTSTRAP_REQUIRED", "1")) == 1
+BOOTSTRAP_TIMEOUT_SEC = int(os.getenv("BOOTSTRAP_TIMEOUT_SEC", "20"))
+
+# Security knobs
+NETWORK_TLS_VERIFY = int(os.getenv("NETWORK_TLS_VERIFY", "1")) == 1
+HANDSHAKE_TS_TOLERANCE_SEC = int(os.getenv("HANDSHAKE_TS_TOLERANCE_SEC", "300"))
+HANDSHAKE_RATE_LIMIT_PER_MIN = int(os.getenv("HANDSHAKE_RATE_LIMIT_PER_MIN", "60"))
+
+# Capabilities
+NODE_IS_BOOTSTRAP = int(os.getenv("NODE_IS_BOOTSTRAP", "0")) == 1
+MAX_CONTENT_SIZE_MB = int(os.getenv("MAX_CONTENT_SIZE_MB", "512"))
+
+# Privacy allowlist (for NODE_PRIVACY=private)
+PRIVATE_ALLOWLIST = _csv_list(os.getenv("PRIVATE_ALLOWLIST", "/api/system.version"))

app/core/network/constants.py

@@ -0,0 +1,6 @@
+CURRENT_PROTOCOL_VERSION = "3.0.0"
+
+# Node roles/types
+NODE_TYPE_PUBLIC = "public"
+NODE_TYPE_PRIVATE = "private"
+

app/core/network/guard.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import time
+from typing import Dict, Set
+
+from app.core.network.config import HANDSHAKE_RATE_LIMIT_PER_MIN, HANDSHAKE_TS_TOLERANCE_SEC
+
+
+def check_rate_limit(memory, remote_ip: str) -> bool:
+    """Simple per-IP rate limit within current minute window.
+    Returns True if allowed, False if limited.
+    """
+    now = int(time.time())
+    minute = now // 60
+    rl = getattr(memory, "_handshake_rl", None)
+    if rl is None or rl.get("minute") != minute:
+        rl = {"minute": minute, "counts": {}}
+        memory._handshake_rl = rl
+    counts = rl["counts"]
+    cnt = counts.get(remote_ip, 0)
+    if cnt >= HANDSHAKE_RATE_LIMIT_PER_MIN:
+        return False
+    counts[remote_ip] = cnt + 1
+    return True
+
+
+def check_timestamp_fresh(ts: int) -> bool:
+    now = int(time.time())
+    return abs(now - int(ts)) <= HANDSHAKE_TS_TOLERANCE_SEC
+
+
+def check_and_remember_nonce(memory, pubkey_b58: str, nonce: str) -> bool:
+    """Return True if nonce is new; remember nonce with TTL ~ tolerance window.
+    We keep a compact in-memory set per pubkey.
+    """
+    now = int(time.time())
+    store = getattr(memory, "_handshake_nonces", None)
+    if store is None:
+        store = {}
+        memory._handshake_nonces = store
+
+    entry = store.get(pubkey_b58)
+    if entry is None:
+        entry = {"nonces": {}, "updated": now}
+        store[pubkey_b58] = entry
+
+    nonces: Dict[str, int] = entry["nonces"]
+    # prune old nonces
+    to_delete = [k for k, t in nonces.items() if now - int(t) > HANDSHAKE_TS_TOLERANCE_SEC]
+    for k in to_delete:
+        nonces.pop(k, None)
+
+    if nonce in nonces:
+        return False
+    # prevent unbounded growth
+    if len(nonces) > 2048:
+        # drop half oldest
+        for k, _ in sorted(nonces.items(), key=lambda kv: kv[1])[:1024]:
+            nonces.pop(k, None)
+    nonces[nonce] = now
+    entry["updated"] = now
+    return True
+

app/core/network/handshake.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+import os
+import time
+import shutil
+import secrets
+from typing import Dict, Any
+
+from base58 import b58encode
+from sqlalchemy import select
+
+from app.core._secrets import hot_pubkey, hot_seed
+from app.core._crypto.signer import Signer
+from app.core.logger import make_log
+from app.core.models.my_network import KnownNode
+from app.core.models.node_storage import StoredContent
+from app.core.storage import db_session
+from .constants import CURRENT_PROTOCOL_VERSION
+from .nodes import list_known_public_nodes
+from .config import PUBLIC_HOST, NODE_PRIVACY, NODE_IS_BOOTSTRAP, MAX_CONTENT_SIZE_MB
+from app.core._config import ALLOWED_CONTENT_TYPES
+from .constants import NODE_TYPE_PUBLIC
+
+
+START_TS = time.time()
+
+
+async def _metrics(session) -> Dict[str, Any]:
+    # Lightweight metrics for handshake
+    # Count total content (any type)
+    total_contents = (await session.execute(select(StoredContent))).scalars().all()
+    content_count = len(total_contents)
+    # Basic system metrics
+    try:
+        load1, load5, load15 = os.getloadavg()
+    except Exception:
+        load1 = load5 = load15 = 0.0
+    try:
+        from app.core._config import UPLOADS_DIR
+        du = shutil.disk_usage(UPLOADS_DIR)
+        disk_total_gb = round(du.total / (1024 ** 3), 2)
+        disk_free_gb = round(du.free / (1024 ** 3), 2)
+    except Exception:
+        disk_total_gb = disk_free_gb = -1
+    uptime_sec = int(time.time() - START_TS)
+    return {
+        "content_count": content_count,
+        "uptime_sec": uptime_sec,
+        "loadavg": [load1, load5, load15],
+        "disk_total_gb": disk_total_gb,
+        "disk_free_gb": disk_free_gb,
+    }
+
+
+def _sign(obj: Dict[str, Any]) -> str:
+    signer = Signer(hot_seed)
+    blob = json.dumps(obj, sort_keys=True, separators=(",", ":")).encode()
+    return signer.sign(blob)
+
+
+async def build_handshake_payload(session) -> Dict[str, Any]:
+    payload = {
+        "version": CURRENT_PROTOCOL_VERSION,
+        "public_key": b58encode(hot_pubkey).decode(),
+        # public_host is optional for private nodes
+        **({"public_host": PUBLIC_HOST} if PUBLIC_HOST else {}),
+        "node_type": NODE_PRIVACY if NODE_PRIVACY != NODE_TYPE_PUBLIC else NODE_TYPE_PUBLIC,
+        "metrics": await _metrics(session),
+        "capabilities": {
+            "accepts_inbound": NODE_PRIVACY == NODE_TYPE_PUBLIC,
+            "is_bootstrap": NODE_IS_BOOTSTRAP,
+            "supported_types": ALLOWED_CONTENT_TYPES,
+            "max_content_size_mb": MAX_CONTENT_SIZE_MB,
+        },
+        "timestamp": int(datetime.utcnow().timestamp()),
+        "nonce": secrets.token_hex(16),
+    }
+    try:
+        payload["known_public_nodes"] = await list_known_public_nodes(session)
+    except Exception:
+        payload["known_public_nodes"] = []
+    payload["signature"] = _sign(payload)
+    return payload
+
+
+async def compute_node_info(session) -> Dict[str, Any]:
+    node_info = {
+        "id": b58encode(hot_pubkey).decode(),
+        "public_key": b58encode(hot_pubkey).decode(),
+        **({"public_host": PUBLIC_HOST} if PUBLIC_HOST else {}),
+        "version": CURRENT_PROTOCOL_VERSION,
+        "node_type": NODE_PRIVACY,
+        "metrics": await _metrics(session),
+        "capabilities": {
+            "accepts_inbound": NODE_PRIVACY == NODE_TYPE_PUBLIC,
+            "is_bootstrap": NODE_IS_BOOTSTRAP,
+            "supported_types": ALLOWED_CONTENT_TYPES,
+            "max_content_size_mb": MAX_CONTENT_SIZE_MB,
+        },
+    }
+    return node_info
+
+def sign_response(data: Dict[str, Any]) -> Dict[str, Any]:
+    body = {
+        **data,
+        "timestamp": int(datetime.utcnow().timestamp()),
+    }
+    sig = _sign(body)
+    body["server_public_key"] = b58encode(hot_pubkey).decode()
+    body["server_signature"] = sig
+    return body

app/core/network/key_client.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import base64
+from typing import Optional
+
+import httpx
+from base58 import b58encode
+
+from app.core._secrets import hot_seed, hot_pubkey
+from app.core.crypto.x25519 import ed25519_to_x25519
+from app.core.logger import make_log
+from app.core.network.nodesig import sign_headers
+
+
+async def request_key_from_peer(base_url: str, encrypted_cid: str) -> Optional[bytes]:
+    """
+    Request a sealed key from peer and decrypt it using our X25519 private key.
+    Returns plaintext DEK bytes or None on failure.
+    """
+    try:
+        sk_x, pk_x = ed25519_to_x25519(hot_seed)
+        node_id = b58encode(hot_pubkey).decode()
+        body = {
+            "encrypted_cid": encrypted_cid,
+            "requestor_node_id": node_id,
+            "recipient_box_pub": base64.b64encode(bytes(pk_x)).decode(),
+        }
+        path = "/api/v1/keys.request"
+        headers = sign_headers("POST", path, json.dumps(body).encode(), hot_seed, b58encode(hot_pubkey).decode())
+        async with httpx.AsyncClient(timeout=15) as client:
+            r = await client.post(f"{base_url.rstrip('/')}{path}", json=body, headers=headers)
+            if r.status_code != 200:
+                make_log('key_client', f"{base_url} returned {r.status_code}: {r.text}", level='warning')
+                return None
+            j = r.json()
+        sealed_b64 = j.get('sealed_key_b64')
+        if not sealed_b64:
+            return None
+        sealed = base64.b64decode(sealed_b64)
+        from nacl.public import SealedBox
+        sb = SealedBox(sk_x)
+        dek = sb.decrypt(sealed)
+        return dek
+    except Exception as e:
+        make_log('key_client', f"request/decrypt failed: {e}", level='error')
+        return None

app/core/network/nodes.py

@@ -0,0 +1,261 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timedelta
+import json
+from typing import Dict, Any, Optional, List
+
+import httpx
+from base58 import b58encode
+from sqlalchemy import select, update
+
+from app.core.logger import make_log
+from app.core.models.my_network import KnownNode
+from app.core.storage import db_session
+from app.core._secrets import hot_pubkey
+from .config import (
+    HANDSHAKE_INTERVAL_SEC,
+    UNSUPPORTED_RECHECK_INTERVAL_SEC,
+    BOOTSTRAP_SEEDS,
+    BOOTSTRAP_REQUIRED,
+    BOOTSTRAP_TIMEOUT_SEC,
+    NODE_PRIVACY,
+    NETWORK_TLS_VERIFY,
+)
+from .constants import NODE_TYPE_PRIVATE
+from .semver import compatibility
+from .constants import CURRENT_PROTOCOL_VERSION
+
+
+def _now() -> datetime:
+    return datetime.utcnow()
+
+
+async def upsert_known_node(session, host: str, port: int, public_key: str, meta: Dict[str, Any]) -> KnownNode:
+    # Host can be full URL; normalize host/ip and port if available
+    host = (host or "").replace("http://", "").replace("https://", "").strip("/")
+    h_only = host
+    if ":" in host:
+        h_only, port_str = host.rsplit(":", 1)
+        try:
+            port = int(port_str)
+        except Exception:
+            pass
+    # Prefer match by public_key (stable identity)
+    if public_key:
+        result = await session.execute(select(KnownNode).where(KnownNode.public_key == public_key))
+        row = result.scalars().first()
+        if row:
+            row.ip = h_only or row.ip
+            row.port = port or row.port
+            row.public_key = public_key or row.public_key
+            row.meta = {**(row.meta or {}), **(meta or {})}
+            row.last_sync = _now()
+            await session.commit()
+            return row
+    # Fallback by IP/host
+    result = await session.execute(select(KnownNode).where(KnownNode.ip == h_only))
+    row = result.scalars().first()
+    if row:
+        row.port = port or row.port
+        row.public_key = public_key or row.public_key
+        row.meta = {**(row.meta or {}), **(meta or {})}
+        row.last_sync = _now()
+        await session.commit()
+        return row
+    node = KnownNode(
+        ip=h_only,
+        port=port or 80,
+        public_key=public_key,
+        reputation=0,
+        last_sync=_now(),
+        meta=meta or {},
+        located_at=_now(),
+    )
+    session.add(node)
+    await session.commit()
+    return node
+
+
+def _compatibility_for_meta(remote_version: str) -> str:
+    if not remote_version or remote_version == "0.0.0":
+        return "warning"
+    return compatibility(remote_version, CURRENT_PROTOCOL_VERSION)
+
+
+async def list_known_public_nodes(session) -> List[Dict[str, Any]]:
+    rows = (await session.execute(select(KnownNode))).scalars().all()
+    result = []
+    for r in rows:
+        meta = r.meta or {}
+        if not meta.get("is_public", True):
+            continue
+        result.append({
+            "host": r.ip,
+            "port": r.port,
+            "public_key": r.public_key,
+            "version": meta.get("version"),
+            "compatibility": _compatibility_for_meta(meta.get("version", "0.0.0")),
+            "last_seen": (r.last_sync.isoformat() + "Z") if r.last_sync else None,
+            "public_host": meta.get("public_host"),
+            "capabilities": meta.get("capabilities") or {},
+        })
+    return result
+
+
+async def _handshake_with(session, base_url: str) -> Optional[Dict[str, Any]]:
+    url = base_url.rstrip("/") + "/api/v1/network.handshake"
+    from .handshake import build_handshake_payload
+    payload = await build_handshake_payload(session)
+    timeout = httpx.Timeout(5.0, read=10.0)
+    async with httpx.AsyncClient(timeout=timeout, verify=NETWORK_TLS_VERIFY) as client:
+        r = await client.post(url, json=payload)
+        if r.status_code == 403 and NODE_PRIVACY == NODE_TYPE_PRIVATE:
+            # We are private; outbound is allowed, inbound denied by peers is fine
+            pass
+        r.raise_for_status()
+        data = r.json()
+        # Verify server signature if present
+        try:
+            import nacl.signing
+            from base58 import b58decode
+            required = ["server_signature", "server_public_key", "timestamp"]
+            if all(k in data for k in required):
+                signed_fields = {k: data[k] for k in data if k not in ("server_signature", "server_public_key")}
+                blob = json.dumps(signed_fields, sort_keys=True, separators=(",", ":")).encode()
+                vk = nacl.signing.VerifyKey(b58decode(data["server_public_key"]))
+                vk.verify(blob, b58decode(data["server_signature"]))
+        except Exception as e:
+            make_log("Handshake", f"Server signature verification failed for {base_url}: {e}", level='warning')
+        return data
+
+
+async def pick_next_node(session) -> Optional[KnownNode]:
+    rows = (await session.execute(select(KnownNode))).scalars().all()
+    if not rows:
+        return None
+    # Prefer nodes with oldest last_sync
+    rows.sort(key=lambda r: (r.last_sync or datetime.fromtimestamp(0)))
+    now = _now()
+    for r in rows:
+        meta = r.meta or {}
+        compat = _compatibility_for_meta(meta.get("version", "0.0.0"))
+        if compat == "blocked":
+            last = datetime.fromisoformat(meta.get("unsupported_last_checked_at")) if meta.get("unsupported_last_checked_at") else None
+            if last and (now - last) < timedelta(seconds=UNSUPPORTED_RECHECK_INTERVAL_SEC):
+                continue
+        # Backoff after failures
+        if meta.get("last_failure_at"):
+            try:
+                last_fail = datetime.fromisoformat(meta.get("last_failure_at"))
+                fail_count = int(meta.get("fail_count", 1))
+                # Exponential backoff: 30s * 2^fail_count, capped 2h
+                wait = min(7200, 30 * (2 ** max(0, fail_count)))
+                if (now - last_fail) < timedelta(seconds=wait):
+                    continue
+            except Exception:
+                pass
+        return r
+    # If we only have unsupported nodes and all are within cooldown, skip this round
+    return None
+
+
+async def perform_handshake_round():
+    async with db_session(auto_commit=True) as session:
+        # Private nodes still do outbound handshakes; inbound typically unreachable without public endpoint
+        node = await pick_next_node(session)
+        if not node:
+            return
+        base_url = node.meta.get("public_host") or f"http://{node.ip}:{node.port}"
+        try:
+            resp = await _handshake_with(session, base_url)
+            # Merge known nodes received
+            for peer in (resp or {}).get("known_public_nodes", []):
+                try:
+                    await upsert_known_node(
+                        session,
+                        host=peer.get("host") or peer.get("public_host") or "",
+                        port=int(peer.get("port") or 80),
+                        public_key=peer.get("public_key") or "",
+                        meta={
+                            "is_public": True,
+                            "version": peer.get("version") or "0.0.0",
+                            "public_host": peer.get("public_host") or (f"http://{peer.get('host')}:{peer.get('port')}" if peer.get('host') else None),
+                        }
+                    )
+                except Exception as e:
+                    make_log("Handshake", f"Ignore bad peer from {base_url}: {e}", level='warning')
+            # Update last_sync and meta for node
+            node.last_sync = _now()
+            node.meta = {**(node.meta or {}), "last_response": resp, "fail_count": 0}
+            await session.commit()
+            make_log("Handshake", f"Handshake OK with {base_url}")
+        except Exception as e:
+            make_log("Handshake", f"Handshake failed with {base_url}: {e}", level='warning')
+            # Record incident-lite in meta
+            meta = node.meta or {}
+            meta["last_error"] = str(e)
+            meta["last_failure_at"] = _now().isoformat()
+            meta["fail_count"] = int(meta.get("fail_count", 0)) + 1
+            node.meta = meta
+            await session.commit()
+
+
+async def network_handshake_daemon(app):
+    # Stagger start a bit to allow HTTP server to come up
+    await asyncio.sleep(3)
+    make_log("Handshake", f"Daemon started; interval={HANDSHAKE_INTERVAL_SEC}s")
+    while True:
+        try:
+            await perform_handshake_round()
+        except Exception as e:
+            make_log("Handshake", f"Round error: {e}", level='error')
+        await asyncio.sleep(HANDSHAKE_INTERVAL_SEC)
+
+
+async def bootstrap_once_and_exit_if_failed():
+    # Do not try to bootstrap private nodes as inbound is blocked, but outbound required for seeds discovery
+    seeds = BOOTSTRAP_SEEDS or []
+    if not seeds:
+        return  # Nothing to do
+    async with db_session(auto_commit=True) as session:
+        # If we already know nodes, skip bootstrap
+        have_any = (await session.execute(select(KnownNode))).scalars().first()
+        if have_any:
+            return
+    make_log("Bootstrap", f"Starting bootstrap with seeds={seeds}; required={BOOTSTRAP_REQUIRED}")
+
+    deadline = _now() + timedelta(seconds=BOOTSTRAP_TIMEOUT_SEC)
+    ok = False
+    for seed in seeds:
+        try:
+            async with db_session(auto_commit=True) as session:
+                resp = await _handshake_with(session, seed)
+                if resp:
+                    ok = True
+                    # Seed itself gets inserted by handshake handling route; also insert it explicitly
+                    try:
+                        await upsert_known_node(
+                            session,
+                            host=seed,
+                            port=80,
+                            public_key=resp.get("node", {}).get("public_key", ""),
+                            meta={
+                                "is_public": True,
+                                "version": resp.get("node", {}).get("version", "0.0.0"),
+                                "public_host": resp.get("node", {}).get("public_host") or seed,
+                            }
+                        )
+                    except Exception:
+                        pass
+                    break
+        except Exception as e:
+            make_log("Bootstrap", f"Seed failed {seed}: {e}", level='warning')
+        if _now() > deadline:
+            break
+
+    if BOOTSTRAP_REQUIRED and not ok:
+        make_log("Bootstrap", "Failed to reach any bootstrap seeds; exiting", level='error')
+        # Hard exit; Sanic won't stop otherwise
+        import os
+        os._exit(2)

app/core/network/nodesig.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import secrets
+import time
+from typing import Dict, Tuple
+
+from base58 import b58decode, b58encode
+
+from app.core.network.guard import check_timestamp_fresh, check_and_remember_nonce
+
+
+def _body_sha256(body: bytes) -> str:
+    h = hashlib.sha256()
+    h.update(body or b"")
+    return h.hexdigest()
+
+
+def canonical_string(method: str, path: str, body: bytes, ts: int, nonce: str, node_id: str) -> bytes:
+    parts = [
+        method.upper(),
+        path,
+        _body_sha256(body),
+        str(int(ts)),
+        str(nonce),
+        node_id,
+    ]
+    return ("\n".join(parts)).encode()
+
+
+def sign_headers(method: str, path: str, body: bytes, sk_bytes: bytes, pk_b58: str) -> Dict[str, str]:
+    import nacl.signing
+    ts = int(time.time())
+    nonce = secrets.token_hex(16)
+    msg = canonical_string(method, path, body, ts, nonce, pk_b58)
+    sig = nacl.signing.SigningKey(sk_bytes).sign(msg).signature
+    return {
+        "X-Node-Id": pk_b58,
+        "X-Node-Ts": str(ts),
+        "X-Node-Nonce": nonce,
+        "X-Node-Sig": b58encode(sig).decode(),
+    }
+
+
+def verify_request(request, memory) -> Tuple[bool, str, str]:
+    """Verify NodeSig headers of an incoming Sanic request.
+    Returns (ok, node_id, error). ok==True if signature valid, timestamp fresh, nonce unused.
+    """
+    try:
+        node_id = request.headers.get("X-Node-Id", "").strip()
+        ts = int(request.headers.get("X-Node-Ts", "0").strip() or 0)
+        nonce = request.headers.get("X-Node-Nonce", "").strip()
+        sig_b58 = request.headers.get("X-Node-Sig", "").strip()
+        if not node_id or not ts or not nonce or not sig_b58:
+            return False, "", "MISSING_HEADERS"
+        if not check_timestamp_fresh(ts):
+            return False, node_id, "STALE_TS"
+        if not check_and_remember_nonce(memory, node_id, nonce):
+            return False, node_id, "NONCE_REPLAY"
+        import nacl.signing
+        vk = nacl.signing.VerifyKey(b58decode(node_id))
+        sig = b58decode(sig_b58)
+        msg = canonical_string(request.method, request.path, request.body or b"", ts, nonce, node_id)
+        vk.verify(msg, sig)
+        return True, node_id, ""
+    except Exception as e:
+        return False, "", f"BAD_SIGNATURE: {e}"
+

app/core/network/semver.py

@@ -0,0 +1,26 @@
+from typing import Tuple
+
+
+def parse_semver(v: str) -> Tuple[int, int, int]:
+    try:
+        parts = v.split(".")
+        major = int(parts[0])
+        minor = int(parts[1]) if len(parts) > 1 else 0
+        patch = int(parts[2]) if len(parts) > 2 else 0
+        return major, minor, patch
+    except Exception:
+        return 0, 0, 0
+
+
+def compatibility(peer: str, current: str) -> str:
+    """Return one of: compatible, warning, blocked"""
+    pM, pm, pp = parse_semver(peer)
+    cM, cm, cp = parse_semver(current)
+    if pM != cM:
+        return "blocked"
+    # Same major
+    if pm == cm:
+        return "compatible"
+    # Different minor within same major => warning
+    return "warning"
+

app/core/projscale_logger.py

@@ -1,13 +1,15 @@
-from datetime import datetime
 import logging
 import time
 import httpx
 import threading
 import os
+from logging.handlers import TimedRotatingFileHandler
 
 PROJSCALE_APP_NAME = os.getenv('APP_PROJSCALE_NAME', 'my-uploader')
 LOGS_DIRECTORY = os.getenv('APP_LOGS_DIRECTORY', 'logs')
 os.makedirs(LOGS_DIRECTORY, exist_ok=True)
+LOG_FILE_BASENAME = os.getenv('APP_LOG_FILE_BASENAME', 'app.log')
+LOG_ROTATION_KEEP_HOURS = max(int(os.getenv('APP_LOG_ROTATION_KEEP_HOURS', '168')), 1)
 
 FORMAT_STRING = '%(asctime)s - %(levelname)s – %(pathname)s – %(funcName)s – %(lineno)d - %(message)s'
 
@@ -62,8 +64,14 @@ projscale_handler = ProjscaleLoggingHandler()
 projscale_handler.setLevel(logging.DEBUG)
 logger.addHandler(projscale_handler)
 
-log_filepath = f"{LOGS_DIRECTORY}/{datetime.now().strftime('%Y-%m-%d_%H')}.log"
+log_filepath = os.path.join(LOGS_DIRECTORY, LOG_FILE_BASENAME)
-file_handler = logging.FileHandler(log_filepath)
+file_handler = TimedRotatingFileHandler(
+    log_filepath,
+    when='H',
+    interval=1,
+    backupCount=LOG_ROTATION_KEEP_HOURS,
+    utc=False
+)
 file_handler.setLevel(logging.DEBUG)
 file_handler.setFormatter(logging.Formatter(FORMAT_STRING))
 logger.addHandler(file_handler)

app/core/storage.py

@@ -1,45 +1,57 @@
 import time
-from contextlib import contextmanager
+from contextlib import asynccontextmanager
 
-from sqlalchemy import create_engine
+from sqlalchemy import text
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
-from sqlalchemy.sql import text
 
-from app.core._config import MYSQL_URI, MYSQL_DATABASE
+from app.core._config import DATABASE_URL
 from app.core.logger import make_log
-from sqlalchemy.pool import NullPool
-
-engine = create_engine(MYSQL_URI, poolclass=NullPool)  #, echo=True)
-Session = sessionmaker(bind=engine)
 
 
-database_initialized = False
+def _to_async_dsn(url: str) -> str:
-while not database_initialized:
+    # Convert psycopg2 DSN to asyncpg DSN
+    # postgresql+psycopg2://user:pass@host:5432/db -> postgresql+asyncpg://user:pass@host:5432/db
+    return url.replace("+psycopg2", "+asyncpg")
+
+
+# Async engine for PostgreSQL
+engine = create_async_engine(
+    _to_async_dsn(DATABASE_URL),
+    pool_size=10,
+    max_overflow=20,
+    pool_timeout=30,
+    pool_recycle=1800,
+    pool_pre_ping=True,
+)
+
+AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
+
+
+async def wait_db_ready():
+    ready = False
+    while not ready:
         try:
-        with Session() as session:
+            async with engine.connect() as conn:
-            databases_list = session.execute(text("SHOW DATABASES;"))
+                await conn.execute(text("SELECT 1"))
-            databases_list = [row[0] for row in databases_list]
+                ready = True
-            make_log("SQL", 'Database list: ' + str(databases_list), level='debug')
-            assert MYSQL_DATABASE in databases_list, 'Database not found'
-            database_initialized = True
         except Exception as e:
-        make_log("SQL", 'MariaDB is not ready yet: ' + str(e), level='debug')
+            make_log("SQL", 'PostgreSQL is not ready yet: ' + str(e), level='debug')
             time.sleep(1)
 
-engine = create_engine(f"{MYSQL_URI}/{MYSQL_DATABASE}", poolclass=NullPool)
-Session = sessionmaker(bind=engine)
 
-
+@asynccontextmanager
-@contextmanager
+async def db_session(auto_commit: bool = False):
-def db_session(auto_commit=False):
+    session: AsyncSession = AsyncSessionLocal()
-    _session = Session()
     try:
-        yield _session
+        yield session
-        if auto_commit is True:
+        if auto_commit:
-            _session.commit()
+            await session.commit()
     except BaseException as e:
-        _session.rollback()
+        await session.rollback()
         raise e
     finally:
-        _session.close()
+        await session.close()
 
+
+def new_session() -> AsyncSession:
+    return AsyncSessionLocal()

app/core/transactions.py

@@ -1,19 +1,19 @@
 from datetime import datetime
 
+from sqlalchemy import select, and_, func
 from app.core.logger import make_log
 from app.core.models import Memory, User, UserBalance, Asset, InternalTransaction
 from app.core.storage import db_session
 
 
-def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
+async def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
     assert user, "No user"
     assert asset, "No asset"
-    result = session.query(UserBalance).filter(
+    result = await session.execute(select(UserBalance).where(
-        UserBalance.user_id == user.id,
+        and_(UserBalance.user_id == user.id, UserBalance.asset_id == asset.id)
-        UserBalance.asset_id == asset.id
+    ))
-    )
+    row = result.scalars().first()
-    results_count = result.count()
+    if not row:
-    if results_count == 0:
         user_balance = UserBalance(
             user_id=user.id,
             asset_id=asset.id,
@@ -21,12 +21,9 @@ def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
             created=datetime.now(),
         )
         session.add(user_balance)
-        session.commit()
+        await session.commit()
-        return get_user_balance(session, user, asset)
+        return await get_user_balance(session, user, asset)
-    elif results_count == 1:
+    return row
-        return result.first()
-    else:
-        raise Exception(f"Multiple user balances found: {results_count}")
 
 
 async def make_internal_transaction(
@@ -46,13 +43,13 @@ async def make_internal_transaction(
         raise Exception(f"Invalid amount: {amount}")
 
     abs_amount = abs(amount)
-    with db_session(auto_commit=False) as session:
+    async with db_session(auto_commit=False) as session:
         async with memory.transaction():
-            user = session.query(User).filter_by(id=user_id).first()
+            user = (await session.execute(select(User).where(User.id == user_id))).scalars().first()
             assert user, "No user"
-            asset = session.query(Asset).filter_by(id=asset_id).first()
+            asset = (await session.execute(select(Asset).where(Asset.id == asset_id))).scalars().first()
             assert asset, "No asset"
-            user_balance = get_user_balance(session, user, asset)
+            user_balance = await get_user_balance(session, user, asset)
             assert user_balance, "No user balance"
             if is_spent is True:
                 if abs_amount > user_balance.balance:
@@ -71,6 +68,6 @@ async def make_internal_transaction(
                 created=datetime.now(),
             )
             session.add(internal_transaction)
-            session.commit()
+            await session.commit()
 
         make_log(user, f"Made internal transaction: {'-' if is_spent else ''}{abs_amount} {asset.symbol}, type: {type}")

docker-compose.yml

@@ -1,105 +0,0 @@
-version: '3'
-services:
-  maria_db:
-    image: mariadb:11.2
-    ports:
-      - "3307:3306"
-    env_file:
-      - .env
-    volumes:
-      - /Storage/sqlStorage:/var/lib/mysql
-    restart: always
-    healthcheck:
-      test: [ "CMD", "healthcheck.sh", "--connect", "--innodb_initialized" ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-
-  app:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: python -m app
-    env_file:
-      - .env
-    restart: always
-    links:
-      - maria_db
-    ports:
-      - "15100:15100"
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-    depends_on:
-      maria_db:
-        condition: service_healthy
-
-  indexer:  # Отправка уведомления о появлении новой NFT-listen. Установка CID поля у всего контента. Проверка следующего за последним индексом item коллекции и поиск нового контента, отправка информации о том что контент найден его загружателю. Присваивание encrypted_content onchain_index
-    build:
-      context: .
-      dockerfile: Dockerfile
-    restart: always
-    command: python -m app indexer
-    env_file:
-      - .env
-    links:
-      - maria_db
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-    depends_on:
-      maria_db:
-        condition: service_healthy
-
-  ton_daemon: # Работа с TON-сетью. Задачи сервисного кошелька и деплой контрактов
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: python -m app ton_daemon
-    restart: always
-    env_file:
-      - .env
-    links:
-      - maria_db
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-    depends_on:
-      maria_db:
-        condition: service_healthy
-
-  license_index: # Проверка кошельков пользователей на новые NFT. Опрос этих NFT на определяемый GET-метод по которому мы определяем что это определенная лицензия и сохранение информации по ней
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: python -m app license_index
-    restart: always
-    env_file:
-      - .env
-    links:
-      - maria_db
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-    depends_on:
-      maria_db:
-        condition: service_healthy
-
-  convert_process:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: python -m app convert_process
-    restart: always
-    env_file:
-      - .env
-    links:
-      - maria_db
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-      - /var/run/docker.sock:/var/run/docker.sock
-    depends_on:
-      maria_db:
-        condition: service_healthy
-

docs/indexation.md

@@ -38,4 +38,73 @@ values:^[
 2. User uploads content cover to server (/api/v1/storage)
 3. User send /api/v1/blockchain.sendNewContentMessage to server and accept the transaction in wallet
 4. Indexer receives the transaction and indexes the content. And send telegram notification to user.
+# Network Index & Sync (v3)
 
+This document describes the simplified, production‑ready stack for content discovery and sync:
+
+- Upload via tus → stream encrypt (ENCF v1, AES‑256‑GCM, 1 MiB chunks) → `ipfs add --cid-version=1 --raw-leaves --chunker=size-1048576 --pin`.
+- Public index exposes only encrypted sources (CID) and safe metadata; no plaintext ids.
+- Nodes full‑sync by pinning encrypted CIDs; keys are auto‑granted to trusted peers for preview/full access.
+
+## ENCF v1 (Encrypted Content Format)
+
+Unencrypted header and framed body; same bytes on all nodes ⇒ stable CID.
+
+Header (all big endian):
+
+```
+MAGIC(4):  'ENCF'
+VER(1):    0x01
+SCHEME(1): 0x03 = AES_GCM (0x01 AES_GCM_SIV legacy, 0x02 AES_SIV legacy)
+CHUNK(4):  plaintext chunk bytes (1048576)
+SALT_LEN(1)
+SALT(N)
+RESERVED(5): zeros
+```
+
+Body: repeated frames `[p_len:4][cipher][tag(16)]` where `p_len <= CHUNK` for last frame.
+
+AES‑GCM (scheme `0x03`) encrypts each frame with deterministic `nonce = HMAC_SHA256(salt, u64(frame_idx))[:12]`. Legacy scheme `0x01` keeps AES‑GCM‑SIV with the same nonce derivation.
+
+For new uploads (v2025-09), the pipeline defaults to AES‑256‑GCM. Legacy AES‑GCM‑SIV/AES‑SIV content is still readable — the decoder auto-detects the scheme byte.
+
+### Local encryption/decryption helpers
+
+```
+python -m app.core.crypto.cli encrypt --input demo.wav --output demo.encf \
+  --key AAAAEyHSVws5O8JGrg3kUSVtk5dQSc5x5e7jh0S2WGE= --salt-bytes 16
+
+python -m app.core.crypto.cli decrypt --input demo.encf --output demo.wav \
+  --wrapped-key <ContentKey.key_ciphertext_b64>
+```
+
+Because we use standard AES‑GCM, you can also re-hydrate frames manually with tools like `openssl aes-256-gcm`. The header exposes `chunk_bytes` and salt; derive the per-frame nonce via `HMAC_SHA256(salt, idx)` where `idx` is the frame number (0-based) and feed the 12-byte prefix as IV.
+
+## API
+
+- `GET /api/v1/content.index` → `{ items:[...], schema, ETag }` with signed items.
+- `GET /api/v1/content.delta?since=ISO8601` → `{ items:[...], next_since, schema }` with ETag.
+- `POST /api/v1/sync.pin` (NodeSig required) → queue/pin CID.
+- `POST /api/v1/keys.request` (NodeSig required) → sealed DEK for trusted peers.
+- `GET /api/v1/content.derivatives?cid=` → local ready derivatives (low/high/preview).
+
+## NodeSig
+
+Canonical string:
+
+```
+METHOD\nPATH\nSHA256(body)\nTS\nNONCE\nNODE_ID
+```
+
+Headers: `X-Node-Id`, `X-Node-Ts`, `X-Node-Nonce`, `X-Node-Sig`.
+Window ±120s, nonce cache ~10min; replay → 401.
+
+## Sync daemon
+
+- Jitter 0–30s per peer; uses ETag/`since`.
+- Disk watermark (`SYNC_DISK_LOW_WATERMARK_PCT`) stops pin burst.
+- Pinned concurrently (`SYNC_MAX_CONCURRENT_PINS`) with pre‑`findprovs` `swarm/connect`.
+
+## Keys policy
+
+`KEY_AUTO_GRANT_TRUSTED_ONLY=1` — only KnownNode.meta.role=='trusted' gets DEK automatically. Preview lease TTL via `KEY_GRANT_PREVIEW_TTL_SEC`.

env.example

@@ -3,3 +3,4 @@ TELEGRAM_API_KEY=Paste your telegram api key from @BotFather here
 MYSQL_URI=mysql+pymysql://user:password@maria_db:3306
 MYSQL_ROOT_PASSWORD=playground
 MYSQL_DATABASE=bot_database
+CONTENT_KEY_KEK_B64=Paste base64-encoded 32-byte key for wrapping DEKs

requirements.txt

@@ -2,7 +2,8 @@ sanic==21.9.1
 websockets==10.0
 sqlalchemy==2.0.23
 python-dotenv==1.0.0
-pymysql==1.1.0
+psycopg2-binary==2.9.9
+asyncpg==0.29.0
 aiogram==3.13.0
 pytonconnect==0.3.0
 base58==2.1.1
@@ -16,5 +17,4 @@ pydub==0.25.1
 pillow==10.2.0
 ffmpeg-python==0.2.0
 python-magic==0.4.27
-
+cryptography==42.0.5
-

scripts/provision_node.sh

@@ -0,0 +1,393 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $(id -u) -ne 0 ]]; then
+  echo "This script must be run as root (use sudo)." >&2
+  exit 1
+fi
+
+if ! command -v lsb_release >/dev/null 2>&1; then
+  apt-get update -y
+  apt-get install -y lsb-release >/dev/null
+fi
+
+UBUNTU_CODENAME=$(lsb_release -sc)
+UBUNTU_MAJOR=$(lsb_release -rs | cut -d'.' -f1)
+if [[ "$UBUNTU_MAJOR" != "22" ]]; then
+  echo "Warning: this script targets Ubuntu 22.04. Detected $(lsb_release -ds)." >&2
+  read -r -p "Continue anyway? [y/N]: " _cont
+  _cont=${_cont:-N}
+  if [[ ! $_cont =~ ^[Yy]$ ]]; then
+    exit 1
+  fi
+fi
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+BACKEND_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)
+PROJECT_ROOT=$(cd "$BACKEND_ROOT/.." && pwd)
+CONFIGS_DIR="$PROJECT_ROOT/configs"
+FRONTEND_DIR="$PROJECT_ROOT/web2-client"
+
+if [[ ! -d "$CONFIGS_DIR" ]]; then
+  echo "Expected configs directory at $CONFIGS_DIR." >&2
+  exit 1
+fi
+if [[ ! -f "$CONFIGS_DIR/docker-compose.yml" ]]; then
+  echo "Missing docker-compose.yml in $CONFIGS_DIR." >&2
+  exit 1
+fi
+if [[ ! -d "$FRONTEND_DIR" ]]; then
+  echo "Warning: web2-client directory not found at $FRONTEND_DIR (frontend build will fail)." >&2
+fi
+
+ENV_FILE="$CONFIGS_DIR/.env"
+ENV_EXAMPLE="$BACKEND_ROOT/env.example"
+
+trim() {
+  local val="$1"
+  val="${val#${val%%[![:space:]]*}}"
+  val="${val%${val##*[![:space:]]}}"
+  printf '%s' "$val"
+}
+
+ini_val() {
+  local key="$1"
+  if [[ -f "$ENV_FILE" ]]; then
+    awk -F'=' -v k="$key" 'BEGIN{found=0} $1==k{print substr($0,index($0,$2)); found=1; exit} END{if(!found){} }' "$ENV_FILE"
+  fi
+}
+
+update_env() {
+  local key="$1"
+  local value="$2"
+  if [[ -f "$ENV_FILE" ]]; then
+    if grep -qE "^${key}=" "$ENV_FILE"; then
+      sed -i "s|^${key}=.*$|${key}=${value}|" "$ENV_FILE"
+      return
+    fi
+  fi
+  echo "${key}=${value}" >> "$ENV_FILE"
+}
+
+prompt_required() {
+  local var="$1"
+  local label="$2"
+  local default_val="${3:-}"
+  local value
+  while true; do
+    if [[ -n "$default_val" ]]; then
+      read -r -p "$label [$default_val]: " value || true
+      value=${value:-$default_val}
+    else
+      read -r -p "$label: " value || true
+    fi
+    value=$(trim "$value")
+    if [[ -n "$value" ]]; then
+      printf -v "$var" '%s' "$value"
+      return
+    fi
+    echo "Value is required."
+  done
+}
+
+prompt_optional() {
+  local var="$1"
+  local label="$2"
+  local default_val="${3:-}"
+  local value
+  if [[ -n "$default_val" ]]; then
+    read -r -p "$label [$default_val]: " value || true
+    value=${value:-$default_val}
+  else
+    read -r -p "$label: " value || true
+  fi
+  value=$(trim "$value")
+  printf -v "$var" '%s' "$value"
+}
+
+# Prepare environment file
+if [[ ! -f "$ENV_FILE" ]]; then
+  echo "Creating $ENV_FILE from example template." >&2
+  if [[ -f "$ENV_EXAMPLE" ]]; then
+    cp "$ENV_EXAMPLE" "$ENV_FILE"
+  else
+    cp "$CONFIGS_DIR/.env.example" "$ENV_FILE" 2>/dev/null || true
+    if [[ ! -f "$ENV_FILE" ]]; then
+      touch "$ENV_FILE"
+    fi
+  fi
+fi
+
+# Install base dependencies
+apt-get update -y
+apt-get install -y ca-certificates curl gnupg apt-transport-https software-properties-common git make nginx certbot python3-certbot-nginx python3 jq openssl
+
+# Docker repository setup
+install -m 0755 -d /etc/apt/keyrings
+if [[ ! -f /etc/apt/keyrings/docker.gpg ]]; then
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.gpg
+  chmod a+r /etc/apt/keyrings/docker.gpg
+fi
+cat <<EOF_REPO >/etc/apt/sources.list.d/docker.list
+deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $UBUNTU_CODENAME stable
+EOF_REPO
+apt-get update -y
+apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+systemctl enable --now docker
+
+if [[ -n "${SUDO_USER:-}" ]]; then
+  usermod -aG docker "$SUDO_USER"
+fi
+
+# Collect interactive inputs
+EXISTING_DOMAIN=$(ini_val PUBLIC_HOST)
+EXISTING_DOMAIN=${EXISTING_DOMAIN#https://}
+EXISTING_DOMAIN=${EXISTING_DOMAIN#http://}
+DEFAULT_DOMAIN=$(trim "$EXISTING_DOMAIN")
+prompt_required DOMAIN "Public domain (e.g. node.example.com)" "$DEFAULT_DOMAIN"
+DOMAIN=$(trim "$DOMAIN")
+PUBLIC_HOST="https://$DOMAIN"
+
+prompt_required EMAIL "Email for Let's Encrypt notifications" "$(ini_val CERTBOT_EMAIL)"
+
+DEFAULT_SEEDS=$(trim "${DEFAULT_DOMAIN:+https://$DEFAULT_DOMAIN}")
+if [[ -z "$DEFAULT_SEEDS" ]]; then
+  DEFAULT_SEEDS="https://my-public-node-8.projscale.dev"
+fi
+prompt_required BOOTSTRAP_SEEDS "Bootstrap seed URLs (comma-separated)" "$DEFAULT_SEEDS"
+IFS=',' read -r -a _seed_array <<< "$BOOTSTRAP_SEEDS"
+BOOTSTRAP_SEEDS=""
+for entry in "${_seed_array[@]}"; do
+  entry=$(trim "$entry")
+  [[ -z "$entry" ]] && continue
+  if [[ $entry != http://* && $entry != https://* ]]; then
+    entry="https://$entry"
+  fi
+  if [[ -z "$BOOTSTRAP_SEEDS" ]]; then
+    BOOTSTRAP_SEEDS="$entry"
+  else
+    BOOTSTRAP_SEEDS="$BOOTSTRAP_SEEDS,$entry"
+  fi
+done
+if [[ -z "$BOOTSTRAP_SEEDS" ]]; then
+  echo 'At least one bootstrap seed is required.' >&2
+  exit 1
+fi
+
+prompt_optional NODE_PRIVACY "Node privacy (public/private)" "$(ini_val NODE_PRIVACY)"
+if [[ -z "$NODE_PRIVACY" ]]; then
+  NODE_PRIVACY="public"
+fi
+NODE_PRIVACY=$(echo "$NODE_PRIVACY" | tr '[:upper:]' '[:lower:]')
+if [[ "$NODE_PRIVACY" != "public" && "$NODE_PRIVACY" != "private" ]]; then
+  echo "Invalid privacy option, defaulting to public." >&2
+  NODE_PRIVACY="public"
+fi
+
+DEFAULT_SANIC=$(ini_val SANIC_PORT)
+DEFAULT_SANIC=${DEFAULT_SANIC:-13200}
+prompt_optional SANIC_PORT "Internal backend port" "$DEFAULT_SANIC"
+DEFAULT_BACKEND_PORT=$(ini_val BACKEND_HTTP_PORT)
+DEFAULT_BACKEND_PORT=${DEFAULT_BACKEND_PORT:-13200}
+prompt_optional BACKEND_HTTP_PORT "Public backend port" "$DEFAULT_BACKEND_PORT"
+DEFAULT_FRONTEND_PORT=$(ini_val FRONTEND_HTTP_PORT)
+DEFAULT_FRONTEND_PORT=${DEFAULT_FRONTEND_PORT:-13300}
+prompt_optional FRONTEND_HTTP_PORT "Public frontend port" "$DEFAULT_FRONTEND_PORT"
+DEFAULT_TUSD_PORT=$(ini_val TUSD_HTTP_PORT)
+DEFAULT_TUSD_PORT=${DEFAULT_TUSD_PORT:-13400}
+prompt_optional TUSD_HTTP_PORT "Public tusd port" "$DEFAULT_TUSD_PORT"
+DEFAULT_PG_PORT=$(ini_val POSTGRES_FORWARD_PORT)
+DEFAULT_PG_PORT=${DEFAULT_PG_PORT:-13580}
+prompt_optional POSTGRES_FORWARD_PORT "Public Postgres port" "$DEFAULT_PG_PORT"
+
+prompt_required TELEGRAM_API_KEY "Telegram uploader bot token" "$(ini_val TELEGRAM_API_KEY)"
+prompt_required CLIENT_TELEGRAM_API_KEY "Telegram client bot token" "$(ini_val CLIENT_TELEGRAM_API_KEY)"
+
+prompt_optional ADMIN_API_TOKEN "Admin API token" "$(ini_val ADMIN_API_TOKEN)"
+if [[ -z "$ADMIN_API_TOKEN" ]]; then
+  ADMIN_API_TOKEN=$(openssl rand -hex 16)
+  echo "Generated ADMIN_API_TOKEN=$ADMIN_API_TOKEN"
+fi
+
+prompt_required TONCENTER_API_KEY "TON Center API key" "$(ini_val TONCENTER_API_KEY)"
+prompt_optional TON_INIT_HOT_SEED "TON hot wallet seed hex (leave blank to auto-generate)" "$(ini_val TON_INIT_HOT_SEED)"
+
+prompt_required CONTENT_KEY_KEK_B64 "CONTENT_KEY_KEK_B64 (32-byte base64)" "$(ini_val CONTENT_KEY_KEK_B64)"
+if ! python3 - "$CONTENT_KEY_KEK_B64" <<'PY'; then
+import base64, sys
+val = sys.argv[1]
+raw = base64.b64decode(val + '===' , validate=False)
+if len(raw) != 32:
+    raise SystemExit(1)
+PY
+  echo 'Invalid CONTENT_KEY_KEK_B64: must be base64-encoded 32 bytes.' >&2
+  exit 1
+fi
+
+prompt_required SWARM_KEY_HEX "IPFS swarm secret (32-byte hex)" ""
+SWARM_KEY_HEX=$(echo "$SWARM_KEY_HEX" | tr '[:lower:]' '[:upper:]')
+if ! python3 - "$SWARM_KEY_HEX" <<'PY'; then
+import sys
+val = sys.argv[1].strip()
+bytes.fromhex(val)
+if len(val) != 64:
+    raise SystemExit(1)
+PY
+  echo 'Invalid IPFS swarm key: must be 64 hex chars (32 bytes).' >&2
+  exit 1
+fi
+
+update_env CERTBOT_EMAIL "$EMAIL"
+update_env PUBLIC_HOST "$PUBLIC_HOST"
+update_env PROJECT_HOST "$PUBLIC_HOST"
+update_env NODE_PRIVACY "$NODE_PRIVACY"
+update_env BOOTSTRAP_SEEDS "$BOOTSTRAP_SEEDS"
+update_env SANIC_PORT "$SANIC_PORT"
+update_env BACKEND_HTTP_PORT "$BACKEND_HTTP_PORT"
+update_env FRONTEND_HTTP_PORT "$FRONTEND_HTTP_PORT"
+update_env TUSD_HTTP_PORT "$TUSD_HTTP_PORT"
+update_env POSTGRES_FORWARD_PORT "$POSTGRES_FORWARD_PORT"
+update_env TELEGRAM_API_KEY "$TELEGRAM_API_KEY"
+update_env CLIENT_TELEGRAM_API_KEY "$CLIENT_TELEGRAM_API_KEY"
+update_env ADMIN_API_TOKEN "$ADMIN_API_TOKEN"
+update_env TONCENTER_API_KEY "$TONCENTER_API_KEY"
+if [[ -n "$TON_INIT_HOT_SEED" ]]; then
+  update_env TON_INIT_HOT_SEED "$TON_INIT_HOT_SEED"
+fi
+update_env CONTENT_KEY_KEK_B64 "$CONTENT_KEY_KEK_B64"
+update_env HANDSHAKE_INTERVAL_SEC "60"
+update_env BOOTSTRAP_REQUIRED "1"
+update_env VITE_API_BASE_URL "${PUBLIC_HOST}/api/v1"
+update_env VITE_API_BASE_STORAGE_URL "${PUBLIC_HOST}/api/v1.5/storage"
+update_env VITE_TUS_ENDPOINT "${PUBLIC_HOST}/tus/files"
+update_env TON_CONNECT_MANIFEST_URI "${PUBLIC_HOST}/api/tonconnect-manifest.json"
+
+# Ensure swarm key file
+mkdir -p "$CONFIGS_DIR/ipfs"
+cat <<EOF_SWARM >"$CONFIGS_DIR/ipfs/swarm.key"
+/key/swarm/psk/1.0.0/
+/base16/
+$SWARM_KEY_HEX
+EOF_SWARM
+chmod 600 "$CONFIGS_DIR/ipfs/swarm.key"
+update_env IPFS_SWARM_KEY_FILE "$CONFIGS_DIR/ipfs/swarm.key"
+
+# Issue TLS certificate
+if [[ ! -d "/etc/letsencrypt/live/$DOMAIN" ]]; then
+  systemctl stop nginx || true
+  certbot certonly --standalone --agree-tos --non-interactive -m "$EMAIL" -d "$DOMAIN"
+  systemctl start nginx
+fi
+systemctl enable --now nginx
+
+# Nginx configuration
+cat <<EOF_NGX >/etc/nginx/sites-available/my-network.conf
+upstream backend_app {
+    server 127.0.0.1:$SANIC_PORT;
+    keepalive 32;
+}
+
+upstream frontend_web {
+    server 127.0.0.1:$FRONTEND_HTTP_PORT;
+    keepalive 16;
+}
+
+upstream tusd_backend {
+    server 127.0.0.1:$TUSD_HTTP_PORT;
+    keepalive 16;
+}
+
+server {
+    listen 80;
+    server_name $DOMAIN;
+    return 301 https://$DOMAIN$request_uri;
+}
+
+server {
+    listen 443 ssl http2;
+    server_name $DOMAIN;
+
+    ssl_certificate     /etc/letsencrypt/live/$DOMAIN/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/$DOMAIN/privkey.pem;
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_session_cache shared:SSL:10m;
+    ssl_session_timeout 1d;
+    add_header X-Content-Type-Options nosniff always;
+    add_header X-Frame-Options SAMEORIGIN always;
+
+    client_max_body_size 10G;
+    proxy_read_timeout 300s;
+
+    location / {
+        proxy_pass http://frontend_web;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    location /assets/ {
+        proxy_pass http://frontend_web;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_hide_header Cache-Control;
+        add_header Cache-Control "public, max-age=31536000, immutable" always;
+    }
+
+    location /tus/ {
+        proxy_pass http://tusd_backend/;
+        proxy_request_buffering off;
+        proxy_buffering off;
+        proxy_max_temp_file_size 0;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    location /files/ {
+        proxy_pass http://tusd_backend;
+        proxy_request_buffering off;
+        proxy_buffering off;
+        proxy_max_temp_file_size 0;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    location /api/ {
+        proxy_pass http://backend_app;
+        proxy_http_version 1.1;
+        proxy_request_buffering off;
+        proxy_buffering off;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    location = /health {
+        proxy_pass http://backend_app/api/system.version;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}
+EOF_NGX
+
+ln -sf /etc/nginx/sites-available/my-network.conf /etc/nginx/sites-enabled/my-network.conf
+rm -f /etc/nginx/sites-enabled/default
+nginx -t
+systemctl reload nginx
+
+# Bootstrap docker-compose stack
+make -C "$CONFIGS_DIR" bootstrap
+
+echo "\nNode provisioning complete."
+echo "- Admin panel: ${PUBLIC_HOST}/admin (use ADMIN_API_TOKEN)"
+echo "- To trust this node on peers, mark it via admin API on existing node."
+echo "- Docker services: run 'docker ps -a' or 'make -C $CONFIGS_DIR ps'."