fix config

Dockerfile

@@ -12,7 +12,8 @@ RUN apt-get update && apt-get install -y \
     ca-certificates \
     curl \
     gnupg \
-    lsb-release && \
+    lsb-release \
+    ffmpeg && \
     install -m 0755 -d /etc/apt/keyrings && \
     curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \
     chmod a+r /etc/apt/keyrings/docker.asc && \
@@ -21,7 +22,7 @@ RUN apt-get update && apt-get install -y \
     apt-get update && \
     apt-get install -y docker-ce-cli
 
-RUN apt-get install -y ffmpeg
+RUN apt-get install libmagic1 -y
 
 
 CMD ["python", "app"]

ENDPOINTS.md

@@ -48,6 +48,18 @@ TODO: реализовать поиск файла на других нодах
 16. GET /api/v1/content.view
 
 
+17. GET /api/v1/network.info
+Возвращает информацию о ноде: id/public_key, version, node_type, metrics, capabilities.
+
+18. GET /api/v1/network.nodes
+Возвращает список известных публичных нод с совместимостью и метаданными.
+
+19. POST /api/v1/network.handshake
+Рукопожатие между нодами. Тело запроса подписано приватным ключом ноды; ответ подписан приватным ключом сервера.
+Поля запроса: version, public_key (base58), node_type, metrics, capabilities, timestamp, nonce, signature.
+Поле public_host обязательно для public-нод, и опционально/пустое для private-нод.
+Поля ответа: compatibility, node, known_public_nodes, timestamp, server_public_key, server_signature (+ warning при несовпадении MINOR).
+Private-ноды не сохраняются на стороне принимающей ноды (никакого учета peer-а), но получают список публичных нод и могут синхронизироваться через них.
 
 
 

app/__main__.py

@@ -12,17 +12,12 @@ try:
 except BaseException:
     pass
 
-from app.core._utils.create_maria_tables import create_maria_tables
+from app.core._utils.create_maria_tables import create_db_tables
 from app.core.storage import engine
-if startup_target == '__main__':
-    create_maria_tables(engine)
-else:
+if startup_target != '__main__':
+    # Background services get a short delay before startup
     time.sleep(7)
 
-from app.api import app
-from app.bot import dp as uploader_bot_dp
-from app.client_bot import dp as client_bot_dp
-from app.core._config import SANIC_PORT, MYSQL_URI, PROJECT_HOST
 from app.core.logger import make_log
 
 if int(os.getenv("SANIC_MAINTENANCE", '0')) == 1:
@@ -52,7 +47,11 @@ async def execute_queue(app):
     make_log(None, f"Application normally started. HTTP port: {SANIC_PORT}")
     make_log(None, f"Telegram bot: https://t.me/{telegram_bot_username}")
     make_log(None, f"Client Telegram bot: https://t.me/{client_telegram_bot_username}")
-    make_log(None, f"MariaDB host: {MYSQL_URI.split('@')[1].split('/')[0].replace('/', '')}")
+    try:
+        _db_host = DATABASE_URL.split('@')[1].split('/')[0].replace('/', '')
+    except Exception:
+        _db_host = 'postgres://'
+    make_log(None, f"PostgreSQL host: {_db_host}")
     make_log(None, f"API host: {PROJECT_HOST}")
     while True:
         try:
@@ -81,16 +80,47 @@ async def execute_queue(app):
 if __name__ == '__main__':
     main_memory = Memory()
     if startup_target == '__main__':
+        # Defer heavy imports to avoid side effects in background services
+        # Mark this process as the primary node for seeding/config init
+        os.environ.setdefault('NODE_ROLE', 'primary')
+        # Create DB tables synchronously before importing HTTP app to satisfy _secrets
+        try:
+            from sqlalchemy import create_engine
+            from app.core.models import AlchemyBase  # imports all models
+            db_url = os.environ.get('DATABASE_URL')
+            if not db_url:
+                raise RuntimeError('DATABASE_URL is not set')
+            # Normalize to sync driver
+            if '+asyncpg' in db_url:
+                db_url_sync = db_url.replace('+asyncpg', '+psycopg2')
+            else:
+                db_url_sync = db_url
+            sync_engine = create_engine(db_url_sync, pool_pre_ping=True)
+            AlchemyBase.metadata.create_all(sync_engine)
+        except Exception as e:
+            make_log('Startup', f'DB sync init failed: {e}', level='error')
+        from app.api import app
+        from app.bot import dp as uploader_bot_dp
+        from app.client_bot import dp as client_bot_dp
+        from app.core._config import SANIC_PORT, PROJECT_HOST, DATABASE_URL
+        from app.core.network.nodes import network_handshake_daemon, bootstrap_once_and_exit_if_failed
+
         app.ctx.memory = main_memory
         for _target in [uploader_bot_dp, client_bot_dp]:
             _target._s_memory = app.ctx.memory
 
         app.ctx.memory._app = app
 
+        # Ensure DB schema exists using the same event loop as Sanic (idempotent)
+        app.add_task(create_db_tables(engine))
+
         app.add_task(execute_queue(app))
         app.add_task(queue_daemon(app))
         app.add_task(uploader_bot_dp.start_polling(app.ctx.memory._telegram_bot))
         app.add_task(client_bot_dp.start_polling(app.ctx.memory._client_telegram_bot))
+        # Start network handshake daemon and bootstrap step
+        app.add_task(network_handshake_daemon(app))
+        app.add_task(bootstrap_once_and_exit_if_failed())
 
         app.run(host='0.0.0.0', port=SANIC_PORT)
     else:
@@ -112,6 +142,15 @@ if __name__ == '__main__':
         elif startup_target == 'convert_process':
             from app.core.background.convert_service import main_fn as target_fn
             time.sleep(9)
+        elif startup_target == 'convert_v3':
+            from app.core.background.convert_v3_service import main_fn as target_fn
+            time.sleep(9)
+        elif startup_target == 'index_scout_v3':
+            from app.core.background.index_scout_v3 import main_fn as target_fn
+            time.sleep(7)
+        elif startup_target == 'derivative_janitor':
+            from app.core.background.derivative_cache_janitor import main_fn as target_fn
+            time.sleep(5)
 
         startup_fn = startup_fn or target_fn
         assert startup_fn
@@ -126,6 +165,7 @@ if __name__ == '__main__':
 
         loop = asyncio.get_event_loop()
         try:
+            # Background services no longer perform schema initialization
             loop.run_until_complete(wrapped_startup_fn(main_memory))
         except BaseException as e:
             make_log(startup_target[0].upper() + startup_target[1:], f"Error: {e}" + '\n' + str(traceback.format_exc()),

app/api/__init__.py

@@ -1,6 +1,8 @@
 import traceback
 
 from sanic import Sanic, response
+from uuid import uuid4
+import traceback as _traceback
 
 from app.core.logger import make_log
 
@@ -12,30 +14,55 @@ app.register_middleware(attach_user_to_request, "request")
 app.register_middleware(close_db_session, "response")
 
 from app.api.routes._index import s_index, s_favicon
-from app.api.routes._system import s_api_v1_node, s_api_system_version, s_api_system_send_status
-from app.api.routes.auth import s_api_v1_auth_twa
+from app.api.routes._system import s_api_v1_node, s_api_system_version, s_api_system_send_status, s_api_v1_node_friendly
+from app.api.routes.network import (
+    s_api_v1_network_info,
+    s_api_v1_network_nodes,
+    s_api_v1_network_handshake,
+)
+from app.api.routes.auth import s_api_v1_auth_twa, s_api_v1_auth_select_wallet, s_api_v1_auth_me
 from app.api.routes.statics import s_api_tonconnect_manifest, s_api_platform_metadata
 from app.api.routes.node_storage import s_api_v1_storage_post, s_api_v1_storage_get, \
     s_api_v1_storage_decode_cid
 from app.api.routes.progressive_storage import s_api_v1_5_storage_get, s_api_v1_5_storage_post
+from app.api.routes.upload_tus import s_api_v1_upload_tus_hook
 from app.api.routes.account import s_api_v1_account_get
 from app.api.routes._blockchain import s_api_v1_blockchain_send_new_content_message, \
     s_api_v1_blockchain_send_purchase_content_message
 from app.api.routes.content import s_api_v1_content_list, s_api_v1_content_view, s_api_v1_content_friendly_list, s_api_v1_5_content_list
+from app.api.routes.content_index import s_api_v1_content_index, s_api_v1_content_delta
+from app.api.routes.derivatives import s_api_v1_content_derivatives
+from app.api.routes.admin import (
+    s_api_v1_admin_node_setrole,
+    s_api_v1_admin_nodes,
+    s_api_v1_admin_status,
+    s_api_v1_admin_cache_setlimits,
+    s_api_v1_admin_cache_cleanup,
+    s_api_v1_admin_sync_setlimits,
+)
 from app.api.routes.tonconnect import s_api_v1_tonconnect_new, s_api_v1_tonconnect_logout
+from app.api.routes.keys import s_api_v1_keys_request
+from app.api.routes.sync import s_api_v1_sync_pin, s_api_v1_sync_status
+from app.api.routes.upload_status import s_api_v1_upload_status
 
 
 app.add_route(s_index, "/", methods=["GET", "OPTIONS"])
 app.add_route(s_favicon, "/favicon.ico", methods=["GET", "OPTIONS"])
 
 app.add_route(s_api_v1_node, "/api/v1/node", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_node_friendly, "/api/v1/nodeFriendly", methods=["GET", "OPTIONS"])
 app.add_route(s_api_system_version, "/api/system.version", methods=["GET", "OPTIONS"])
 app.add_route(s_api_system_send_status, "/api/system.sendStatus", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_network_info, "/api/v1/network.info", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_network_nodes, "/api/v1/network.nodes", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_network_handshake, "/api/v1/network.handshake", methods=["POST", "OPTIONS"])
 
 app.add_route(s_api_tonconnect_manifest, "/api/tonconnect-manifest.json", methods=["GET", "OPTIONS"])
 app.add_route(s_api_platform_metadata, "/api/platform-metadata.json", methods=["GET", "OPTIONS"])
 
 app.add_route(s_api_v1_auth_twa, "/api/v1/auth.twa", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_auth_me, "/api/v1/auth.me", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_auth_select_wallet, "/api/v1/auth.selectWallet", methods=["POST", "OPTIONS"])
 
 app.add_route(s_api_v1_tonconnect_new, "/api/v1/tonconnect.new", methods=["GET", "OPTIONS"])
 app.add_route(s_api_v1_tonconnect_logout, "/api/v1/tonconnect.logout", methods=["POST", "OPTIONS"])
@@ -56,22 +83,83 @@ app.add_route(s_api_v1_content_list, "/api/v1/content.list", methods=["GET", "OP
 app.add_route(s_api_v1_content_view, "/api/v1/content.view/<content_address>", methods=["GET", "OPTIONS"])
 app.add_route(s_api_v1_content_friendly_list, "/api/v1/content.friendlyList", methods=["GET", "OPTIONS"])
 app.add_route(s_api_v1_5_content_list, "/api/v1.5/content.list", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_content_index, "/api/v1/content.index", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_content_delta, "/api/v1/content.delta", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_content_derivatives, "/api/v1/content.derivatives", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_node_setrole, "/api/v1/admin.node.setRole", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_admin_nodes, "/api/v1/admin.nodes", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_status, "/api/v1/admin.status", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_admin_cache_setlimits, "/api/v1/admin.cache.setLimits", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_admin_cache_cleanup, "/api/v1/admin.cache.cleanup", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_admin_sync_setlimits, "/api/v1/admin.sync.setLimits", methods=["POST", "OPTIONS"])
+
+# tusd HTTP hooks
+app.add_route(s_api_v1_upload_tus_hook, "/api/v1/upload.tus-hook", methods=["POST", "OPTIONS"])
+
+# Keys auto-grant
+app.add_route(s_api_v1_keys_request, "/api/v1/keys.request", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_sync_pin, "/api/v1/sync.pin", methods=["POST", "OPTIONS"])
+app.add_route(s_api_v1_sync_status, "/api/v1/sync.status", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_upload_status, "/api/v1/upload.status/<upload_id>", methods=["GET", "OPTIONS"])
 
 
 @app.exception(BaseException)
 async def s_handle_exception(request, exception):
-    response_buffer = response.json({"error": "An internal server error occurred"}, status=500)
+    # Correlate error to request
+    session_id = getattr(request.ctx, 'session_id', None) or uuid4().hex[:16]
+    error_id = uuid4().hex[:8]
+
+    status = 500
+    code = type(exception).__name__
+    message = "Internal HTTP Error"
+
     try:
         raise exception
     except AssertionError as e:
-        response_buffer = response.json({"error": str(e)}, status=400)
+        status = 400
+        code = 'AssertionError'
+        message = str(e) or 'Bad Request'
     except BaseException as e:
-        make_log("sanic_exception", f"Exception: {e}" + '\n' + str(traceback.format_exc()), level='error')
+        # keep default 500, but expose exception message to aid debugging
+        message = str(e) or message
 
+    # Build structured log with full context and traceback
+    try:
+        tb = _traceback.format_exc()
+        user_id = getattr(getattr(request.ctx, 'user', None), 'id', None)
+        log_ctx = {
+            'sid': session_id,
+            'eid': error_id,
+            'path': request.path,
+            'method': request.method,
+            'query': dict(request.args) if hasattr(request, 'args') else {},
+            'user_id': user_id,
+            'remote': (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip),
+            'code': code,
+            'message': message,
+            'traceback': tb,
+        }
+        make_log('http_exception', 'API exception', level='error', **log_ctx)
+    except BaseException:
+        pass
+
+    # Return enriched error response for the client
+    payload = {
+        'error': True,
+        'code': code,
+        'message': message,
+        'session_id': session_id,
+        'error_id': error_id,
+        'path': request.path,
+        'method': request.method,
+    }
+
+    response_buffer = response.json(payload, status=status)
     response_buffer = await close_db_session(request, response_buffer)
     response_buffer.headers["Access-Control-Allow-Origin"] = "*"
     response_buffer.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
-    response_buffer.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site"
+    response_buffer.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-request-id"
     response_buffer.headers["Access-Control-Allow-Credentials"] = "true"
+    response_buffer.headers["X-Session-Id"] = session_id
+    response_buffer.headers["X-Error-Id"] = error_id
     return response_buffer
-

app/api/middleware.py

@@ -1,5 +1,6 @@
 from base58 import b58decode
 from sanic import response as sanic_response
+from uuid import uuid4
 
 from app.core._crypto.signer import Signer
 from app.core._secrets import hot_seed
@@ -8,15 +9,25 @@ from app.core.models.keys import KnownKey
 from app.core.models._telegram.wrapped_bot import Wrapped_CBotChat
 from app.core.models.user_activity import UserActivity
 from app.core.models.user import User
-from app.core.storage import Session
+from sqlalchemy import select
+from app.core.storage import new_session
 from datetime import datetime, timedelta
+from app.core.log_context import (
+    ctx_session_id, ctx_user_id, ctx_method, ctx_path, ctx_remote
+)
 
 
-def attach_headers(response):
+def attach_headers(response, request=None):
     response.headers["Access-Control-Allow-Origin"] = "*"
     response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
-    response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id"
-    response.headers["Access-Control-Allow-Credentials"] = "true"
+    response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id, x-request-id"
+    # response.headers["Access-Control-Allow-Credentials"] = "true"
+    try:
+        sid = getattr(request.ctx, 'session_id', None) if request else None
+        if sid:
+            response.headers["X-Session-Id"] = sid
+    except BaseException:
+        pass
     return response
 
 
@@ -30,7 +41,8 @@ async def try_authorization(request):
         make_log("auth", "Invalid token length", level="warning")
         return
 
-    known_key = request.ctx.db_session.query(KnownKey).filter(KnownKey.seed == token).first()
+    result = await request.ctx.db_session.execute(select(KnownKey).where(KnownKey.seed == token))
+    known_key = result.scalars().first()
     if not known_key:
         make_log("auth", "Unknown key", level="warning")
         return
@@ -58,7 +70,8 @@ async def try_authorization(request):
         make_log("auth", f"User ID mismatch: {known_key.meta.get('I_user_id', -1)} != {user_id}", level="warning")
         return
 
-    user = request.ctx.db_session.query(User).filter(User.id == known_key.meta['I_user_id']).first()
+    result = await request.ctx.db_session.execute(select(User).where(User.id == known_key.meta['I_user_id']))
+    user = result.scalars().first()
     if not user:
         make_log("auth", "No user from key", level="warning")
         return
@@ -118,7 +131,14 @@ async def save_activity(request):
         pass
 
     try:
-        activity_meta["headers"] = dict(request.headers)
+        # Sanitize sensitive headers
+        headers = dict(request.headers)
+        for hk in list(headers.keys()):
+            if str(hk).lower() in [
+                'authorization', 'cookie', 'x-service-signature', 'x-message-hash'
+            ]:
+                headers[hk] = '<redacted>'
+        activity_meta["headers"] = headers
     except:
         pass
 
@@ -127,23 +147,51 @@ async def save_activity(request):
         meta=activity_meta,
         user_id=request.ctx.user.id if request.ctx.user else None,
         user_ip=activity_meta.get("ip", "0.0.0.0"),
-        created=datetime.now()
+        created=datetime.utcnow()
     )
     request.ctx.db_session.add(new_user_activity)
-    request.ctx.db_session.commit()
+    await request.ctx.db_session.commit()
 
 
 async def attach_user_to_request(request):
     if request.method == 'OPTIONS':
-        return attach_headers(sanic_response.text("OK"))
+        return attach_headers(sanic_response.text("OK"), request)
 
-    request.ctx.db_session = Session()
+    request.ctx.db_session = new_session()
     request.ctx.verified_hash = None
     request.ctx.user = None
     request.ctx.user_key = None
     request.ctx.user_uploader_wrapper = Wrapped_CBotChat(request.app.ctx.memory._telegram_bot, db_session=request.ctx.db_session)
     request.ctx.user_client_wrapper = Wrapped_CBotChat(request.app.ctx.memory._client_telegram_bot, db_session=request.ctx.db_session)
+    # Correlation/session id for this request: prefer proxy-provided X-Request-ID
+    incoming_req_id = request.headers.get('X-Request-Id') or request.headers.get('X-Request-ID')
+    request.ctx.session_id = (incoming_req_id or uuid4().hex)[:32]
+    # Populate contextvars for automatic logging context
+    try:
+        ctx_session_id.set(request.ctx.session_id)
+        ctx_method.set(request.method)
+        ctx_path.set(request.path)
+        _remote = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip)
+        if _remote and isinstance(_remote, str) and ',' in _remote:
+            _remote = _remote.split(',')[0].strip()
+        ctx_remote.set(_remote)
+    except BaseException:
+        pass
+    try:
+        make_log(
+            "HTTP",
+            f"Request start sid={request.ctx.session_id} {request.method} {request.path}",
+            level='info'
+        )
+    except BaseException:
+        pass
     await try_authorization(request)
+    # Update user_id in context after auth
+    try:
+        if request.ctx.user and request.ctx.user.id:
+            ctx_user_id.set(request.ctx.user.id)
+    except BaseException:
+        pass
     await save_activity(request)
     await try_service_authorization(request)
 
@@ -153,16 +201,34 @@ async def close_request_handler(request, response):
         response = sanic_response.text("OK")
 
     try:
-        request.ctx.db_session.close()
-    except BaseException as e:
+        await request.ctx.db_session.close()
+    except BaseException:
         pass
 
-    response = attach_headers(response)
+    try:
+        make_log(
+            "HTTP",
+            f"Request end sid={getattr(request.ctx, 'session_id', None)} {request.method} {request.path} status={getattr(response, 'status', None)}",
+            level='info'
+        )
+    except BaseException:
+        pass
+
+    response = attach_headers(response, request)
 
     return request, response
 
 
 async def close_db_session(request, response):
     request, response = await close_request_handler(request, response)
-    response = attach_headers(response)
+    response = attach_headers(response, request)
+    # Clear contextvars
+    try:
+        ctx_session_id.set(None)
+        ctx_user_id.set(None)
+        ctx_method.set(None)
+        ctx_path.set(None)
+        ctx_remote.set(None)
+    except BaseException:
+        pass
     return response

app/api/routes/_blockchain.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import traceback
 
 from sanic import response
+from sqlalchemy import and_, select, func
 from tonsdk.boc import begin_cell, begin_dict
 from tonsdk.utils import Address
 
@@ -18,6 +19,8 @@ from app.core.models.content.user_content import UserContent
 from app.core.models.node_storage import StoredContent
 from app.core.models._telegram import Wrapped_CBotChat
 from app.core._keyboards import get_inline_keyboard
+from app.core.models.promo import PromoAction
+from app.core.models.tasks import BlockchainTask
 
 
 def valid_royalty_params(royalty_params):
@@ -43,7 +46,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
         for field_key, field_value in {
             'title': lambda x: isinstance(x, str),
             'authors': lambda x: isinstance(x, list),
-            'content': lambda x: isinstance(x, str),
+            'content': lambda x: isinstance(x, str),  # may be plaintext CID (legacy) or encrypted IPFS CID (bafy...)
             'image': lambda x: isinstance(x, str),
             'description': lambda x: isinstance(x, str),
             'price': lambda x: (isinstance(x, str) and x.isdigit()),
@@ -54,42 +57,64 @@ async def s_api_v1_blockchain_send_new_content_message(request):
             assert field_key in request.json, f"No {field_key} provided"
             assert field_value(request.json[field_key]), f"Invalid {field_key} provided"
 
-        decrypted_content_cid, err = resolve_content(request.json['content'])
-        assert not err, f"Invalid content CID"
-
-        # Поиск исходного файла загруженного
-        decrypted_content = request.ctx.db_session.query(StoredContent).filter(
-            StoredContent.hash == decrypted_content_cid.content_hash_b58
-        ).first()
-        assert decrypted_content, "No content locally found"
-        assert decrypted_content.type == "local/content_bin", "Invalid content type"
-
-        # Создание фиктивного encrypted_content. Не шифруем для производительности, тк зашифрованная нигде дальше не используется
-        encrypted_content = await create_encrypted_content(request.ctx.db_session, decrypted_content)
-        encrypted_content_cid = encrypted_content.cid
+        # Support legacy: 'content' as decrypted ContentId; and new: 'content' as encrypted IPFS CID
+        encrypted_content_cid = None
+        try:
+            # Legacy path
+            decrypted_content_cid, err = resolve_content(request.json['content'])
+            assert not err
+            decrypted_content = (await request.ctx.db_session.execute(
+                select(StoredContent).where(StoredContent.hash == decrypted_content_cid.content_hash_b58)
+            )).scalars().first()
+            assert decrypted_content and decrypted_content.type == "local/content_bin"
+            encrypted_content = await create_encrypted_content(request.ctx.db_session, decrypted_content)
+            encrypted_content_cid = encrypted_content.cid
+        except BaseException:
+            # New path: treat provided string as encrypted IPFS CID (ENCF v1)
+            encrypted_ipfs_cid = request.json['content']
+            class _EC:  # tiny adapter to mimic .serialize_v2()
+                def __init__(self, s: str):
+                    self._s = s
+                def serialize_v2(self, include_accept_type: bool = False):
+                    return self._s
+            encrypted_content_cid = _EC(encrypted_ipfs_cid)
 
         if request.json['image']:
             image_content_cid, err = resolve_content(request.json['image'])
             assert not err, f"Invalid image CID"
-            image_content = request.ctx.db_session.query(StoredContent).filter(
-                StoredContent.hash == image_content_cid.content_hash_b58
-            ).first()
+            image_content = (await request.ctx.db_session.execute(
+                select(StoredContent).where(StoredContent.hash == image_content_cid.content_hash_b58)
+            )).scalars().first()
             assert image_content, "No image locally found"
         else:
             image_content_cid = None
             image_content = None
 
 
-        content_title = f"{', '.join(request.json['authors'])} - {request.json['title']}" if request.json['authors'] else request.json['title']
+        content_title = f"{', '.join(request.json['authors'])} – {request.json['title']}" if request.json['authors'] else request.json['title']
 
         metadata_content = await create_metadata_for_item(
             request.ctx.db_session,
             title=content_title,
             cover_url=f"{PROJECT_HOST}/api/v1.5/storage/{image_content_cid.serialize_v2()}" if image_content_cid else None,
             authors=request.json['authors'],
-            hashtags=request.json['hashtags']
+            hashtags=request.json['hashtags'],
+            downloadable=request.json['downloadable'] if 'downloadable' in request.json else False,
         )
 
+        # Try to update ContentIndexItem with cover_url for this encrypted content
+        try:
+            from app.core.models.content_v3 import ContentIndexItem
+            ecid_str = encrypted_content_cid.serialize_v2()
+            row = (await request.ctx.db_session.execute(select(ContentIndexItem).where(ContentIndexItem.encrypted_cid == ecid_str))).scalars().first()
+            if row:
+                payload = row.payload or {}
+                payload['cover_url'] = f"{PROJECT_HOST}/api/v1.5/storage/{image_content_cid.serialize_v2()}" if image_content_cid else None
+                row.payload = payload
+                await request.ctx.db_session.commit()
+        except Exception as _e:
+            make_log("Blockchain", f"index cover update failed: {_e}", level='warning')
+
         royalties_dict = begin_dict(8)
         i = 0
         for royalty_param in request.json['royaltyParams']:
@@ -101,6 +126,99 @@ async def s_api_v1_blockchain_send_new_content_message(request):
             )
             i += 1
 
+        _cnt = (await request.ctx.db_session.execute(
+            select(func.count()).select_from(PromoAction).where(
+                and_(
+                    PromoAction.user_internal_id == request.ctx.user.id,
+                    PromoAction.action_type == 'freeUpload'
+                )
+            )
+        )).scalar()
+        promo_free_upload_available = 3 - int(_cnt or 0)
+
+        has_pending_task = (await request.ctx.db_session.execute(
+            select(BlockchainTask).where(
+                and_(BlockchainTask.user_id == request.ctx.user.id, BlockchainTask.status != 'done')
+            )
+        )).scalars().first()
+        if has_pending_task:
+            make_log("Blockchain", f"User {request.ctx.user.id} already has a pending task", level='warning')
+            promo_free_upload_available = 0
+
+        make_log("Blockchain", f"User {request.ctx.user.id} has {promo_free_upload_available} free uploads available", level='info')
+
+        if promo_free_upload_available > 0:
+            promo_action = PromoAction(
+                user_id = str(request.ctx.user.id),
+                user_internal_id=request.ctx.user.id,
+                action_type='freeUpload',
+                action_ref=str(encrypted_content_cid.content_hash),
+                created=datetime.now()
+            )
+            request.ctx.db_session.add(promo_action)
+
+            blockchain_task = BlockchainTask(
+                destination=platform.address.to_string(1, 1, 1),
+                amount=str(int(0.03 * 10 ** 9)),
+                payload=b64encode(
+                    begin_cell()
+                        .store_uint(0x5491d08c, 32)
+                        .store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
+                        .store_address(Address(await request.ctx.user.wallet_address_async(request.ctx.db_session)))
+                        .store_ref(
+                            begin_cell()
+                                .store_ref(
+                                    begin_cell()
+                                        .store_coins(int(0))
+                                        .store_coins(int(0))
+                                        .store_coins(int(request.json['price']))
+                                        .end_cell()
+                                )
+                                .store_maybe_ref(royalties_dict.end_dict())
+                                .store_uint(0, 1)
+                                .end_cell()
+                        )
+                        .store_ref(
+                            begin_cell()
+                                .store_ref(
+                                    begin_cell()
+                                        .store_bytes(f"{PROJECT_HOST}/api/v1.5/storage/{metadata_content.cid.serialize_v2(include_accept_type=True)}".encode())
+                                        .end_cell()
+                                )
+                                .store_ref(
+                                    begin_cell()
+                                        .store_ref(begin_cell().store_bytes(f"{encrypted_content_cid.serialize_v2()}".encode()).end_cell())
+                                        .store_ref(begin_cell().store_bytes(f"{image_content_cid.serialize_v2() if image_content_cid else ''}".encode()).end_cell())
+                                        .store_ref(begin_cell().store_bytes(f"{metadata_content.cid.serialize_v2()}".encode()).end_cell())
+                                        .end_cell()
+                                )
+                                .end_cell()
+                        )
+                        .end_cell().to_boc(False)
+                ).decode(),
+                epoch=None, seqno=None,
+                created = datetime.now(),
+                status='wait',
+                user_id = request.ctx.user.id
+            )
+            request.ctx.db_session.add(blockchain_task)
+            await request.ctx.db_session.commit()
+
+            await request.ctx.user_uploader_wrapper.send_message(
+                request.ctx.user.translated('p_uploadContentTxPromo').format(
+                    title=content_title,
+                    free_count=(promo_free_upload_available - 1)
+                ), message_type='hint', message_meta={
+                    'encrypted_content_hash': b58encode(encrypted_content_cid.content_hash).decode(),
+                    'hint_type': 'uploadContentTxRequested'
+                }
+            )
+            return response.json({
+                'address': "free",
+                'amount': str(int(0.03 * 10 ** 9)),
+                'payload': ""
+            })
+
         await request.ctx.user_uploader_wrapper.send_message(
             request.ctx.user.translated('p_uploadContentTxRequested').format(
                 title=content_title,
@@ -117,6 +235,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                 begin_cell()
                     .store_uint(0x5491d08c, 32)
                     .store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
+                    .store_uint(0, 2)
                     .store_ref(
                         begin_cell()
                             .store_ref(
@@ -164,15 +283,37 @@ async def s_api_v1_blockchain_send_purchase_content_message(request):
         assert field_key in request.json, f"No {field_key} provided"
         assert field_value(request.json[field_key]), f"Invalid {field_key} provided"
 
-    license_exist = request.ctx.db_session.query(UserContent).filter_by(
-        onchain_address=request.json['content_address'],
-    ).first()
+    if not (await request.ctx.user.wallet_address_async(request.ctx.db_session)):
+        return response.json({"error": "No wallet address provided"}, status=400)
+
+    from sqlalchemy import select
+    license_exist = (await request.ctx.db_session.execute(select(UserContent).where(
+        UserContent.onchain_address == request.json['content_address']
+    ))).scalars().first()
     if license_exist:
-        r_content = StoredContent.from_cid(request.ctx.db_session, license_exist.content.cid.serialize_v2())
+        from app.core.content.content_id import ContentId
+        _cid = ContentId.deserialize(license_exist.content.cid.serialize_v2())
+        r_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
     else:
-        r_content = StoredContent.from_cid(request.ctx.db_session, request.json['content_address'])
+        from app.core.content.content_id import ContentId
+        _cid = ContentId.deserialize(request.json['content_address'])
+        r_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
     
-    content = r_content.open_content(request.ctx.db_session)
+    async def open_content_async(session, sc: StoredContent):
+        if not sc.encrypted:
+            decrypted = sc
+            encrypted = (await session.execute(select(StoredContent).where(StoredContent.decrypted_content_id == sc.id))).scalars().first()
+        else:
+            encrypted = sc
+            decrypted = (await session.execute(select(StoredContent).where(StoredContent.id == sc.decrypted_content_id))).scalars().first()
+        assert decrypted and encrypted, "Can't open content"
+        ctype = decrypted.json_format().get('content_type', 'application/x-binary')
+        try:
+            content_type = ctype.split('/')[0]
+        except Exception:
+            content_type = 'application'
+        return {'encrypted_content': encrypted, 'decrypted_content': decrypted, 'content_type': content_type}
+    content = await open_content_async(request.ctx.db_session, r_content)
 
     licenses_cost = content['encrypted_content'].json_format()['license']
     assert request.json['license_type'] in licenses_cost

app/api/routes/_system.py

@@ -6,6 +6,7 @@ from base58 import b58encode, b58decode
 from sanic import response
 
 from app.core.models.node_storage import StoredContent
+from sqlalchemy import select
 from app.core._blockchain.ton.platform import platform
 from app.core._crypto.signer import Signer
 from app.core._secrets import hot_pubkey, service_wallet, hot_seed
@@ -19,10 +20,10 @@ def get_git_info():
 
 
 async def s_api_v1_node(request):  # /api/v1/node
-    last_known_index = request.ctx.db_session.query(StoredContent).filter(
-        StoredContent.onchain_index != None
-    ).order_by(StoredContent.onchain_index.desc()).first()
-    last_known_index = last_known_index.onchain_index if last_known_index else 0
+    last_known_index_obj = (await request.ctx.db_session.execute(
+        select(StoredContent).where(StoredContent.onchain_index != None).order_by(StoredContent.onchain_index.desc())
+    )).scalars().first()
+    last_known_index = last_known_index_obj.onchain_index if last_known_index_obj else 0
     last_known_index = max(last_known_index, 0)
     return response.json({
         'id': b58encode(hot_pubkey).decode(),
@@ -38,6 +39,27 @@ async def s_api_v1_node(request):  # /api/v1/node
         }
     })
 
+async def s_api_v1_node_friendly(request):
+    last_known_index_obj = (await request.ctx.db_session.execute(
+        select(StoredContent).where(StoredContent.onchain_index != None).order_by(StoredContent.onchain_index.desc())
+    )).scalars().first()
+    last_known_index = last_known_index_obj.onchain_index if last_known_index_obj else 0
+    last_known_index = max(last_known_index, 0)
+    response_plain_text = f"""
+    Node address: {service_wallet.address.to_string(1, 1, 1)}
+    Node ID: {b58encode(hot_pubkey).decode()}
+    Master address: {platform.address.to_string(1, 1, 1)}
+    Indexer height: {last_known_index}
+    Services:
+    """
+    for service_key, service in request.app.ctx.memory.known_states.items():
+        response_plain_text += f"""
+            {service_key}:
+                status: {service['status'] if (service['timestamp'] and (datetime.now() - service['timestamp']).total_seconds() < 120) else 'not working: timeout'}
+                delay: {round((datetime.now() - service['timestamp']).total_seconds(), 3) if service['timestamp'] else -1}
+        """
+    return response.text(response_plain_text, content_type='text/plain')
+
 
 async def s_api_system_send_status(request):
     if not request.json:

app/api/routes/admin.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+import os
+from sanic import response
+from sqlalchemy import select
+
+from app.core.models.my_network import KnownNode
+from app.core.models.content_v3 import EncryptedContent, IpfsSync, ContentDerivative
+from app.core.models._config import ServiceConfig
+from app.core.ipfs_client import bitswap_stat, repo_stat
+
+
+def _auth_ok(request) -> bool:
+    token = os.getenv('ADMIN_API_TOKEN')
+    if not token:
+        return False
+    auth = request.headers.get('Authorization', '')
+    return auth.strip() == f"Bearer {token}"
+
+
+async def s_api_v1_admin_node_setrole(request):
+    if not _auth_ok(request):
+        return response.json({"error": "UNAUTHORIZED"}, status=401)
+    data = request.json or {}
+    role = (data.get('role') or '').strip()
+    if role not in ('trusted', 'read-only', 'deny'):
+        return response.json({"error": "BAD_ROLE"}, status=400)
+    pub = (data.get('public_key') or '').strip()
+    host = (data.get('host') or '').strip()
+    if not pub and not host:
+        return response.json({"error": "MISSING_TARGET"}, status=400)
+    session = request.ctx.db_session
+    row = None
+    if pub:
+        row = (await session.execute(select(KnownNode).where(KnownNode.public_key == pub))).scalars().first()
+    if not row and host:
+        row = (await session.execute(select(KnownNode).where(KnownNode.ip == host))).scalars().first()
+    if not row:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    meta = row.meta or {}
+    meta['role'] = role
+    row.meta = meta
+    await session.commit()
+    return response.json({"ok": True, "node": {"ip": row.ip, "public_key": row.public_key, "role": role}})
+
+
+async def s_api_v1_admin_nodes(request):
+    if not _auth_ok(request):
+        return response.json({"error": "UNAUTHORIZED"}, status=401)
+    session = request.ctx.db_session
+    rows = (await session.execute(select(KnownNode))).scalars().all()
+    items = []
+    for r in rows:
+        meta = r.meta or {}
+        items.append({
+            "ip": r.ip,
+            "port": r.port,
+            "public_key": r.public_key,
+            "role": meta.get('role') or 'read-only',
+            "version": meta.get('version'),
+            "last_seen": (r.last_sync.isoformat() + 'Z') if r.last_sync else None,
+        })
+    return response.json({"items": items})
+
+
+async def s_api_v1_admin_status(request):
+    if not _auth_ok(request):
+        return response.json({"error": "UNAUTHORIZED"}, status=401)
+    session = request.ctx.db_session
+    # DB metrics
+    pin_counts = {k: 0 for k in ('not_pinned','queued','pinning','pinned','failed')}
+    rows = (await session.execute(select(IpfsSync))).scalars().all()
+    for r in rows:
+        pin_counts[r.pin_state] = pin_counts.get(r.pin_state, 0) + 1
+    deriv = (await session.execute(select(ContentDerivative))).scalars().all()
+    deriv_counts = {
+        'ready': sum(1 for d in deriv if d.status=='ready'),
+        'processing': sum(1 for d in deriv if d.status=='processing'),
+        'pending': sum(1 for d in deriv if d.status=='pending'),
+        'failed': sum(1 for d in deriv if d.status=='failed'),
+    }
+    total_deriv_bytes = sum(int(d.size_bytes or 0) for d in deriv)
+    # Backlog: number of EC needing conversion
+    ec = (await session.execute(select(EncryptedContent))).scalars().all()
+    backlog = 0
+    for e in ec:
+        if not e.preview_enabled:
+            continue
+        kinds = [d.kind for d in deriv if d.content_id==e.id and d.status=='ready']
+        req = {'decrypted_low','decrypted_high','decrypted_preview'}
+        if not req.issubset(set(kinds)):
+            backlog += 1
+    # IPFS metrics
+    try:
+        bs = await bitswap_stat()
+    except Exception:
+        bs = {}
+    try:
+        rs = await repo_stat()
+    except Exception:
+        rs = {}
+    # Limits
+    cfg = ServiceConfig(session)
+    max_gb = await cfg.get('DERIVATIVE_CACHE_MAX_GB', os.getenv('DERIVATIVE_CACHE_MAX_GB','50'))
+    ttl_days = await cfg.get('DERIVATIVE_CACHE_TTL_DAYS', os.getenv('DERIVATIVE_CACHE_TTL_DAYS','0'))
+    max_pins = await cfg.get('SYNC_MAX_CONCURRENT_PINS', os.getenv('SYNC_MAX_CONCURRENT_PINS','4'))
+    disk_pct = await cfg.get('SYNC_DISK_LOW_WATERMARK_PCT', os.getenv('SYNC_DISK_LOW_WATERMARK_PCT','90'))
+    return response.json({
+        'ipfs': {'bitswap': bs, 'repo': rs},
+        'pin_counts': pin_counts,
+        'derivatives': {**deriv_counts, 'total_bytes': total_deriv_bytes},
+        'convert_backlog': backlog,
+        'limits': {
+            'DERIVATIVE_CACHE_MAX_GB': float(max_gb),
+            'DERIVATIVE_CACHE_TTL_DAYS': int(ttl_days),
+            'SYNC_MAX_CONCURRENT_PINS': int(max_pins),
+            'SYNC_DISK_LOW_WATERMARK_PCT': int(disk_pct),
+        }
+    })
+
+
+async def s_api_v1_admin_cache_setlimits(request):
+    if not _auth_ok(request):
+        return response.json({"error": "UNAUTHORIZED"}, status=401)
+    data = request.json or {}
+    max_gb = float(data.get('max_gb'))
+    ttl_days = int(data.get('ttl_days'))
+    cfg = ServiceConfig(request.ctx.db_session)
+    await cfg.set('DERIVATIVE_CACHE_MAX_GB', max_gb)
+    await cfg.set('DERIVATIVE_CACHE_TTL_DAYS', ttl_days)
+    return response.json({"ok": True})
+
+
+async def s_api_v1_admin_cache_cleanup(request):
+    if not _auth_ok(request):
+        return response.json({"error": "UNAUTHORIZED"}, status=401)
+    data = request.json or {}
+    mode = (data.get('mode') or 'fit')
+    # reuse janitor logic inline
+    removed = 0
+    from datetime import datetime, timedelta
+    if mode == 'ttl':
+        # Evict by TTL
+        async with request.ctx.db_session as session:
+            from app.core.models._config import ServiceConfig
+            ttl = int(await ServiceConfig(session).get('DERIVATIVE_CACHE_TTL_DAYS', os.getenv('DERIVATIVE_CACHE_TTL_DAYS','0')))
+        if ttl > 0:
+            now = datetime.utcnow()
+            rows = (await request.ctx.db_session.execute(select(ContentDerivative).where(ContentDerivative.status=='ready'))).scalars().all()
+            for r in rows:
+                la = r.last_access_at or r.created_at
+                if la and (now - la) > timedelta(days=ttl):
+                    try:
+                        if r.local_path and os.path.exists(r.local_path):
+                            os.remove(r.local_path)
+                    except Exception:
+                        pass
+                    r.status='pending'; r.local_path=None; r.size_bytes=None; r.last_access_at=None
+                    removed += 1
+            await request.ctx.db_session.commit()
+    else:
+        # Fit to size
+        target_gb = float(data.get('max_gb') or 0)
+        if target_gb <= 0:
+            return response.json({"error": "BAD_MAX_GB"}, status=400)
+        limit_bytes = int(target_gb * (1024**3))
+        rows = (await request.ctx.db_session.execute(select(ContentDerivative).where(ContentDerivative.status=='ready'))).scalars().all()
+        rows.sort(key=lambda r: (r.last_access_at or r.created_at or datetime.utcfromtimestamp(0)))
+        total = sum(int(r.size_bytes or 0) for r in rows)
+        for r in rows:
+            if total <= limit_bytes:
+                break
+            try:
+                if r.local_path and os.path.exists(r.local_path):
+                    os.remove(r.local_path)
+            except Exception:
+                pass
+            total -= int(r.size_bytes or 0)
+            r.status='pending'; r.local_path=None; r.size_bytes=None; r.last_access_at=None
+            removed += 1
+        await request.ctx.db_session.commit()
+    return response.json({"ok": True, "removed": removed})
+
+
+async def s_api_v1_admin_sync_setlimits(request):
+    if not _auth_ok(request):
+        return response.json({"error": "UNAUTHORIZED"}, status=401)
+    data = request.json or {}
+    max_pins = int(data.get('max_concurrent_pins'))
+    disk_pct = int(data.get('disk_low_watermark_pct'))
+    cfg = ServiceConfig(request.ctx.db_session)
+    await cfg.set('SYNC_MAX_CONCURRENT_PINS', max_pins)
+    await cfg.set('SYNC_DISK_LOW_WATERMARK_PCT', disk_pct)
+    return response.json({"ok": True})

app/api/routes/auth.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from uuid import uuid4
 
 from aiogram.utils.web_app import safe_parse_webapp_init_data
 from sanic import response
@@ -36,7 +37,9 @@ async def s_api_v1_auth_twa(request):
         make_log("auth", "Invalid TWA data", level="warning")
         return response.json({"error": "Invalid TWA data"}, status=401)
 
-    known_user = request.ctx.db_session.query(User).filter(User.telegram_id == twa_data.user.id).first()
+    known_user = (await request.ctx.db_session.execute(
+        select(User).where(User.telegram_id == twa_data.user.id)
+    )).scalars().first()
     if not known_user:
         new_user = User(
             telegram_id=twa_data.user.id,
@@ -51,9 +54,11 @@ async def s_api_v1_auth_twa(request):
             created=datetime.now()
         )
         request.ctx.db_session.add(new_user)
-        request.ctx.db_session.commit()
+        await request.ctx.db_session.commit()
 
-        known_user = request.ctx.db_session.query(User).filter(User.telegram_id == twa_data.user.id).first()
+        known_user = (await request.ctx.db_session.execute(
+            select(User).where(User.telegram_id == twa_data.user.id)
+        )).scalars().first()
         assert known_user, "User not created"
 
     new_user_key = await known_user.create_api_token_v1(request.ctx.db_session, "USER_API_V1")
@@ -64,12 +69,12 @@ async def s_api_v1_auth_twa(request):
             wallet_info.account = Account.from_dict(auth_data['ton_proof']['account'])
             wallet_info.ton_proof = TonProof.from_dict({'proof': auth_data['ton_proof']['ton_proof']})
             connection_payload = auth_data['ton_proof']['ton_proof']['payload']
-            known_payload = (request.ctx.db_session.execute(select(KnownKey).where(KnownKey.seed == connection_payload))).scalars().first()
+            known_payload = (await request.ctx.db_session.execute(select(KnownKey).where(KnownKey.seed == connection_payload))).scalars().first()
             assert known_payload, "Unknown payload"
             assert known_payload.meta['I_user_id'] == known_user.id, "Invalid user_id"
             assert wallet_info.check_proof(connection_payload), "Invalid proof"
 
-            for known_connection in (request.ctx.db_session.execute(select(WalletConnection).where(
+            for known_connection in (await request.ctx.db_session.execute(select(WalletConnection).where(
                     and_(
                         WalletConnection.user_id == known_user.id,
                         WalletConnection.network == 'ton'
@@ -77,7 +82,7 @@ async def s_api_v1_auth_twa(request):
             ))).scalars().all():
                 known_connection.invalidated = True
 
-            for other_connection in (request.ctx.db_session.execute(select(WalletConnection).where(
+            for other_connection in (await request.ctx.db_session.execute(select(WalletConnection).where(
                     WalletConnection.wallet_address == Address(wallet_info.account.address).to_string(1, 1, 1)
             ))).scalars().all():
                 other_connection.invalidated = True
@@ -98,24 +103,94 @@ async def s_api_v1_auth_twa(request):
                 without_pk=False
             )
             request.ctx.db_session.add(new_connection)
-            request.ctx.db_session.commit()
+            await request.ctx.db_session.commit()
         except BaseException as e:
             make_log("auth", f"Invalid ton_proof: {e}", level="warning")
             return response.json({"error": "Invalid ton_proof"}, status=400)
 
-    ton_connection = (request.ctx.db_session.execute(select(WalletConnection).where(
+    ton_connection = (await request.ctx.db_session.execute(select(WalletConnection).where(
         and_(
             WalletConnection.user_id == known_user.id,
             WalletConnection.network == 'ton',
             WalletConnection.invalidated == False
         )
-    ))).scalars().first()
-
+    ).order_by(WalletConnection.created.desc()))).scalars().first()
     known_user.last_use = datetime.now()
-    request.ctx.db_session.commit()
+    await request.ctx.db_session.commit()
 
     return response.json({
         'user': known_user.json_format(),
         'connected_wallet': ton_connection.json_format() if ton_connection else None,
         'auth_v1_token': new_user_key['auth_v1_token']
     })
+
+async def s_api_v1_auth_me(request):
+    if not request.ctx.user:
+        return response.json({"error": "Unauthorized"}, status=401)
+
+    ton_connection = (await request.ctx.db_session.execute(
+        select(WalletConnection).where(
+            and_(
+                WalletConnection.user_id == request.ctx.user.id,
+                WalletConnection.network == 'ton',
+                WalletConnection.invalidated == False
+            )
+        ).order_by(WalletConnection.created.desc())
+    )).scalars().first()
+
+    return response.json({
+        'user': request.ctx.user.json_format(),
+        'connected_wallet': ton_connection.json_format() if ton_connection else None
+    })
+
+async def s_api_v1_auth_select_wallet(request):
+    if not request.ctx.user:
+        return response.json({"error": "Unauthorized"}, status=401)
+
+    try:
+        data = request.json
+    except Exception as e:
+        return response.json({"error": "Invalid JSON"}, status=400)
+
+    if "wallet_address" not in data:
+        return response.json({"error": "wallet_address is required"}, status=400)
+
+    # Convert raw wallet address to canonical format using Address from tonsdk.utils
+    raw_addr = data["wallet_address"]
+    canonical_address = Address(raw_addr).to_string(1, 1, 1)
+
+    db_session = request.ctx.db_session
+    user = request.ctx.user
+
+    # Check if a WalletConnection already exists for this user with the given canonical wallet address
+    existing_connection = (await db_session.execute(select(WalletConnection).where(
+        and_(
+            WalletConnection.user_id == user.id,
+            WalletConnection.wallet_address == canonical_address
+        )
+    ))).scalars().first()
+
+    if not existing_connection:
+        return response.json({"error": "Wallet connection not found"}, status=404)
+
+    saved_values = {
+        'keys': existing_connection.keys,
+        'meta': existing_connection.meta,
+        'wallet_key': existing_connection.wallet_key,
+        'connection_id': existing_connection.connection_id + uuid4().hex,
+        'network': existing_connection.network,
+    }
+    
+    new_connection = WalletConnection(
+        **saved_values,
+        user_id=user.id,
+        wallet_address=canonical_address,
+        created=datetime.now(),
+        updated=datetime.now(),
+        invalidated=False,
+        without_pk=False
+    )
+    db_session.add(new_connection)
+    await db_session.commit()
+
+    return response.empty(status=200)

app/api/routes/content.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta
 from sanic import response
+from sqlalchemy import select, and_, func
 from aiogram import Bot, types
 from sqlalchemy import and_
 from app.core.logger import make_log
@@ -9,6 +10,7 @@ from app.core.models.keys import KnownKey
 from app.core.models import StarsInvoice
 from app.core.models.content.user_content import UserContent
 from app.core._config import CLIENT_TELEGRAM_API_KEY, PROJECT_HOST
+from app.core.models.content_v3 import EncryptedContent as ECv3, ContentDerivative as CDv3
 import json
 import uuid
 
@@ -22,13 +24,20 @@ async def s_api_v1_content_list(request):
     store = request.args.get('store', 'local')
     assert store in ('local', 'onchain'), "Invalid store"
 
-    content_list = request.ctx.db_session.query(StoredContent).filter(
-        StoredContent.type.like(store + '%'),
-        StoredContent.disabled == False
-    ).order_by(StoredContent.created.desc()).offset(offset).limit(limit)
-    make_log("Content", f"Listed {content_list.count()} contents", level='info')
+    stmt = (
+        select(StoredContent)
+        .where(
+            StoredContent.type.like(store + '%'),
+            StoredContent.disabled == False
+        )
+        .order_by(StoredContent.created.desc())
+        .offset(offset)
+        .limit(limit)
+    )
+    rows = (await request.ctx.db_session.execute(stmt)).scalars().all()
+    make_log("Content", f"Listed {len(rows)} contents", level='info')
     result = {}
-    for content in content_list.all():
+    for content in rows:
         content_json = content.json_format()
         result[content_json["cid"]] = content_json
 
@@ -38,23 +47,41 @@ async def s_api_v1_content_list(request):
 async def s_api_v1_content_view(request, content_address: str):
     # content_address can be CID or TON address
 
-    license_exist = request.ctx.db_session.query(UserContent).filter_by(
-        onchain_address=content_address,
-    ).first()
+    license_exist = (await request.ctx.db_session.execute(
+        select(UserContent).where(UserContent.onchain_address == content_address)
+    )).scalars().first()
     if license_exist:
         content_address = license_exist.content.cid.serialize_v2()
 
-    r_content = StoredContent.from_cid(request.ctx.db_session, content_address)
-    content = r_content.open_content(request.ctx.db_session)
+    from app.core.content.content_id import ContentId
+    cid = ContentId.deserialize(content_address)
+    r_content = (await request.ctx.db_session.execute(
+        select(StoredContent).where(StoredContent.hash == cid.content_hash_b58)
+    )).scalars().first()
+    async def open_content_async(session, sc: StoredContent):
+        if not sc.encrypted:
+            decrypted = sc
+            encrypted = (await session.execute(select(StoredContent).where(StoredContent.decrypted_content_id == sc.id))).scalars().first()
+        else:
+            encrypted = sc
+            decrypted = (await session.execute(select(StoredContent).where(StoredContent.id == sc.decrypted_content_id))).scalars().first()
+        assert decrypted and encrypted, "Can't open content"
+        ctype = decrypted.json_format().get('content_type', 'application/x-binary')
+        try:
+            content_type = ctype.split('/')[0]
+        except Exception:
+            content_type = 'application'
+        return {'encrypted_content': encrypted, 'decrypted_content': decrypted, 'content_type': content_type}
+    content = await open_content_async(request.ctx.db_session, r_content)
 
     opts = {
         'content_type': content['content_type'],  # возможно с ошибками, нужно переделать на ffprobe
         'content_address': content['encrypted_content'].meta.get('item_address', '')
     }
     if content['encrypted_content'].key_id:
-        known_key = request.ctx.db_session.query(KnownKey).filter(
-            KnownKey.id == content['encrypted_content'].key_id
-        ).first()
+        known_key = (await request.ctx.db_session.execute(
+            select(KnownKey).where(KnownKey.id == content['encrypted_content'].key_id)
+        )).scalars().first()
         if known_key:
             opts['key_hash'] = known_key.seed_hash  # нахер не нужно на данный момент
 
@@ -64,22 +91,23 @@ async def s_api_v1_content_view(request, content_address: str):
 
     have_access = False
     if request.ctx.user:
-        user_wallet_address = request.ctx.user.wallet_address(request.ctx.db_session)
+        user_wallet_address = await request.ctx.user.wallet_address_async(request.ctx.db_session)
         have_access = (
                 (content['encrypted_content'].owner_address == user_wallet_address)
-                or bool(request.ctx.db_session.query(UserContent).filter_by(owner_address=user_wallet_address, status='active',
-                                                                     content_id=content['encrypted_content'].id).first()) \
-                or bool(request.ctx.db_session.query(StarsInvoice).filter(
+                or bool((await request.ctx.db_session.execute(select(UserContent).where(
+                    and_(UserContent.owner_address == user_wallet_address, UserContent.status == 'active', UserContent.content_id == content['encrypted_content'].id)
+                ))).scalars().first()) \
+                or bool((await request.ctx.db_session.execute(select(StarsInvoice).where(
                     and_(
                         StarsInvoice.user_id == request.ctx.user.id,
                         StarsInvoice.content_hash == content['encrypted_content'].hash,
                         StarsInvoice.paid == True
                     )
-                ).first())
+                ))).scalars().first())
         )
 
         if not have_access:
-            current_star_rate = ServiceConfig(request.ctx.db_session).get('live_tonPerStar', [0, 0])[0]
+            current_star_rate = (await ServiceConfig(request.ctx.db_session).get('live_tonPerStar', [0, 0]))[0]
             if current_star_rate < 0:
                 current_star_rate = 0.00000001
 
@@ -88,14 +116,14 @@ async def s_api_v1_content_view(request, content_address: str):
                 stars_cost = 2
 
             invoice_id = f"access_{uuid.uuid4().hex}"
-            exist_invoice = request.ctx.db_session.query(StarsInvoice).filter(
+            exist_invoice = (await request.ctx.db_session.execute(select(StarsInvoice).where(
                 and_(
                     StarsInvoice.user_id == request.ctx.user.id,
                     StarsInvoice.created > datetime.now() - timedelta(minutes=25),
                     StarsInvoice.amount == stars_cost,
                     StarsInvoice.content_hash == content['encrypted_content'].hash,
                 )
-            ).first()
+            ))).scalars().first()
             if exist_invoice:
                 invoice_url = exist_invoice.invoice_url
             else:
@@ -119,7 +147,7 @@ async def s_api_v1_content_view(request, content_address: str):
                             invoice_url=invoice_url
                         )
                     )
-                    request.ctx.db_session.commit()
+                    await request.ctx.db_session.commit()
                 except BaseException as e:
                     make_log("Content", f"Can't create invoice link: {e}", level='warning')
 
@@ -140,25 +168,53 @@ async def s_api_v1_content_view(request, content_address: str):
     if converted_content:
         user_content_option = 'low_preview'
         if have_access:
-            user_content_option = 'low'  # TODO: подключать high если человек внезапно меломан
+            user_content_option = 'low'
 
-        converted_content = request.ctx.db_session.query(StoredContent).filter(
+        converted_content = (await request.ctx.db_session.execute(select(StoredContent).where(
             StoredContent.hash == converted_content[user_content_option]
-        ).first()
+        ))).scalars().first()
         if converted_content:
             display_options['content_url'] = converted_content.web_url
+            opts['content_ext'] = converted_content.filename.split('.')[-1]
+    else:
+        # v3 fallback: use derivatives table linked via encrypted_cid from onchain meta
+        enc_cid = content['encrypted_content'].meta.get('content_cid') or content['encrypted_content'].meta.get('encrypted_cid')
+        if enc_cid:
+            ec = (await request.ctx.db_session.execute(select(ECv3).where(ECv3.encrypted_cid == enc_cid))).scalars().first()
+            if ec:
+                # choose preview for non-access; low for access
+                desired = ['decrypted_preview'] if not have_access else ['decrypted_low', 'decrypted_high']
+                rows = (await request.ctx.db_session.execute(select(CDv3).where(CDv3.content_id == ec.id, CDv3.status == 'ready'))).scalars().all()
+                chosen = None
+                for kind in desired:
+                    chosen = next((r for r in rows if r.kind == kind), None)
+                    if chosen:
+                        break
+                if chosen and chosen.local_path:
+                    h = chosen.local_path.split('/')[-1]
+                    display_options['content_url'] = f"{PROJECT_HOST}/api/v1.5/storage/{h}"
+                    opts['content_ext'] = (chosen.content_type or '').split('/')[-1] if chosen.content_type else None
 
     content_meta = content['encrypted_content'].json_format()
-    content_metadata = StoredContent.from_cid(request.ctx.db_session, content_meta.get('metadata_cid') or None)
+    from app.core.content.content_id import ContentId
+    _mcid = content_meta.get('metadata_cid') or None
+    content_metadata = None
+    if _mcid:
+        _cid = ContentId.deserialize(_mcid)
+        content_metadata = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
     with open(content_metadata.filepath, 'r') as f:
         content_metadata_json = json.loads(f.read())
 
     display_options['metadata'] = content_metadata_json
+    opts['downloadable'] = content_metadata_json.get('downloadable', False)
+    if opts['downloadable']:
+        if not ('listen' in opts['have_licenses']):
+            opts['downloadable'] = False
 
     return response.json({
         **opts,
         'encrypted': content['encrypted_content'].json_format(),
-        'display_options': display_options
+        'display_options': display_options,
     })
     
 
@@ -182,14 +238,17 @@ async def s_api_v1_content_friendly_list(request):
                     </tr>
                 </thead>
 """
-    for content in request.ctx.db_session.query(StoredContent).filter(
+    contents = (await request.ctx.db_session.execute(select(StoredContent).where(
         StoredContent.type == 'onchain/content'
-    ).all():
+    ))).scalars().all()
+    for content in contents:
         if not content.meta.get('metadata_cid'):
             make_log("Content", f"Content {content.cid.serialize_v2()} has no metadata", level='warning')
             continue
 
-        metadata_content = StoredContent.from_cid(request.ctx.db_session, content.meta.get('metadata_cid'))
+        from app.core.content.content_id import ContentId
+        _cid = ContentId.deserialize(content.meta.get('metadata_cid'))
+        metadata_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
         with open(metadata_content.filepath, 'r') as f:
             metadata = json.loads(f.read())
         
@@ -223,10 +282,12 @@ async def s_api_v1_5_content_list(request):
         return response.json({'error': 'Invalid limit'}, status=400)
 
     # Query onchain contents which are not disabled
-    contents = request.ctx.db_session.query(StoredContent).filter(
-        StoredContent.type == 'onchain/content',
-        StoredContent.disabled == False
-    ).order_by(StoredContent.created.desc()).offset(offset).limit(limit).all()
+    contents = (await request.ctx.db_session.execute(
+        select(StoredContent)
+        .where(StoredContent.type == 'onchain/content', StoredContent.disabled == False)
+        .order_by(StoredContent.created.desc())
+        .offset(offset).limit(limit)
+    )).scalars().all()
 
     result = []
     for content in contents:
@@ -235,7 +296,9 @@ async def s_api_v1_5_content_list(request):
         if not metadata_cid:
             continue  # Skip if no metadata_cid is found
 
-        metadata_content = StoredContent.from_cid(request.ctx.db_session, metadata_cid)
+        from app.core.content.content_id import ContentId
+        _cid = ContentId.deserialize(metadata_cid)
+        metadata_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
         try:
             with open(metadata_content.filepath, 'r') as f:
                 metadata = json.load(f)
@@ -251,9 +314,9 @@ async def s_api_v1_5_content_list(request):
         preview_link = None
         converted_content = content.meta.get('converted_content')
         if converted_content:
-            converted_content = request.ctx.db_session.query(StoredContent).filter(
+            converted_content = (await request.ctx.db_session.execute(select(StoredContent).where(
                 StoredContent.hash == converted_content['low_preview']
-            ).first()
+            ))).scalars().first()
             preview_link = converted_content.web_url
             if converted_content.filename.split('.')[-1] in ('mp4', 'mov'):
                 media_type = 'video'

app/api/routes/content_index.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from sanic import response
+from sqlalchemy import select
+from datetime import datetime
+
+from app.core.models.content_v3 import ContentIndexItem
+from app.core.logger import make_log
+
+
+async def s_api_v1_content_index(request):
+    rows = (await request.ctx.db_session.execute(select(ContentIndexItem))).scalars().all()
+    items = [{**r.payload, "encrypted_cid": r.encrypted_cid, "sig": r.sig, "_updated_at": (r.updated_at.isoformat() + 'Z') if r.updated_at else None} for r in rows]
+    # ETag by max updated_at + count
+    max_ts = max((it.get("_updated_at") for it in items if it.get("_updated_at")), default="1970-01-01T00:00:00Z")
+    etag = f'W/"{max_ts}.{len(items)}"'
+    inm = request.headers.get('If-None-Match')
+    if inm and inm == etag:
+        resp = response.empty(status=304)
+        resp.headers['ETag'] = etag
+        return resp
+    for it in items:
+        it.pop("_updated_at", None)
+    make_log("content.index", f"items={len(items)} etag={etag}")
+    resp = response.json({"items": items, "schema": "my-network/index@1"})
+    resp.headers['ETag'] = etag
+    return resp
+
+
+async def s_api_v1_content_delta(request):
+    since = request.args.get('since')
+    if not since:
+        # No since provided → act as full index
+        return await s_api_v1_content_index(request)
+    try:
+        # basic parse
+        _ = datetime.fromisoformat(since.replace('Z', '+00:00'))
+    except Exception:
+        return response.json({"error": "BAD_SINCE"}, status=400)
+
+    rows = (await request.ctx.db_session.execute(select(ContentIndexItem))).scalars().all()
+    out = []
+    max_ts = since
+    for r in rows:
+        upd = (r.updated_at.isoformat() + 'Z') if r.updated_at else None
+        if upd and upd > since:
+            out.append({**r.payload, "encrypted_cid": r.encrypted_cid, "sig": r.sig})
+            if upd > max_ts:
+                max_ts = upd
+    resp = response.json({"items": out, "next_since": max_ts, "schema": "my-network/index@1"})
+    # Weak ETag for delta response
+    resp.headers['ETag'] = f'W/"{max_ts}.{len(out)}"'
+    return resp

app/api/routes/derivatives.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from sanic import response
+from sqlalchemy import select
+
+from app.core.models.content_v3 import EncryptedContent, ContentDerivative
+from app.core._config import PROJECT_HOST
+
+
+async def s_api_v1_content_derivatives(request):
+    cid = request.args.get('cid')
+    if not cid:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+    session = request.ctx.db_session
+    ec = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+    if not ec:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id))).scalars().all()
+    out = []
+    for r in rows:
+        # Derive /api/v1.5/storage/<hash> from local_path if possible
+        path_hash = (r.local_path or '').split('/')[-1]
+        storage_url = f"{PROJECT_HOST}/api/v1.5/storage/{path_hash}" if path_hash else None
+        out.append({
+            'kind': r.kind,
+            'interval': [r.interval_start_ms, r.interval_end_ms] if r.interval_start_ms is not None else None,
+            'content_type': r.content_type,
+            'size_bytes': r.size_bytes,
+            'status': r.status,
+            'url': storage_url,
+        })
+    return response.json({'cid': cid, 'derivatives': out})
+

app/api/routes/keys.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import base64
+import json
+import os
+from datetime import datetime
+from typing import Dict, Any
+
+from base58 import b58encode
+from sanic import response
+from sqlalchemy import select
+
+from app.core._secrets import hot_pubkey
+from app.core.logger import make_log
+from app.core.models.content_v3 import EncryptedContent, ContentKey, KeyGrant
+from app.core.network.nodesig import verify_request
+from app.core.network.guard import check_rate_limit
+from app.core.models.my_network import KnownNode
+from app.core.crypto.keywrap import unwrap_dek, KeyWrapError
+
+
+def _b64(b: bytes) -> str:
+    return base64.b64encode(b).decode()
+
+
+async def s_api_v1_keys_request(request):
+    # Rate limit per remote IP (reuse handshake limiter)
+    remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip()
+    if not check_rate_limit(request.app.ctx.memory, remote_ip):
+        return response.json({"error": "RATE_LIMIT"}, status=429)
+
+    # Verify NodeSig
+    ok, hdr_node, reason = verify_request(request, request.app.ctx.memory)
+    if not ok:
+        return response.json({"error": reason or "UNAUTHORIZED"}, status=401)
+
+    data: Dict[str, Any] = request.json or {}
+    cid = data.get("encrypted_cid")
+    requester_node = data.get("requestor_node_id")
+    recipient_box_pub_b64 = data.get("recipient_box_pub")
+    if not cid or not requester_node or not recipient_box_pub_b64:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+
+    if requester_node != hdr_node:
+        return response.json({"error": "NODE_ID_MISMATCH"}, status=401)
+
+    session = request.ctx.db_session
+    row = (await session.execute(select(EncryptedContent, ContentKey).join(ContentKey, ContentKey.content_id == EncryptedContent.id).where(EncryptedContent.encrypted_cid == cid))).first()
+    if not row:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    ec: EncryptedContent = row[0]
+    ck: ContentKey = row[1]
+    # Allow only trusted nodes unless explicitly disabled via env
+    TRUSTED_ONLY = (os.getenv('KEY_AUTO_GRANT_TRUSTED_ONLY', '1') == '1')
+    if TRUSTED_ONLY:
+        kn = (await session.execute(select(KnownNode).where(KnownNode.public_key == requester_node))).scalars().first()
+        role = (kn.meta or {}).get('role') if kn else None
+        if role != 'trusted':
+            return response.json({"error": "DENIED_NOT_TRUSTED"}, status=403)
+    if not ck.allow_auto_grant:
+        return response.json({"error": "DENIED"}, status=403)
+
+    # Seal the DEK for recipient using libsodium sealed box
+    try:
+        dek_plain = unwrap_dek(ck.key_ciphertext_b64)
+        import nacl.public
+        pk = nacl.public.PublicKey(base64.b64decode(recipient_box_pub_b64))
+        box = nacl.public.SealedBox(pk)
+        sealed = box.encrypt(dek_plain)
+        sealed_b64 = _b64(sealed)
+    except KeyWrapError as e:
+        make_log("keys", f"unwrap failed: {e}", level="error")
+        return response.json({"error": "KEY_UNWRAP_FAILED"}, status=500)
+    except Exception as e:
+        make_log("keys", f"seal failed: {e}", level="error")
+        return response.json({"error": "SEAL_FAILED"}, status=500)
+
+    issuer = b58encode(hot_pubkey).decode()
+    purpose = (data.get('purpose') or 'full')
+    ttl_sec = int(os.getenv('KEY_GRANT_PREVIEW_TTL_SEC', '0')) if purpose == 'preview' else 0
+    grant_body = {
+        "encrypted_cid": cid,
+        "to_node_id": requester_node,
+        "sealed_key_b64": sealed_b64,
+        "aead_scheme": ec.aead_scheme,
+        "chunk_bytes": ec.chunk_bytes,
+        "constraints": {"ttl_sec": ttl_sec, "scope": purpose},
+        "issued_at": datetime.utcnow().isoformat(),
+        "issuer_node_id": issuer,
+    }
+    try:
+        from app.core._crypto.signer import Signer
+        from app.core._secrets import hot_seed
+        signer = Signer(hot_seed)
+        blob = json.dumps(grant_body, sort_keys=True, separators=(",", ":")).encode()
+        sig = signer.sign(blob)
+    except Exception:
+        sig = ""
+
+    grant = KeyGrant(
+        encrypted_cid=cid,
+        issuer_node_id=issuer,
+        to_node_id=requester_node,
+        sealed_key_b64=sealed_b64,
+        aead_scheme=ec.aead_scheme,
+        chunk_bytes=ec.chunk_bytes,
+        constraints={"ttl_sec": 0, "scope": "full"},
+        sig=sig,
+    )
+    session.add(grant)
+    await session.commit()
+    grant_row = {
+        **grant_body,
+        "sig": sig,
+        "grant_id": grant.id,
+    }
+    return response.json(grant_row)

app/api/routes/network.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from typing import Dict, Any
+
+from base58 import b58decode
+from sanic import response
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.models.my_network import KnownNode
+from app.core.network.constants import CURRENT_PROTOCOL_VERSION, NODE_TYPE_PRIVATE
+from app.core.network.config import NODE_PRIVACY
+from app.core.network.handshake import build_handshake_payload, compute_node_info, sign_response
+from app.core.network.nodes import upsert_known_node, list_known_public_nodes
+from app.core.network.semver import compatibility
+from app.core.network.guard import check_rate_limit, check_timestamp_fresh, check_and_remember_nonce
+from app.core.network.config import HANDSHAKE_TS_TOLERANCE_SEC
+
+
+async def s_api_v1_network_info(request):
+    async with request.app.ctx.memory.transaction("network.info"):
+        node = await compute_node_info(request.ctx.db_session)
+    make_log("Network", "info served")
+    return response.json({"node": node})
+
+
+async def s_api_v1_network_nodes(request):
+    rows = await list_known_public_nodes(request.ctx.db_session)
+    make_log("Network", f"nodes list count={len(rows)}")
+    return response.json({
+        "count": len(rows),
+        "nodes": rows,
+    })
+
+
+async def s_api_v1_network_handshake(request):
+    # Handshake accepted regardless of our privacy; private nodes typically have no external endpoint
+
+    # Rate limit per remote IP
+    remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip()
+    if not check_rate_limit(request.app.ctx.memory, remote_ip):
+        return response.json({"error": "RATE_LIMIT"}, status=429)
+
+    data = request.json or {}
+    required = ["version", "public_key", "node_type", "metrics", "timestamp", "signature"]
+    for f in required:
+        if f not in data:
+            return response.json({"error": f"Missing field {f}"}, status=400)
+    # public_host is required for public nodes only
+    if data.get("node_type") != "private" and not data.get("public_host"):
+        return response.json({"error": "Missing field public_host"}, status=400)
+
+    # Timestamp freshness
+    if not check_timestamp_fresh(data.get("timestamp")):
+        return response.json({"error": "STALE_TIMESTAMP", "tolerance_sec": HANDSHAKE_TS_TOLERANCE_SEC}, status=400)
+
+    # Nonce replay protection (best-effort)
+    if not data.get("nonce") or not check_and_remember_nonce(request.app.ctx.memory, data.get("public_key"), data.get("nonce")):
+        return response.json({"error": "NONCE_REPLAY"}, status=400)
+
+    peer_version = str(data.get("version"))
+    comp = compatibility(peer_version, CURRENT_PROTOCOL_VERSION)
+    if comp == "blocked":
+        # We still store the node but respond with 409
+        try:
+            await upsert_known_node(
+                request.ctx.db_session,
+                host=data.get("public_host"),
+                port=int(str(data.get("public_host") or "").split(":")[-1]) if ":" in str(data.get("public_host") or "") else 80,
+                public_key=str(data.get("public_key")),
+                meta={
+                    "version": peer_version,
+                    "compatibility": comp,
+                    "is_public": data.get("node_type", "public") != "private",
+                    "public_host": data.get("public_host"),
+                    "unsupported_last_checked_at": datetime.utcnow().isoformat(),
+                }
+            )
+        except Exception:
+            pass
+        make_log("Handshake", f"Reject incompatible peer {data.get('public_host')} peer={peer_version} current={CURRENT_PROTOCOL_VERSION}")
+        return response.json({
+            "error": "INCOMPATIBLE_VERSION",
+            "compatibility": comp,
+            "current": CURRENT_PROTOCOL_VERSION,
+            "peer": peer_version,
+        }, status=409)
+
+    # Verify signature
+    try:
+        # Verify signature over the entire payload except the signature itself
+        signed_fields = {k: v for (k, v) in data.items() if k != "signature"}
+        blob = json.dumps(signed_fields, sort_keys=True, separators=(",", ":")).encode()
+        import nacl.signing, nacl.encoding
+        vk = nacl.signing.VerifyKey(b58decode(data["public_key"]))
+        sig = b58decode(data["signature"])
+        vk.verify(blob, sig)
+        ok = True
+    except Exception:
+        ok = False
+    if not ok:
+        make_log("Handshake", f"Signature verification failed from {data.get('public_host')}", level='warning')
+        return response.json({"error": "BAD_SIGNATURE"}, status=400)
+
+    # Upsert node and respond with our info + known public nodes
+    # Do not persist private peers (ephemeral)
+    if data.get("node_type") != "private" and data.get("public_host"):
+        try:
+            await upsert_known_node(
+                request.ctx.db_session,
+                host=data.get("public_host"),
+                port=int(str(data.get("public_host") or "").split(":")[-1]) if ":" in str(data.get("public_host") or "") else 80,
+                public_key=str(data.get("public_key")),
+                meta={
+                    "version": peer_version,
+                    "compatibility": comp,
+                    "is_public": True,
+                    "public_host": data.get("public_host"),
+                    "last_metrics": data.get("metrics", {}),
+                    "capabilities": data.get("capabilities", {}),
+                }
+            )
+        except Exception as e:
+            make_log("Handshake", f"Upsert peer failed: {e}", level='warning')
+
+    # Merge advertised peers from the caller (optional field)
+    for n in data.get("known_public_nodes", []) or []:
+        try:
+            await upsert_known_node(
+                request.ctx.db_session,
+                host=n.get("public_host") or n.get("host"),
+                port=int(n.get("port") or 80),
+                public_key=n.get("public_key") or "",
+                meta={
+                    "version": n.get("version") or "0.0.0",
+                    "compatibility": compatibility(n.get("version") or "0.0.0", CURRENT_PROTOCOL_VERSION),
+                    "is_public": True,
+                    "public_host": n.get("public_host") or n.get("host"),
+                    "capabilities": n.get("capabilities") or {},
+                }
+            )
+        except Exception:
+            pass
+
+    node = await compute_node_info(request.ctx.db_session)
+    known = await list_known_public_nodes(request.ctx.db_session)
+    resp = sign_response({
+        "compatibility": comp,
+        "node": node,
+        "known_public_nodes": known,
+    })
+    make_log("Handshake", f"OK with {data.get('public_host')} compat={comp}")
+    status = 200
+    if comp == "warning":
+        status = 200
+        resp["warning"] = "MINOR version differs; proceed with caution"
+    return response.json(resp, status=status)

app/api/routes/node_storage.py

@@ -11,6 +11,7 @@ from sanic import response
 import json
 
 from app.core._config import UPLOADS_DIR
+from sqlalchemy import select
 from app.core._utils.resolve_content import resolve_content
 from app.core.logger import make_log
 from app.core.models.node_storage import StoredContent
@@ -52,7 +53,9 @@ async def s_api_v1_storage_post(request):
     try:
         file_hash_bin = hashlib.sha256(file_content).digest()
         file_hash = b58encode(file_hash_bin).decode()
-        stored_content = request.ctx.db_session.query(StoredContent).filter(StoredContent.hash == file_hash).first()
+        stored_content = (await request.ctx.db_session.execute(
+            select(StoredContent).where(StoredContent.hash == file_hash)
+        )).scalars().first()
         if stored_content:
             stored_cid = stored_content.cid.serialize_v1()
             stored_cid_v2 = stored_content.cid.serialize_v2()
@@ -80,7 +83,7 @@ async def s_api_v1_storage_post(request):
             key_id=None,
         )
         request.ctx.db_session.add(new_content)
-        request.ctx.db_session.commit()
+        await request.ctx.db_session.commit()
 
         file_path = os.path.join(UPLOADS_DIR, file_hash)
         async with aiofiles.open(file_path, "wb") as file:
@@ -97,7 +100,7 @@ async def s_api_v1_storage_post(request):
             "content_url": f"dmy://storage?cid={new_cid}",
         })
     except BaseException as e:
-        make_log("Storage", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
+        make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error: {e}" + '\n' + traceback.format_exc(), level="error")
         return response.json({"error": f"Error: {e}"}, status=500)
 
 
@@ -112,14 +115,16 @@ async def s_api_v1_storage_get(request, file_hash=None):
         return response.json({"error": errmsg}, status=400)
 
     content_sha256 = b58encode(cid.content_hash).decode()
-    content = request.ctx.db_session.query(StoredContent).filter(StoredContent.hash == content_sha256).first()
+    content = (await request.ctx.db_session.execute(
+        select(StoredContent).where(StoredContent.hash == content_sha256)
+    )).scalars().first()
     if not content:
         return response.json({"error": "File not found"}, status=404)
 
-    make_log("Storage", f"File {content_sha256} requested by {request.ctx.user}")
+    make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} File {content_sha256} requested by user={getattr(getattr(request.ctx, 'user', None), 'id', None)}")
     file_path = os.path.join(UPLOADS_DIR, content_sha256)
     if not os.path.exists(file_path):
-        make_log("Storage", f"File {content_sha256} not found locally", level="error")
+        make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} File {content_sha256} not found locally", level="error")
         return response.json({"error": "File not found"}, status=404)
 
     async with aiofiles.open(file_path, "rb") as file:
@@ -139,7 +144,16 @@ async def s_api_v1_storage_get(request, file_hash=None):
             tempfile_path += "_mpeg" + (f"_{seconds_limit}" if seconds_limit else "")
             if not os.path.exists(tempfile_path):
                 try:
-                    cover_content = StoredContent.from_cid(content.meta.get('cover_cid'))
+                    # Resolve cover content by CID (async)
+                    from app.core.content.content_id import ContentId
+                    try:
+                        _cid = ContentId.deserialize(content.meta.get('cover_cid'))
+                        _cover_hash = _cid.content_hash_b58
+                        cover_content = (await request.ctx.db_session.execute(
+                            select(StoredContent).where(StoredContent.hash == _cover_hash)
+                        )).scalars().first()
+                    except Exception:
+                        cover_content = None
                     cover_tempfile_path = os.path.join(UPLOADS_DIR, f"tmp_{cover_content.hash}_jpeg")
                     if not os.path.exists(cover_tempfile_path):
                         cover_image = Image.open(cover_content.filepath)
@@ -173,25 +187,25 @@ async def s_api_v1_storage_get(request, file_hash=None):
                         try:
                             audio = AudioSegment.from_file(file_path)
                         except BaseException as e:
-                            make_log("Storage", f"Error loading audio from file: {e}", level="debug")
+                            make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error loading audio from file: {e}", level="debug")
 
                     if not audio:
                         try:
                             audio = AudioSegment(content_file_bin)
                         except BaseException as e:
-                            make_log("Storage", f"Error loading audio from binary: {e}", level="debug")
+                            make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error loading audio from binary: {e}", level="debug")
 
                     audio = audio[:seconds_limit * 1000] if seconds_limit else audio
                     audio.export(tempfile_path, format="mp3", cover=cover_tempfile_path)
                 except BaseException as e:
-                    make_log("Storage", f"Error converting audio: {e}" + '\n' + traceback.format_exc(), level="error")
+                    make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error converting audio: {e}" + '\n' + traceback.format_exc(), level="error")
 
             if os.path.exists(tempfile_path):
                 async with aiofiles.open(tempfile_path, "rb") as file:
                     content_file_bin = await file.read()
 
                 accept_type = 'audio/mpeg'
-                make_log("Storage", f"Audio {content_sha256} converted successfully")
+                make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Audio {content_sha256} converted successfully", level='debug')
             else:
                 tempfile_path = tempfile_path[:-5]
 
@@ -208,13 +222,13 @@ async def s_api_v1_storage_get(request, file_hash=None):
                             break
                         quality -= 5
                 except BaseException as e:
-                    make_log("Storage", f"Error converting image: {e}" + '\n' + traceback.format_exc(), level="error")
+                    make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Error converting image: {e}" + '\n' + traceback.format_exc(), level="error")
 
             if os.path.exists(tempfile_path):
                 async with aiofiles.open(tempfile_path, "rb") as file:
                     content_file_bin = await file.read()
 
-                make_log("Storage", f"Image {content_sha256} converted successfully")
+                make_log("Storage", f"sid={getattr(request.ctx, 'session_id', None)} Image {content_sha256} converted successfully", level='debug')
                 accept_type = 'image/jpeg'
             else:
                 tempfile_path = tempfile_path[:-5]

app/api/routes/progressive_storage.py

@@ -11,36 +11,38 @@ from base58 import b58encode
 from sanic import response
 
 from app.core.logger import make_log
+from sqlalchemy import select
 from app.core.models.node_storage import StoredContent
 from app.core._config import UPLOADS_DIR
+from app.core.models.content_v3 import ContentDerivative
 from app.core._utils.resolve_content import resolve_content
 
 
 # POST /api/v1.5/storage
 async def s_api_v1_5_storage_post(request):
     # Log the receipt of a chunk upload request
-    make_log("uploader_v1.5", "Received chunk upload request", level="INFO")
+    make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Received chunk upload request", level="INFO")
     
     # Get the provided file name from header and decode it from base64
     provided_filename_b64 = request.headers.get("X-File-Name")
     if not provided_filename_b64:
-        make_log("uploader_v1.5", "Missing X-File-Name header", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Missing X-File-Name header", level="ERROR")
         return response.json({"error": "Missing X-File-Name header"}, status=400)
     try:
         provided_filename = b64decode(provided_filename_b64).decode("utf-8")
     except Exception as e:
-        make_log("uploader_v1.5", f"Invalid X-File-Name header: {e}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Invalid X-File-Name header: {e}", level="ERROR")
         return response.json({"error": "Invalid X-File-Name header"}, status=400)
     
     # Get X-Chunk-Start header (must be provided) and parse it as integer
     chunk_start_header = request.headers.get("X-Chunk-Start")
     if chunk_start_header is None:
-        make_log("uploader_v1.5", "Missing X-Chunk-Start header", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Missing X-Chunk-Start header", level="ERROR")
         return response.json({"error": "Missing X-Chunk-Start header"}, status=400)
     try:
         chunk_start = int(chunk_start_header)
     except Exception as e:
-        make_log("uploader_v1.5", f"Invalid X-Chunk-Start header: {e}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Invalid X-Chunk-Start header: {e}", level="ERROR")
         return response.json({"error": "Invalid X-Chunk-Start header"}, status=400)
     
     # Enforce maximum chunk size (80 MB) using Content-Length header if provided
@@ -50,7 +52,7 @@ async def s_api_v1_5_storage_post(request):
         try:
             content_length = int(content_length)
             if content_length > max_chunk_size:
-                make_log("uploader_v1.5", f"Chunk size {content_length} exceeds maximum allowed", level="ERROR")
+                make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Chunk size {content_length} exceeds maximum allowed", level="ERROR")
                 return response.json({"error": "Chunk size exceeds maximum allowed (80 MB)"}, status=400)
         except:
             pass
@@ -62,9 +64,9 @@ async def s_api_v1_5_storage_post(request):
         # New upload session: generate a new uuid
         upload_id = str(uuid4())
         is_new_upload = True
-        make_log("uploader_v1.5", f"Starting new upload session with ID: {upload_id}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Start new upload session id={upload_id}", level="INFO")
     else:
-        make_log("uploader_v1.5", f"Resuming upload session with ID: {upload_id}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Resume upload session id={upload_id}", level="DEBUG")
     
     # Determine the temporary file path based on upload_id
     temp_path = os.path.join(UPLOADS_DIR, f"v1.5_upload_{upload_id}")
@@ -76,10 +78,10 @@ async def s_api_v1_5_storage_post(request):
     
     # If the provided chunk_start is less than current_size, the chunk is already received
     if chunk_start < current_size:
-        make_log("uploader_v1.5", f"Chunk starting at {chunk_start} already received, current size: {current_size}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Chunk at {chunk_start} already received; size={current_size}", level="DEBUG")
         return response.json({"upload_id": upload_id, "current_size": current_size})
     elif chunk_start > current_size:
-        make_log("uploader_v1.5", f"Chunk start {chunk_start} does not match current file size {current_size}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Chunk start {chunk_start} != current size {current_size}", level="ERROR")
         return response.json({"error": "Chunk start does not match current file size"}, status=400)
     
     # Append the received chunk to the temporary file
@@ -93,9 +95,9 @@ async def s_api_v1_5_storage_post(request):
                 async for chunk in request.stream:
                     await out_file.write(chunk)
         new_size = os.path.getsize(temp_path)
-        make_log("uploader_v1.5", f"Appended chunk. New file size: {new_size}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Appended chunk. size={new_size}", level="DEBUG")
     except Exception as e:
-        make_log("uploader_v1.5", f"Error saving chunk: {e}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Error saving chunk: {e}", level="ERROR")
         return response.json({"error": "Failed to save chunk"}, status=500)
     
     # If computed hash matches the provided one, the final chunk has been received
@@ -111,28 +113,28 @@ async def s_api_v1_5_storage_post(request):
             stdout, stderr = await proc.communicate()
             if proc.returncode != 0:
                 error_msg = stderr.decode().strip()
-                make_log("uploader_v1.5", f"sha256sum error: {error_msg}", level="ERROR")
+                make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} sha256sum error: {error_msg}", level="ERROR")
                 return response.json({"error": "Failed to compute file hash"}, status=500)
             computed_hash_hex = stdout.decode().split()[0].strip()
             computed_hash_bytes = bytes.fromhex(computed_hash_hex)
             computed_hash_b58 = b58encode(computed_hash_bytes).decode()
-            make_log("uploader_v1.5", f"Computed hash (base58): {computed_hash_b58}", level="INFO")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Computed hash (base58): {computed_hash_b58}", level="INFO")
         except Exception as e:
-            make_log("uploader_v1.5", f"Error computing file hash: {e}", level="ERROR")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Error computing file hash: {e}", level="ERROR")
             return response.json({"error": "Error computing file hash"}, status=500)
 
         final_path = os.path.join(UPLOADS_DIR, f"{computed_hash_b58}")
         try:
             os.rename(temp_path, final_path)
-            make_log("uploader_v1.5", f"Final chunk received. File renamed to: {final_path}", level="INFO")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Final chunk received. Renamed to: {final_path}", level="INFO")
         except Exception as e:
-            make_log("uploader_v1.5", f"Error renaming file: {e}", level="ERROR")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Error renaming file: {e}", level="ERROR")
             return response.json({"error": "Failed to finalize file storage"}, status=500)
         
         db_session = request.ctx.db_session
-        existing = db_session.query(StoredContent).filter_by(hash=computed_hash_b58).first()
+        existing = (await db_session.execute(select(StoredContent).where(StoredContent.hash == computed_hash_b58))).scalars().first()
         if existing:
-            make_log("uploader_v1.5", f"File with hash {computed_hash_b58} already exists in DB", level="INFO")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} File already exists in DB: {computed_hash_b58}", level="INFO")
             serialized_v2 = existing.cid.serialize_v2()
             serialized_v1 = existing.cid.serialize_v1()
             return response.json({
@@ -156,10 +158,10 @@ async def s_api_v1_5_storage_post(request):
                 created=datetime.utcnow()
             )
             db_session.add(new_content)
-            db_session.commit()
-            make_log("uploader_v1.5", f"New file stored and indexed for user {user_id} with hash {computed_hash_b58}", level="INFO")
+            await db_session.commit()
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Stored new file user={user_id} hash={computed_hash_b58}", level="INFO")
         except Exception as e:
-            make_log("uploader_v1.5", f"Database error: {e}", level="ERROR")
+            make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Database error: {e}", level="ERROR")
             return response.json({"error": "Database error"}, status=500)
         
         serialized_v2 = new_content.cid.serialize_v2()
@@ -178,7 +180,7 @@ async def s_api_v1_5_storage_post(request):
 
 # GET /api/v1.5/storage/<file_hash>
 async def s_api_v1_5_storage_get(request, file_hash):
-    make_log("uploader_v1.5", f"Received file retrieval request for hash: {file_hash}", level="INFO")
+    make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Retrieve file hash={file_hash}", level="INFO")
     
     try:
         file_hash = b58encode(resolve_content(file_hash)[0].content_hash).decode()
@@ -187,11 +189,11 @@ async def s_api_v1_5_storage_get(request, file_hash):
 
     final_path = os.path.join(UPLOADS_DIR, f"{file_hash}")
     if not os.path.exists(final_path):
-        make_log("uploader_v1.5", f"File not found: {final_path}", level="ERROR")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} File not found: {final_path}", level="ERROR")
         return response.json({"error": "File not found"}, status=404)
 
     db_session = request.ctx.db_session
-    stored = db_session.query(StoredContent).filter_by(hash=file_hash).first()
+    stored = (await db_session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first()
     if stored and stored.filename:
         filename_for_mime = stored.filename
     else:
@@ -204,8 +206,17 @@ async def s_api_v1_5_storage_get(request, file_hash):
     file_size = os.path.getsize(final_path)
     range_header = request.headers.get("Range")
     
+    # touch derivative last_access_at if exists
+    try:
+        cd = (await request.ctx.db_session.execute(select(ContentDerivative).where(ContentDerivative.local_path.like(f"%/{file_hash}")))).scalars().first()
+        if cd:
+            cd.last_access_at = datetime.utcnow()
+            await request.ctx.db_session.commit()
+    except Exception:
+        pass
+
     if range_header:
-        make_log("uploader_v1.5", f"Processing Range header: {range_header}", level="INFO")
+        make_log("uploader_v1.5", f"sid={getattr(request.ctx, 'session_id', None)} Processing Range: {range_header}", level="DEBUG")
         range_spec = range_header.strip().lower()
         if not range_spec.startswith("bytes="):
             make_log("uploader_v1.5", f"Invalid Range header: {range_header}", level="ERROR")

app/api/routes/sync.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from datetime import datetime
+from sanic import response
+from sqlalchemy import select
+
+from app.core.ipfs_client import pin_add, pin_ls
+from app.core.logger import make_log
+from app.core.models.content_v3 import EncryptedContent, IpfsSync
+from app.core.network.nodesig import verify_request
+from app.core.network.guard import check_rate_limit
+
+
+async def s_api_v1_sync_pin(request):
+    # Rate limit per IP and require NodeSig for POST
+    remote_ip = (request.headers.get('X-Forwarded-For') or request.remote_addr or request.ip or '').split(',')[0].strip()
+    if not check_rate_limit(request.app.ctx.memory, remote_ip):
+        return response.json({"error": "RATE_LIMIT"}, status=429)
+
+    ok, node_id, reason = verify_request(request, request.app.ctx.memory)
+    if not ok:
+        return response.json({"error": reason or "UNAUTHORIZED"}, status=401)
+
+    data = request.json or {}
+    cid = data.get("encrypted_cid")
+    if not cid:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+
+    session = request.ctx.db_session
+    row = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+    if not row:
+        # create record with minimal info (unknown meta)
+        row = EncryptedContent(
+            encrypted_cid=cid,
+            title=cid,
+            description="",
+            content_type="application/octet-stream",
+            preview_enabled=False,
+        )
+        session.add(row)
+        await session.flush()
+    sync = (await session.execute(select(IpfsSync).where(IpfsSync.content_id == row.id))).scalars().first()
+    if not sync:
+        sync = IpfsSync(content_id=row.id, pin_state='queued')
+        session.add(sync)
+        await session.flush()
+
+    try:
+        await pin_add(cid, recursive=True)
+        sync.pin_state = 'pinned'
+        sync.pinned_at = datetime.utcnow()
+    except Exception as e:
+        make_log("sync", f"pin failed: {e}", level="error")
+        sync.pin_state = 'failed'
+        sync.pin_error = str(e)
+    await session.commit()
+    return response.json({"ok": True, "state": sync.pin_state})
+
+
+async def s_api_v1_sync_status(request):
+    cid = request.args.get("cid")
+    if not cid:
+        return response.json({"error": "BAD_REQUEST"}, status=400)
+    try:
+        info = await pin_ls(cid)
+        state = 'pinned' if info else 'not_pinned'
+    except Exception:
+        state = 'not_pinned'
+        info = {}
+    return response.json({"cid": cid, "state": state, "info": info})

app/api/routes/tonconnect.py

@@ -4,6 +4,7 @@ from aiogram.utils.web_app import safe_parse_webapp_init_data
 from sanic import response
 
 from app.core._blockchain.ton.connect import TonConnect, unpack_wallet_info, WalletConnection
+from sqlalchemy import select, and_
 from app.core._config import TELEGRAM_API_KEY
 from app.core.models.user import User
 from app.core.logger import make_log
@@ -23,8 +24,19 @@ async def s_api_v1_tonconnect_new(request):
     db_session = request.ctx.db_session
     user = request.ctx.user
     memory = request.ctx.memory
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user)
-    await ton_connect.restore_connection()
+    # Try restore last connection from DB
+    ton_connection = (await db_session.execute(select(WalletConnection).where(
+        and_(
+            WalletConnection.user_id == user.id,
+            WalletConnection.invalidated == False,
+            WalletConnection.network == 'ton'
+        )
+    ).order_by(WalletConnection.created.desc()))).scalars().first()
+    if ton_connection:
+        ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"])
+        await ton_connect.restore_connection()
+    else:
+        ton_connect = TonConnect()
     make_log("TonConnect_API", f"SDK connected?: {ton_connect.connected}", level='info')
     if ton_connect.connected:
         return response.json({"error": "Already connected"}, status=400)
@@ -47,13 +59,11 @@ async def s_api_v1_tonconnect_logout(request):
     user = request.ctx.user
     memory = request.ctx.memory
 
-    wallet_connections = db_session.query(WalletConnection).filter(
-        WalletConnection.user_id == user.id,
-        WalletConnection.invalidated == False
-    ).all()
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
+    ))
+    wallet_connections = result.scalars().all()
     for wallet_connection in wallet_connections:
         wallet_connection.invalidated = True
-
-    db_session.commit()
+    await db_session.commit()
     return response.json({"success": True})
-

app/api/routes/upload_status.py

@@ -0,0 +1,17 @@
+from sanic import response
+from app.core.models.content_v3 import UploadSession
+
+
+async def s_api_v1_upload_status(request, upload_id: str):
+    session = request.ctx.db_session
+    row = await session.get(UploadSession, upload_id)
+    if not row:
+        return response.json({"error": "NOT_FOUND"}, status=404)
+    return response.json({
+        "id": row.id,
+        "state": row.state,
+        "encrypted_cid": row.encrypted_cid,
+        "size_bytes": row.size_bytes,
+        "error": row.error,
+    })
+

app/api/routes/upload_tus.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import base64
+import json
+import os
+from datetime import datetime
+from typing import Dict, Any
+
+from base58 import b58encode
+from sanic import response
+
+from app.core._secrets import hot_pubkey
+from app.core.crypto.aes_gcm_stream import encrypt_file_to_encf, CHUNK_BYTES
+from app.core.crypto.keywrap import wrap_dek, KeyWrapError
+from app.core.ipfs_client import add_streamed_file
+from app.core.logger import make_log
+from app.core.models.content_v3 import EncryptedContent, ContentKey, IpfsSync, ContentIndexItem, UploadSession
+from app.core.storage import db_session
+
+
+def _b64(s: bytes) -> str:
+    return base64.b64encode(s).decode()
+
+
+async def s_api_v1_upload_tus_hook(request):
+    """
+    tusd HTTP hook endpoint. We mainly handle post-finish to: encrypt -> IPFS add+pin -> record DB.
+    """
+    try:
+        payload: Dict[str, Any] = request.json or {}
+    except Exception:
+        payload = {}
+    event = payload.get("Type") or payload.get("type") or payload.get("Event") or payload.get("event")
+    upload = payload.get("Upload") or payload.get("upload") or {}
+
+    if not event:
+        return response.json({"ok": False, "error": "NO_EVENT"}, status=400)
+
+    if event not in ("post-finish", "postfinish"):
+        # accept but ignore other events
+        return response.json({"ok": True})
+
+    # Extract storage path from tusd payload
+    storage = upload.get("Storage") or {}
+    file_path = storage.get("Path") or storage.get("path")
+    if not file_path:
+        return response.json({"ok": False, "error": "NO_STORAGE_PATH"}, status=400)
+
+    meta = upload.get("MetaData") or {}
+    # Common metadata keys
+    title = meta.get("title") or meta.get("Title") or meta.get("name") or "Untitled"
+    description = meta.get("description") or meta.get("Description") or ""
+    content_type = meta.get("content_type") or meta.get("Content-Type") or "application/octet-stream"
+    preview_enabled = content_type.startswith("audio/") or content_type.startswith("video/")
+    # Optional preview window overrides from tus metadata
+    try:
+        start_ms = int(meta.get("preview_start_ms") or 0)
+        dur_ms = int(meta.get("preview_duration_ms") or 30000)
+    except Exception:
+        start_ms, dur_ms = 0, 30000
+
+    # Record/Update upload session
+    upload_id = upload.get("ID") or upload.get("Id") or upload.get("id")
+    try:
+        size = int(upload.get("Size") or 0)
+    except Exception:
+        size = None
+
+    async with db_session() as session:
+        us = (await session.get(UploadSession, upload_id)) if upload_id else None
+        if not us and upload_id:
+            us = UploadSession(
+                id=upload_id,
+                filename=os.path.basename(file_path),
+                size_bytes=size,
+                state='processing',
+                encrypted_cid=None,
+            )
+            session.add(us)
+            await session.commit()
+
+    # Read & encrypt by streaming (ENCF v1 / AES-GCM)
+    # Generate per-content random DEK and salt
+    dek = os.urandom(32)
+    salt = os.urandom(16)
+    key_fpr = b58encode(hot_pubkey).decode()  # fingerprint as our node id for now
+
+    # Stream encrypt into IPFS add
+    try:
+        wrapped_dek = wrap_dek(dek)
+    except KeyWrapError as e:
+        make_log("tus-hook", f"Key wrap failed: {e}", level="error")
+        async with db_session() as session:
+            if upload_id:
+                us = await session.get(UploadSession, upload_id)
+                if us:
+                    us.state = 'failed'
+                    us.error = str(e)
+                    await session.commit()
+        return response.json({"ok": False, "error": "KEY_WRAP_FAILED"}, status=500)
+
+    try:
+        with open(file_path, 'rb') as f:
+            result = await add_streamed_file(
+                encrypt_file_to_encf(f, dek, CHUNK_BYTES, salt),
+                filename=os.path.basename(file_path),
+                params={},
+            )
+    except Exception as e:
+        make_log("tus-hook", f"Encrypt+add failed: {e}", level="error")
+        # mark failed
+        async with db_session() as session:
+            if upload_id:
+                us = await session.get(UploadSession, upload_id)
+                if us:
+                    us.state = 'failed'
+                    us.error = str(e)
+                    await session.commit()
+        return response.json({"ok": False, "error": "ENCRYPT_ADD_FAILED"}, status=500)
+
+    encrypted_cid = result.get("Hash")
+    try:
+        enc_size = int(result.get("Size") or 0)
+    except Exception:
+        enc_size = None
+
+    # Persist records
+    async with db_session() as session:
+        ec = EncryptedContent(
+            encrypted_cid=encrypted_cid,
+            title=title,
+            description=description,
+            content_type=content_type,
+            enc_size_bytes=enc_size,
+            plain_size_bytes=os.path.getsize(file_path),
+            preview_enabled=preview_enabled,
+            preview_conf=({"duration_ms": dur_ms, "intervals": [[start_ms, start_ms + dur_ms]]} if preview_enabled else {}),
+            aead_scheme="AES_GCM",
+            chunk_bytes=CHUNK_BYTES,
+            salt_b64=_b64(salt),
+        )
+        session.add(ec)
+        await session.flush()
+
+        ck = ContentKey(
+            content_id=ec.id,
+            key_ciphertext_b64=wrapped_dek,
+            key_fingerprint=key_fpr,
+            issuer_node_id=key_fpr,
+            allow_auto_grant=True,
+        )
+        session.add(ck)
+
+        sync = IpfsSync(
+            content_id=ec.id,
+            pin_state='pinned',
+            bytes_total=enc_size,
+            bytes_fetched=enc_size,
+            pinned_at=datetime.utcnow(),
+        )
+        session.add(sync)
+
+        # Publish signed index item
+        item = {
+            "encrypted_cid": encrypted_cid,
+            "title": title,
+            "description": description,
+            "content_type": content_type,
+            "size_bytes": enc_size,
+            "preview_enabled": preview_enabled,
+            "preview_conf": ec.preview_conf,
+            "issuer_node_id": key_fpr,
+            "salt_b64": _b64(salt),
+        }
+        try:
+            from app.core._crypto.signer import Signer
+            from app.core._secrets import hot_seed
+            signer = Signer(hot_seed)
+            blob = json.dumps(item, sort_keys=True, separators=(",", ":")).encode()
+            sig = signer.sign(blob)
+        except Exception:
+            sig = ""
+        session.add(ContentIndexItem(encrypted_cid=encrypted_cid, payload=item, sig=sig))
+
+        await session.commit()
+
+    # Update upload session with result and purge staging to avoid duplicates
+    async with db_session() as session:
+        if upload_id:
+            us = await session.get(UploadSession, upload_id)
+            if us:
+                us.state = 'pinned'
+                us.encrypted_cid = encrypted_cid
+                # prefer using IPFS for downstream conversion; remove staging
+                try:
+                    if file_path and os.path.exists(file_path):
+                        os.remove(file_path)
+                except Exception:
+                    pass
+                us.storage_path = None
+                await session.commit()
+
+    make_log("tus-hook", f"Uploaded+encrypted {file_path} -> {encrypted_cid}")
+    return response.json({"ok": True, "encrypted_cid": encrypted_cid, "upload_id": upload_id})

app/bot/middleware.py

@@ -1,6 +1,7 @@
 from app.core.logger import make_log, logger
 from app.core.models._telegram import Wrapped_CBotChat
 from app.core.models.user import User
+from sqlalchemy import select
 from app.core.storage import db_session
 from aiogram import BaseMiddleware, types
 from app.core.models.messages import KnownTelegramMessage
@@ -21,9 +22,9 @@ class UserDataMiddleware(BaseMiddleware):
 
         # TODO: maybe make users cache
 
-        with db_session(auto_commit=False) as session:
+        async with db_session(auto_commit=False) as session:
             try:
-                user = session.query(User).filter_by(telegram_id=user_id).first()
+                user = (await session.execute(select(User).where(User.telegram_id == user_id))).scalars().first()
             except BaseException as e:
                 logger.error(f"Error when middleware getting user: {e}")
                 user = None
@@ -42,7 +43,7 @@ class UserDataMiddleware(BaseMiddleware):
                     created=datetime.now()
                 )
                 session.add(user)
-                session.commit()
+                await session.commit()
             else:
                 if user.username != update_body.from_user.username:
                     user.username = update_body.from_user.username
@@ -60,7 +61,7 @@ class UserDataMiddleware(BaseMiddleware):
                 }
 
                 user.last_use = datetime.now()
-                session.commit()
+                await session.commit()
 
             data['user'] = user
             data['db_session'] = session
@@ -72,11 +73,11 @@ class UserDataMiddleware(BaseMiddleware):
                 if update_body.text.startswith('/start'):
                     message_type = 'start_command'
 
-                if session.query(KnownTelegramMessage).filter_by(
-                    chat_id=update_body.chat.id,
-                    message_id=update_body.message_id,
-                    from_user=True
-                ).first():
+                if (await session.execute(select(KnownTelegramMessage).where(
+                    (KnownTelegramMessage.chat_id == update_body.chat.id) &
+                    (KnownTelegramMessage.message_id == update_body.message_id) &
+                    (KnownTelegramMessage.from_user == True)
+                ))).scalars().first():
                     make_log("UserDataMiddleware", f"Message {update_body.message_id} already processed", level='debug')
                     return
 
@@ -91,7 +92,7 @@ class UserDataMiddleware(BaseMiddleware):
                     meta={}
                 )
                 session.add(new_message)
-                session.commit()
+                await session.commit()
 
             result = await handler(event, data)
             return result

app/bot/routers/content.py

@@ -6,6 +6,7 @@ from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.logger import make_log
 from app.core.models.node_storage import StoredContent
+from sqlalchemy import select, and_
 import json
 
 router = Router()
@@ -20,12 +21,13 @@ def chunks(lst, n):
 async def t_callback_owned_content(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
     message_text = user.translated("ownedContent_menu")
     content_list = []
-    for content in db_session.query(StoredContent).filter_by(
-        owner_address=user.wallet_address(db_session),
-        type='onchain/content'
-    ).all():
+    user_addr = await user.wallet_address_async(db_session)
+    result = await db_session.execute(select(StoredContent).where(
+        and_(StoredContent.owner_address == user_addr, StoredContent.type == 'onchain/content')
+    ))
+    for content in result.scalars().all():
         try:
-            metadata_content = StoredContent.from_cid(db_session, content.json_format()['metadata_cid'])
+            metadata_content = await StoredContent.from_cid_async(db_session, content.json_format()['metadata_cid'])
             with open(metadata_content.filepath, 'r') as f:
                 metadata_content_json = json.loads(f.read())
         except BaseException as e:
@@ -59,10 +61,9 @@ async def t_callback_owned_content(query: types.CallbackQuery, memory=None, user
 
 async def t_callback_node_content(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
     content_oid = int(query.data.split('_')[1])
+    row = (await db_session.execute(select(StoredContent).where(StoredContent.id == content_oid))).scalars().first()
     return await chat_wrap.send_content(
-        db_session, db_session.query(StoredContent).filter_by(
-            id=content_oid
-        ).first(),
+        db_session, row,
         extra_buttons=[
             [{
                 'text': user.translated('back_button'),

app/bot/routers/home.py

@@ -3,6 +3,7 @@ from aiogram.filters import Command
 from tonsdk.utils import Address
 
 from app.core._blockchain.ton.connect import TonConnect
+from sqlalchemy import select, and_
 from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.models.wallet_connection import WalletConnection
@@ -32,8 +33,14 @@ async def send_home_menu(chat_wrap, user, wallet_connection, **kwargs):
 
 
 async def send_connect_wallets_list(db_session, chat_wrap, user, **kwargs):
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
-    await ton_connect.restore_connection()
+    # Try to restore existing connection via DB
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+    if ton_connection:
+        await ton_connect.restore_connection()
     wallets = ton_connect._sdk_client.get_wallets()
     message_text = user.translated("connectWalletsList_menu")
     return await tg_process_template(
@@ -66,10 +73,9 @@ async def t_home_menu(__msg, **extra):
     else:
         message_id = None
 
-    wallet_connection = db_session.query(WalletConnection).filter(
-        WalletConnection.user_id == user.id,
-        WalletConnection.invalidated == False
-    ).first()
+    wallet_connection = (await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
+    ))).scalars().first()
 
     # if not wallet_connection:
     #     return await send_connect_wallets_list(db_session, chat_wrap, user, message_id=message_id)

app/bot/routers/tonconnect.py

@@ -7,6 +7,7 @@ from aiogram.filters import Command
 
 from app.bot.routers.home import send_connect_wallets_list, send_home_menu
 from app.core._blockchain.ton.connect import TonConnect, unpack_wallet_info
+from sqlalchemy import select, and_
 from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.logger import make_log
@@ -33,15 +34,21 @@ async def t_tonconnect_dev_menu(message: types.Message, memory=None, user=None,
 
     keyboard = []
 
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
+    # Restore recent connection
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
     make_log("TonConnect_DevMenu", f"Available wallets: {ton_connect._sdk_client.get_wallets()}", level='debug')
-    await ton_connect.restore_connection()
+    if ton_connection:
+        await ton_connect.restore_connection()
     make_log("TonConnect_DevMenu", f"SDK connected?: {ton_connect.connected}", level='info')
     if not ton_connect.connected:
         if ton_connection:
             make_log("TonConnect_DevMenu", f"Invalidating old connection", level='debug')
             ton_connection.invalidated = True
-            db_session.commit()
+            await db_session.commit()
 
         message_text = f"""<b>Wallet is not connected</b>
         
@@ -71,8 +78,13 @@ Use /dev_tonconnect <code>{wallet_app_name}</code> for connect to wallet."""
 
 async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
     wallet_app_name = query.data.split("_")[1]
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user)
-    await ton_connect.restore_connection()
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+    if ton_connection:
+        await ton_connect.restore_connection()
     connection_link = await ton_connect.new_connection(wallet_app_name)
     ton_connect.connected
     memory.add_task(pause_ton_connection, ton_connect, delay_s=60 * 3)
@@ -98,10 +110,9 @@ async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, us
 
     start_ts = datetime.now()
     while datetime.now() - start_ts < timedelta(seconds=180):
-        new_connection = db_session.query(WalletConnection).filter(
-            WalletConnection.user_id == user.id,
-            WalletConnection.invalidated == False
-        ).first()
+        new_connection = (await db_session.execute(select(WalletConnection).where(
+            and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
+        ))).scalars().first()
         if new_connection:
             await tg_process_template(
                 chat_wrap, user.translated('p_successConnectWallet')
@@ -115,14 +126,13 @@ async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, us
 
 
 async def t_callback_disconnect_wallet(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
-    wallet_connections = db_session.query(WalletConnection).filter(
-        WalletConnection.user_id == user.id,
-        WalletConnection.invalidated == False
-    ).all()
+    wallet_connections = (await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
+    ))).scalars().all()
     for wallet_connection in wallet_connections:
         wallet_connection.invalidated = True
 
-    db_session.commit()
+    await db_session.commit()
 
     return await send_home_menu(chat_wrap, user, None, message_id=query.message.message_id)
 

app/client_bot/routers/content.py

@@ -6,6 +6,7 @@ from aiogram import types, Router, F
 
 from app.core._keyboards import get_inline_keyboard
 from app.core.models.node_storage import StoredContent
+from sqlalchemy import select, and_
 import json
 from app.core.logger import make_log
 from app.core.models.content.user_content import UserAction, UserContent
@@ -30,7 +31,7 @@ CACHE_CHAT_ID = -1002390124789
 async def t_callback_purchase_node_content(query: types.CallbackQuery, memory=None, user=None, db_session=None, chat_wrap=None, **extra):
     content_oid = int(query.data.split('_')[1])
     is_cancel_request = query.data.split('_')[2] == 'cancel' if len(query.data.split('_')) > 2 else False
-    content = db_session.query(StoredContent).filter_by(id=content_oid).first()
+    content = (await db_session.execute(select(StoredContent).where(StoredContent.id == content_oid))).scalars().first()
     if not content:
         return await query.answer(user.translated('error_contentNotFound'), show_alert=True)
 
@@ -43,11 +44,16 @@ async def t_callback_purchase_node_content(query: types.CallbackQuery, memory=No
 
         make_log("Purchase", f"User {user.id} initiated purchase for content ID {content_oid}. License price: {license_price_num}.", level='info')
 
-        ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
-        await ton_connect.restore_connection()
+        result = await db_session.execute(select(WalletConnection).where(
+            and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+        ).order_by(WalletConnection.created.desc()))
+        ton_connection = result.scalars().first()
+        ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+        if ton_connection:
+            await ton_connect.restore_connection()
         assert ton_connect.connected, "No connected wallet"
 
-        user_wallet_address = user.wallet_address(db_session)
+        user_wallet_address = await user.wallet_address_async(db_session)
 
         memory._app.add_task(ton_connect._sdk_client.send_transaction({
             'valid_until': int(datetime.now().timestamp() + 300),
@@ -76,18 +82,15 @@ async def t_callback_purchase_node_content(query: types.CallbackQuery, memory=No
     else:
         # Logging cancellation attempt with detailed information
         make_log("Purchase", f"User {user.id} cancelled purchase for content ID {content_oid}.", level='info')
-        action = db_session.query(UserAction).filter_by(
-            type='purchase',
-            content_id=content_oid,
-            user_id=user.id,
-            status='requested'
-        ).first()
+        action = (await db_session.execute(select(UserAction).where(
+            and_(UserAction.type == 'purchase', UserAction.content_id == content_oid, UserAction.user_id == user.id, UserAction.status == 'requested')
+        ))).scalars().first()
         if not action:
             return await query.answer()
 
         action.status = 'canceled'
 
-    db_session.commit()
+    await db_session.commit()
     await chat_wrap.send_content(db_session, content, message_id=query.message.message_id)
 
 
@@ -104,9 +107,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
         args = None
         if source_args_ext.startswith('Q'):
             license_onchain_address = source_args_ext[1:]
-            licensed_content = db_session.query(UserContent).filter_by(
-                onchain_address=license_onchain_address,
-            ).first().content
+            licensed_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == license_onchain_address))).scalars().first().content
             make_log("InlineSearch", f"Query '{query.query}' is a license query for content ID {licensed_content.id}.", level='info')
             args = licensed_content.cid.serialize_v2()
         else:
@@ -118,15 +119,15 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
         content_list = []
         search_query = {'hash': cid.content_hash_b58}
         make_log("InlineSearch", f"Searching with query '{search_query}'.", level='info')
-        content = db_session.query(StoredContent).filter_by(**search_query).first()
-        content_prod = content.open_content(db_session)
+        content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == cid.content_hash_b58))).scalars().first()
+        content_prod = await content.open_content_async(db_session)
         # Get both encrypted and decrypted content objects
         encrypted_content = content_prod['encrypted_content']
         decrypted_content = content_prod['decrypted_content']
         decrypted_content_meta = decrypted_content.json_format()
 
         try:
-            metadata_content = StoredContent.from_cid(db_session, content.json_format()['metadata_cid'])
+            metadata_content = await StoredContent.from_cid_async(db_session, content.json_format()['metadata_cid'])
             with open(metadata_content.filepath, 'r') as f:
                 metadata_content_json = json.loads(f.read())
         except BaseException as e:
@@ -144,7 +145,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
 
         result_kwargs = {}
         try:
-            cover_content = StoredContent.from_cid(db_session, decrypted_content_meta.get('cover_cid') or None)
+            cover_content = await StoredContent.from_cid_async(db_session, decrypted_content_meta.get('cover_cid') or None)
         except BaseException as e:
             cover_content = None
 
@@ -152,9 +153,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
             result_kwargs['thumb_url'] = cover_content.web_url
 
         content_type_declared = decrypted_content_meta.get('content_type', 'application/x-binary').split('/')[0]
-        preview_content = db_session.query(StoredContent).filter_by(
-            hash=content.meta.get('converted_content', {}).get('low_preview')
-        ).first()
+        preview_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == content.meta.get('converted_content', {}).get('low_preview')))).scalars().first()
         content_type_declared = {
             'mp3': 'audio',
             'flac': 'audio',
@@ -162,7 +161,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
             'mov': 'video'
         }.get(preview_content.filename.split('.')[-1], content_type_declared)
 
-        hashtags_str = (' '.join(f"#{_h}" for _h in metadata_content_json.get('hashtags', []))).strip()
+        hashtags_str = metadata_content_json.get('description', '').strip()
         if hashtags_str:
             hashtags_str = hashtags_str + '\n'
 
@@ -196,7 +195,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
                         **decrypted_content.meta,
                         'telegram_file_cache_preview': preview_file_id
                     }
-                    db_session.commit()
+                    await db_session.commit()
                 except Exception as e:
                     # Logging error during preview upload with detailed content type and query information
                     make_log("InlineSearch", f"Error uploading preview for content type '{content_type_declared}' during inline query '{query.query}': {e}", level='error')
@@ -212,7 +211,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
                 types.InlineQueryResultCachedAudio(
                     id=f"NC_{content.id}_{int(datetime.now().timestamp() // 60)}",
                     audio_file_id=decrypted_content.meta['telegram_file_cache_preview'],
-                    caption=hashtags_str + user.translated('p_playerContext_preview'),
+                    caption=hashtags_str + user.translated('p_playerAudioContext_preview'),
                     parse_mode='html',
                     reply_markup=get_inline_keyboard([
                         [
@@ -242,7 +241,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
                     id=f"NC_{content.id}_{int(datetime.now().timestamp() // 60)}",
                     video_file_id=decrypted_content.meta['telegram_file_cache_preview'],
                     title=title,
-                    caption=hashtags_str + user.translated('p_playerContext_preview'),
+                    caption=hashtags_str + user.translated('p_playerVideoContext_preview'),
                     parse_mode='html',
                     reply_markup=get_inline_keyboard([
                         [

app/client_bot/routers/home.py

@@ -3,6 +3,7 @@ from aiogram.filters import Command
 from tonsdk.utils import Address
 
 from app.core._blockchain.ton.connect import TonConnect
+from sqlalchemy import select, and_
 from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.logger import make_log
@@ -32,8 +33,13 @@ async def send_home_menu(chat_wrap, user, wallet_connection, **kwargs):
 
 
 async def send_connect_wallets_list(db_session, chat_wrap, user, **kwargs):
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
-    await ton_connect.restore_connection()
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+    if ton_connection:
+        await ton_connect.restore_connection()
     wallets = ton_connect._sdk_client.get_wallets()
     message_text = user.translated("connectWalletsList_menu")
     return await tg_process_template(
@@ -66,10 +72,9 @@ async def t_home_menu(__msg, **extra):
     else:
         message_id = None
 
-    wallet_connection = db_session.query(WalletConnection).filter(
-        WalletConnection.user_id == user.id,
-        WalletConnection.invalidated == False
-    ).first()
+    wallet_connection = (await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
+    ))).scalars().first()
 
     # if not wallet_connection:
     #     return await send_connect_wallets_list(db_session, chat_wrap, user, message_id=message_id)

app/client_bot/routers/tonconnect.py

@@ -7,6 +7,7 @@ from aiogram.filters import Command
 
 from app.client_bot.routers.home import send_connect_wallets_list, send_home_menu
 from app.core._blockchain.ton.connect import TonConnect, unpack_wallet_info
+from sqlalchemy import select, and_
 from app.core._keyboards import get_inline_keyboard
 from app.core._utils.tg_process_template import tg_process_template
 from app.core.logger import make_log
@@ -34,15 +35,20 @@ async def t_tonconnect_dev_menu(message: types.Message, memory=None, user=None,
 
     keyboard = []
 
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
     make_log("TonConnect_DevMenu", f"Available wallets: {ton_connect._sdk_client.get_wallets()}", level='debug')
-    await ton_connect.restore_connection()
+    if ton_connection:
+        await ton_connect.restore_connection()
     make_log("TonConnect_DevMenu", f"SDK connected?: {ton_connect.connected}", level='info')
     if not ton_connect.connected:
         if ton_connection:
             make_log("TonConnect_DevMenu", f"Invalidating old connection", level='debug')
             ton_connection.invalidated = True
-            db_session.commit()
+            await db_session.commit()
 
         message_text = f"""<b>Wallet is not connected</b>
 
@@ -73,8 +79,13 @@ Use /dev_tonconnect <code>{wallet_app_name}</code> for connect to wallet."""
 async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, user=None, db_session=None,
                                      chat_wrap=None, **extra):
     wallet_app_name = query.data.split("_")[1]
-    ton_connect, ton_connection = TonConnect.by_user(db_session, user)
-    await ton_connect.restore_connection()
+    result = await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False, WalletConnection.network == 'ton')
+    ).order_by(WalletConnection.created.desc()))
+    ton_connection = result.scalars().first()
+    ton_connect = TonConnect.by_key(ton_connection.keys["connection_key"]) if ton_connection else TonConnect()
+    if ton_connection:
+        await ton_connect.restore_connection()
     connection_link = await ton_connect.new_connection(wallet_app_name)
     ton_connect.connected
     memory.add_task(pause_ton_connection, ton_connect, delay_s=60 * 3)
@@ -100,10 +111,9 @@ async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, us
 
     start_ts = datetime.now()
     while datetime.now() - start_ts < timedelta(seconds=180):
-        new_connection = db_session.query(WalletConnection).filter(
-            WalletConnection.user_id == user.id,
-            WalletConnection.invalidated == False
-        ).first()
+        new_connection = (await db_session.execute(select(WalletConnection).where(
+            and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
+        ))).scalars().first()
         if new_connection:
             await tg_process_template(
                 chat_wrap, user.translated('p_successConnectWallet')
@@ -118,14 +128,13 @@ async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, us
 
 async def t_callback_disconnect_wallet(query: types.CallbackQuery, memory=None, user=None, db_session=None,
                                        chat_wrap=None, **extra):
-    wallet_connections = db_session.query(WalletConnection).filter(
-        WalletConnection.user_id == user.id,
-        WalletConnection.invalidated == False
-    ).all()
+    wallet_connections = (await db_session.execute(select(WalletConnection).where(
+        and_(WalletConnection.user_id == user.id, WalletConnection.invalidated == False)
+    ))).scalars().all()
     for wallet_connection in wallet_connections:
         wallet_connection.invalidated = True
 
-    db_session.commit()
+    await db_session.commit()
     return await send_home_menu(chat_wrap, user, None, message_id=query.message.message_id)
 
 

app/core/_blockchain/ton/contracts/blank.py

@@ -5,7 +5,7 @@ from app.core._secrets import service_wallet
 
 
 class Blank(Contract):
-    code = 'B5EE9C72010104010042000114FF00F4A413F4BCF2C80B010202CA0203004FD043A0E9AE43F48061DA89A1F480618E0BE5C323A803A1A843F60803A1DA3DDAA7A861DAA9E2026F0007A0DD7C12'
+    code = 'b5ee9c72410104010042000114ff00f4a413f4bcf2c80b010202ca03020007a0dd7c12004fd043a0e9ae43f48061da89a1f480618e0be5c323a803a1a843f60803a1da3ddaa7a861daa9e2026f102bdd33'
 
     def __init__(self, **kwargs):
         kwargs['code'] = Cell.one_from_boc(self.code)

app/core/_blockchain/ton/contracts/cop_nft.py

@@ -6,7 +6,7 @@ from app.core._config import MY_PLATFORM_CONTRACT
 
 
 class COP_NFT(Contract):
-    code = 'b5ee9c7241022f01000ac8000114ff00f4a413f4bcf2c80b010201620b020201200803020120070402016a0605004bae43b941781bb12d2f0dfa54dda7472bd6fbc06813c0f71615c0094ee3548791d80b2bfcecc0007dac1ff6a26869ff80fc317d2000fc31906ba4e1807c30fc20c70d698380fc327d2000fc32ea00fc336a00fc33ea00fc346a00fc34ef68fc21fc217c227c22c000cbb8fcfed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f8419ac8c9707f22d023d05503e1f849d0f846f00702987ff84504d4305e21e05b7ff842f843f84504d4301034413080201200a0900d5ba7a3ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f848d0d431d430f8287003c8cbffc94130c85003cf16cb07ccc97020c8cb0113f400f400cb00c9f9007074c8cb02ca07cbffc9d080093bb54ded44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded171f828f842f843f844f845f849f846f848f84780202c71e0c020148180d020272140e0101f40f01bac882f01bc04b5291c26a46d918139138b992d2de976d6851d0893b0476b85bfbdfc6e6228307f40e6fa130d3ff3001cbff82f03dbe3c57aae062079b4712b8a650cf2abe2d6f986e0ecf1785d2ba835974175d228307f40e6fa130cf1610015e82f0e3868b37bc801236c714f134200a4e3211860f3ad67d22be995321452723c516228307f40e6fa1923031e30dc91101c0d3073001cb0782f0862bd78e42e143bb51660c521752437648ef67967d5055093d11be1117c774b0228307f40e6fa130cf1682f0ade34d680d2df2ad88267395ea1c3b159c4df766416d69d2bbcc8e18e87bd2ba228307f40e6fa130d43001cc1201fe82f093354845030274cd4bf1686abd60ab28ec52e1a792fa6f7ccb9cbd0ddff53d12228307f40e6fa130d43001cc82f0852d0f3fd8bdef2aab5f91714d86638ddc57db97743a350fab3394bc9ebd9518228307f40e6fa130d43001cc82f089445ea08b55421faa49919a5fd272e9a520f701b479d6084847e161ca5b7711581300168307f40e6fa130d43001cc01bbf36fc216465ffc1780de025a948e135236c8c09c89c5cc9696f4bb6b428e8449d823b5c2dfdefe3732c4183fa21e47c21e78b41781edf1e2bd5703103cda3895c532867955f16b7cc3707678bc2e95d41acba0baeac4183fa21fc20f18041501fef844c8cb0782f0e3868b37bc801236c714f134200a4e3211860f3ad67d22be995321452723c516588307f443c8f845cf1682f0862bd78e42e143bb51660c521752437648ef67967d5055093d11be1117c774b0588307f443f846c8cc82f0ade34d680d2df2ad88267395ea1c3b159c4df766416d69d2bbcc8e18e87bd2ba581601bc8307f443f847c8cc82f093354845030274cd4bf1686abd60ab28ec52e1a792fa6f7ccb9cbd0ddff53d12588307f443f848c8cc82f0852d0f3fd8bdef2aab5f91714d86638ddc57db97743a350fab3394bc9ebd9518588307f443f849c8cc17004e82f089445ea08b55421faa49919a5fd272e9a520f701b479d6084847e161ca5b7711588307f4430202761b19017df1998e83a6b90fd201876a26869ff80fc317d2000fc31906ba4e1807c30fc20c70d698380fc327d2000fc32ea00fc336a00fc33ea00fc346a00fc34ef6880c1a01c8d3fffa4001f865f8435230c7058e5331fa405312c7058e416c12d30701f864f84271c8cb00cb3f58cf16c9f866d401f867d401f868d430f869f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54e05bf843c705f2e1afe30d2e0201201d1c00415f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54800694ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded180201cb201f0011f686900699ffd20184020148222100113e910c30003cb8536003f5007434c0c05c6c2497c1383e903e900c7e800c5c75c87e800c7e800c1cea6d003b513434ffc07e18be90007e18c835d270c03e187e106386b4c1c07e193e90007e1975007e19b5007e19f5007e1a35007e1a77b47e1078c0dc14c0f5d270882616c0b4c7f4cfd111378860840a8c6564eea3a116c4b6cf380d48202d272302da82105fcc3d14ba8e85304330db3ce031342382102fcb26a2ba8eb23132f846f007706d82108b771735c8cb1f16cb3f049702c8cbff01cf169b6c21f842c8cbfff843cf16e212cf17128040db3ce0320282102fa30f96ba9ff843c705f2e191d401fb04d430ed54e05b840ff2f0242c03e0f84514c705f2e191f844c000f844c003b1f2e191fa4021f001fa40d20031fa000682084c4b40a121945315a0a1de22d70b01c300209206a19136e220c2fff2e19223f865218e9d6d821005138d91c8cb1f5260cb3ff845cf165008cf1610341771db3c039410266c31e202925f03e30d2c26250040f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54012622f0016d8210d53276dbc8cb1f12cb3f71db3c2c01e6f847d0d403d30721c003f2e19af844c000f844c003b1f2e1a4d3fffa403020d70b01c000923025de05f4043003d022a59320c2009601fa003101a5e830fa0030208209312d005320bcf2e208f848d0d4d430f8287007c8cbffc9225088c85003cf16cb07ccc97020c8cb0113f400f400cb00c92803fef9007074c8cb02ca07cbffc9d01ac705965392bef2e208df5192a1f846f007228f5c50675f053333343434028107d0a8812710a904f8456d70c8cb1f8d04935648149959995c9c985b0814185e5bdd5d20cf16102372db3c82084c4b4070fb02706d732282102a319593c8cb1fcb3fcb0715cbff5003cf1613810082db3ce02c2c2901fe35f8287022c8cbffc91038c85003cf16cb07ccc97020c8cb0113f400f400cb00c920f9007074c8cb02ca07cbffc9d0f849d0c803d013cf16f849f8486d6dc870fa0270fa02500afa02c9c8cc19f40018f400c904d3ff3029c8cbfff843cf16c90fc8cc1fccc9c8cc1ecbff2ccf16f828cf1619cb0712cc14cc1acc10234a502a03b472db3ca405c8ca0015cb3f5005cf16c9f866f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed5422c2008e926d708210d53276dbc8cb1fcb3f102472db3c926c21e22282103b9aca00bc925f03e30d2c2c2b02e202821005f5e100a1208106a4a8812710a90466a1f8436d8210247a9ebac8cb1f1023102472db3c593121c2008ec08ebc78f4966fa531208eaf01d430d0fa40d30f305240a8812710a9046d70c8cb1f8d04535648149Line truncated
+    code = 'b5ee9c7241023201000bbc000114ff00f4a413f4bcf2c80b01020120040201f6f2ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f849d0d3ffd31f038308d71820f9015882f053b50ddfe5a9533f2e76ac054411db94432a1f7b7ae17fc64cf7aec5df8705d5f910f2e212fa40f82812c705f2e213d31f5213bc0301d4f2e21401d33f01f823bcf2e215f80082084c4b4070fb02d4308e3cd0d31f218210e3e30001ba8e1731f404216e91319301fb04e2f404216e91319301ed54e28e10018210e3e30002ba96d307d402fb00dee2f40430206ee63002d4d43002c8cbff13cb1f12ccccc9f869280201480e050201200b060201200a0702016a0908004bae43bac17829da86eff2d4a99f973b5602a208edca21950fbdbd70bfe3267bd762efc382eac0007dac1ff6a26869ff80fc317d2000fc31906ba4e1807c30fc20c70d698380fc327d2000fc32ea00fc336a00fc33ea00fc346a00fc34ef68fc21fc217c227c22c000cbb8fcfed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f8419ac8c9707f22d023d05503e1f849d0f846f00702987ff84504d4305e21e05b7ff842f843f84504d4301034413080201200d0c00d5ba7a3ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded1f848d0d431d430f8287003c8cbffc94130c85003cf16cb07ccc97020c8cb0113f400f400cb00c9f9007074c8cb02ca07cbffc9d080093bb54ded44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded171f828f842f843f844f845f849f846f848f84780202c7210f0201481b1002027217110101f41201bac882f01bc04b5291c26a46d918139138b992d2de976d6851d0893b0476b85bfbdfc6e6228307f40e6fa130d3ff3001cbff82f03dbe3c57aae062079b4712b8a650cf2abe2d6f986e0ecf1785d2ba835974175d228307f40e6fa130cf1613015e82f0e3868b37bc801236c714f134200a4e3211860f3ad67d22be995321452723c516228307f40e6fa1923031e30dc91401c0d3073001cb0782f0862bd78e42e143bb51660c521752437648ef67967d5055093d11be1117c774b0228307f40e6fa130cf1682f0ade34d680d2df2ad88267395ea1c3b159c4df766416d69d2bbcc8e18e87bd2ba228307f40e6fa130d43001cc1501fe82f093354845030274cd4bf1686abd60ab28ec52e1a792fa6f7ccb9cbd0ddff53d12228307f40e6fa130d43001cc82f0852d0f3fd8bdef2aab5f91714d86638ddc57db97743a350fab3394bc9ebd9518228307f40e6fa130d43001cc82f089445ea08b55421faa49919a5fd272e9a520f701b479d6084847e161ca5b7711581600168307f40e6fa130d43001cc01bbf36fc216465ffc1780de025a948e135236c8c09c89c5cc9696f4bb6b428e8449d823b5c2dfdefe3732c4183fa21e47c21e78b41781edf1e2bd5703103cda3895c532867955f16b7cc3707678bc2e95d41acba0baeac4183fa21fc20f18041801fef844c8cb0782f0e3868b37bc801236c714f134200a4e3211860f3ad67d22be995321452723c516588307f443c8f845cf1682f0862bd78e42e143bb51660c521752437648ef67967d5055093d11be1117c774b0588307f443f846c8cc82f0ade34d680d2df2ad88267395ea1c3b159c4df766416d69d2bbcc8e18e87bd2ba581901bc8307f443f847c8cc82f093354845030274cd4bf1686abd60ab28ec52e1a792fa6f7ccb9cbd0ddff53d12588307f443f848c8cc82f0852d0f3fd8bdef2aab5f91714d86638ddc57db97743a350fab3394bc9ebd9518588307f443f849c8cc1a004e82f089445ea08b55421faa49919a5fd272e9a520f701b479d6084847e161ca5b7711588307f4430202761e1c017df1998e83a6b90fd201876a26869ff80fc317d2000fc31906ba4e1807c30fc20c70d698380fc327d2000fc32ea00fc336a00fc33ea00fc346a00fc34ef6880c1d01c8d3fffa4001f865f8435230c7058e5331fa405312c7058e416c12d30701f864f84271c8cb00cb3f58cf16c9f866d401f867d401f868d430f869f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54e05bf843c705f2e1afe30d31020120201f00415f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54800694ed44d0d3ff01f862fa4001f86320d749c300f861f8418e1ad30701f864fa4001f865d401f866d401f867d401f868d401f869ded180201cb23220011f686900699ffd20184020148252400113e910c30003cb8536003f5007434c0c05c6c2497c1383e903e900c7e800c5c75c87e800c7e800c1cea6d003b513434ffc07e18be90007e18c835d270c03e187e106386b4c1c07e193e90007e1975007e19b5007e19f5007e1a35007e1a77b47e1078c0dc14c0f5d270882616c0b4c7f4cfd111378860840a8c6564eea3a116c4b6cf380d4820302a2602da82105fcc3d14ba8e85304330db3ce031342382102fcb26a2ba8eb23132f846f007706d82108b771735c8cb1f16cb3f049702c8cbff01cf169b6c21f842c8cbfff843cf16e212cf17128040db3ce0320282102fa30f96ba9ff843c705f2e191d401fb04d430ed54e05b840ff2f0272f03e0f84514c705f2e191f844c000f844c003b1f2e191fa4021f001fa40d20031fa000682084c4b40a121945315a0a1de22d70b01c300209206a19136e220c2fff2e19223f865218e9d6d821005138d91c8cb1f5260cb3ff845cf165008cf1610341771db3c039410266c31e202925f03e30d2f29280040f849f848f847f846f844f842c8cbfff843cf16cb07f845cf16ccccccccc9ed54012622f0016d8210d53276dbc8cb1f12cb3f71db3c2f01e6f847d0d403d30721c003f2e19af844c000f844c003b1f2e1a4d3fffa403020d70b01c000923025de05f4043003d022a59320c2009601fa003101a5e830fa0030208209312d005320bcf2e208f848d0d4d430f8287007c8cbffc9225088c85003cf16cb07ccc97020c8cb0113f400f400cb00c92b03fef9007074c8cb02ca07cbffc9d01ac705965392bef2e208df5192a1f846f007228f5c50675f053333343434028107d0a8812710a904f8456d70c8cb1f8d04935648149959995c9c985b0814185e5bdd5d20cf16102372db3c82084c4b4070fb02706d732282102a319593c8cb1fcb3fcb0715cbff5003cf1613810082db3ce02f2f2c01fe35f8287022c8cbffc91038c85003cf16cb07ccc97020c8cb0113f400f400cb00c920f9007074c8cb02ca07cbffc9d0f849d0c803d013cf16f849f8486d6dc870fa0270fa02500Line truncated
     codebase_version = 5
 
     def __init__(self, **kwargs):

app/core/_blockchain/ton/contracts/platform.py

@@ -3,7 +3,7 @@ from tonsdk.contract import Contract
 
 
 class Platform(Contract):
-    code = 'b5ee9c7241021601000310000114ff00f4a413f4bcf2c80b010201620d0202012006030201200504004bbac877282f037625a5e1bf4a9bb4e8e57adf780d02781ee2c2b80129dc6a90f23b01657f9d980057b905bed44d0fa4001f861d3ff01f862d401f863f843d0d431d430f864d401f865d1f845d0f84201d430f84180201200a07020120090800a1b4f47da89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba3f089f050e0079197ff92826190a0079e2d960f9992e04191960227e801e801960193f200e0e9919605940f97ff93a10000fb5daeeb00c9f05100201200c0b0059b6a9bda89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba2e1f051f085f087f089f08b00051b56ba63da89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba2e391960f999300202c70f0e0007a0dd7c120201cf111000113e910c30003cb8536002f30cf434c0c05c6c2497c0f83e90087c007e900c7e800c5c75c87e800c7e800c1cea6d0008f5d27048245c2540f4c7d411388830002497c1783b51343e90007e1874ffc07e18b5007e18fe10f4350c750c3e1935007e1974482084091ea7aeaea497c178082084152474232ea3a14c104c36cf380c4cbe1071c160131201dcf2e19120820833cc77ba9730d4d30730fb00e0208210b99cd03bba9701fa4001f86101de208210d81c632fba9601d401f86501de208210b5de5f9eba8e8b30fa40fa00306d6d71db3ce082102fa30f96ba98d401fb04d430ed54e030f845f843f842c8f841cf16cbffccccc9ed541502f082084c4b4001a013bef2e20801d3ffd4d430f844f82870f842c8cbffc9c85003cf16cb07ccc97020c8cb0113f400f400cb00c920f9007074c8cb02ca07cbffc9d0f843d070c804d014cf16f843f842c8cbfff828cf16c903d430c8cc13ccc9c8cc17cbff5007cf1614cc15cccc43308040db3cf842a4f86215140024f845f843f842c8f841cf16cbffccccc9ed540078708010c8cb055006cf165004fa0214cb68216e947032cb019bc858cf17c97158cb00f400e2226e95327058cb0099c85003cf17c958f400e2c901fb004e32cb65'
+    code = 'b5ee9c724102160100032e000114ff00f4a413f4bcf2c80b010201620d0202012006030201200504004bbac877582f053b50ddfe5a9533f2e76ac054411db94432a1f7b7ae17fc64cf7aec5df8705d580057b905bed44d0fa4001f861d3ff01f862d401f863f843d0d431d430f864d401f865d1f845d0f84201d430f84180201200a07020120090800a1b4f47da89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba3f089f050e0079197ff92826190a0079e2d960f9992e04191960227e801e801960193f200e0e9919605940f97ff93a10000fb5daeeb00c9f05100201200c0b0059b6a9bda89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba2e1f051f085f087f089f08b00051b56ba63da89a1f48003f0c3a7fe03f0c5a803f0c7f087a1a863a861f0c9a803f0cba2e391960f999300202c70f0e0007a0dd7c120201cf111000113e910c30003cb8536002f30cf434c0c05c6c2497c0f83e90087c007e900c7e800c5c75c87e800c7e800c1cea6d0008f5d27048245c2540f4c7d411388830002497c1783b51343e90007e1874ffc07e18b5007e18fe10f4350c750c3e1935007e1974482084091ea7aeaea497c178082084152474232ea3a14c104c36cf380c4cbe1071c160131201faf2e19120820833cc77ba9730d4d30730fb00e0208210b99cd03bba9701fa4001f86101de208210d81c632fba9601d401f86501de208210b5de5f9eba8e8b30fa40fa00306d6d71db3ce082102fa30f96ba8e16f404216e91319301fb04e2f40430206e913092ed54e2e030f845f843f842c8f841cf16cbffccccc9ed541501f682084c4b4001a013bef2e20801d3fffa4021d70b01c0009231029133e202d4d430f844f82870f842c8cbffc9c85003cf16cb07ccc97020c8cb0113f400f400cb00c920f9007074c8cb02ca07cbffc9d0f843d070c804d014cf16f843f842c8cbfff828cf16c903d430c8cc13ccc9c8cc17cbff5007cf1614cc15cc14013ccc43308040db3cf842a4f862f845f843f842c8f841cf16cbffccccc9ed54150078708010c8cb055006cf165004fa0214cb68216e947032cb009bc858cf17c97158cb00f400e2226e95327058cb0099c85003cf17c958f400e2c901fb003366cbbe'
     codebase_version = 5
 
     def __init__(self, **kwargs):

app/core/_blockchain/ton/platform.py

@@ -12,11 +12,34 @@ kwargs = {}
 if int(os.getenv('INIT_DEPLOY_PLATFORM_CONTRACT', 0)) == 0:
     kwargs['address'] = Address(MY_PLATFORM_CONTRACT)
 
-platform = Platform(
-    admin_address=Address('UQAjz4Kdqoo4_Obg-UrUmuhoUB2W00vngZoX0MnAAnetZuAk'),
-    blank_code=Cell.one_from_boc(Blank.code),
-    cop_code=Cell.one_from_boc(COP_NFT.code),
+def platform_with_salt(s: int = 0):
+    return Platform(
+        admin_address=Address('UQD3XALhbETNo7ItrdPNFzMJtRHC5u6dIb39DCYa40jnWZdg'),
+        blank_code=Cell.one_from_boc(Blank.code),
+        cop_code=Cell.one_from_boc(COP_NFT.code),
+        collection_content_uri=f'{PROJECT_HOST}/api/platform-metadata.json' + f"?s={s}",
+        **kwargs
+    )
 
-    collection_content_uri=f'{PROJECT_HOST}/api/platform-metadata.json',
-    **kwargs
-)
+platform = platform_with_salt()
+
+if int(os.getenv('INIT_DEPLOY_PLATFORM_CONTRACT', 0)) == 1:
+    def is_nice_address(address: Address):
+        bounceable_addr = address.to_string(True, True, True)
+        non_bounceable_addr = address.to_string(True, True, False)
+
+        if '-' in bounceable_addr or '-' in non_bounceable_addr:
+            return False
+
+        if '_' in bounceable_addr or '_' in non_bounceable_addr:
+            return False
+
+        if bounceable_addr[-1] != 'A':
+            return False
+        
+        return True
+
+    salt_value = 0
+    while not is_nice_address(platform.address):
+        platform = platform_with_salt(salt_value)
+        salt_value += 1

app/core/_config.py

@@ -7,7 +7,12 @@ load_dotenv(dotenv_path='.env')
 
 PROJECT_HOST = os.getenv('PROJECT_HOST', 'http://127.0.0.1:8080')
 SANIC_PORT = int(os.getenv('SANIC_PORT', '8080'))
+# Path inside the running backend container where content files are visible
 UPLOADS_DIR = os.getenv('UPLOADS_DIR', '/app/data')
+# Host path where the same content directory is mounted (used for docker -v from within container)
+BACKEND_DATA_DIR_HOST = os.getenv('BACKEND_DATA_DIR_HOST', '/Storage/storedContent')
+# Host path for converter logs (used for docker -v). Optional.
+BACKEND_LOGS_DIR_HOST = os.getenv('BACKEND_LOGS_DIR_HOST', '/Storage/logs/converter')
 if not os.path.exists(UPLOADS_DIR):
     os.makedirs(UPLOADS_DIR)
 
@@ -16,12 +21,31 @@ assert TELEGRAM_API_KEY, "Telegram API_KEY required"
 CLIENT_TELEGRAM_API_KEY = os.environ.get('CLIENT_TELEGRAM_API_KEY')
 assert CLIENT_TELEGRAM_API_KEY, "Client Telegram API_KEY required"
 import httpx
-TELEGRAM_BOT_USERNAME = httpx.get(f"https://api.telegram.org/bot{TELEGRAM_API_KEY}/getMe").json()['result']['username']
-CLIENT_TELEGRAM_BOT_USERNAME = httpx.get(f"https://api.telegram.org/bot{CLIENT_TELEGRAM_API_KEY}/getMe").json()['result']['username']
 
 
-MYSQL_URI = os.environ['MYSQL_URI']
-MYSQL_DATABASE = os.environ['MYSQL_DATABASE']
+def _resolve_bot_username(token: str, label: str) -> str:
+    try:
+        resp = httpx.get(f"https://api.telegram.org/bot{token}/getMe", timeout=10.0)
+        resp.raise_for_status()
+        payload = resp.json()
+    except Exception as exc:
+        raise RuntimeError(f"{label} Telegram token validation failed: {exc}") from exc
+
+    if not payload.get('ok'):
+        detail = payload.get('description') or 'unknown Telegram API error'
+        raise RuntimeError(f"{label} Telegram token validation failed: {detail}")
+
+    username = (payload.get('result') or {}).get('username')
+    if not username:
+        raise RuntimeError(f"{label} Telegram token validation failed: username missing in Telegram response")
+    return username
+
+
+TELEGRAM_BOT_USERNAME = _resolve_bot_username(TELEGRAM_API_KEY, 'Uploader bot')
+CLIENT_TELEGRAM_BOT_USERNAME = _resolve_bot_username(CLIENT_TELEGRAM_API_KEY, 'Client bot')
+
+# Unified database URL (PostgreSQL)
+DATABASE_URL = os.environ['DATABASE_URL']
 
 LOG_LEVEL = os.getenv('LOG_LEVEL', 'DEBUG')
 LOG_DIR = os.getenv('LOG_DIR', 'logs')
@@ -32,7 +56,7 @@ _now_str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
 LOG_FILEPATH = f"{LOG_DIR}/{_now_str}.log"
 
 WEB_APP_URLS = {
-    'uploadContent': f"https://web2-client.vercel.app/uploadContent"
+    'uploadContent': f"https://my-public-node-8.projscale.dev/uploadContent"
 }
 
 ALLOWED_CONTENT_TYPES = [
@@ -48,5 +72,5 @@ TONCENTER_HOST = os.getenv('TONCENTER_HOST', 'https://toncenter.com/api/v2/')
 TONCENTER_API_KEY = os.getenv('TONCENTER_API_KEY')
 TONCENTER_V3_HOST = os.getenv('TONCENTER_V3_HOST', 'https://toncenter.com/api/v3/')
 
-MY_PLATFORM_CONTRACT = 'EQDmWp6hbJlYUrXZKb9N88sOrTit630ZuRijfYdXEHLtheMY'
+MY_PLATFORM_CONTRACT = 'EQBVjuNuaIK87v9nm7mghgJ41ikqfx3GNBFz05GfmNbRQ9EA'
 MY_FUND_ADDRESS = 'UQDarChHFMOI2On9IdHJNeEKttqepgo0AY4bG1trw8OAAwMY'

app/core/_crypto/content.py

@@ -36,9 +36,10 @@ async def create_new_encryption_key(db_session, user_id: int = None) -> KnownKey
         meta={"I_user_id": user_id} if user_id else None,
         created=datetime.now()
     )
+    from sqlalchemy import select
     db_session.add(new_key)
-    db_session.commit()
-    new_key = db_session.query(KnownKey).filter(KnownKey.seed_hash == new_seed_hash).first()
+    await db_session.commit()
+    new_key = (await db_session.execute(select(KnownKey).where(KnownKey.seed_hash == new_seed_hash))).scalars().first()
     assert new_key, "Key not created"
     return new_key
 
@@ -46,42 +47,51 @@ async def create_new_encryption_key(db_session, user_id: int = None) -> KnownKey
 async def create_encrypted_content(
     db_session, decrypted_content: StoredContent,
 ) -> StoredContent:
-    encrypted_content = db_session.query(StoredContent).filter(
-        StoredContent.id == decrypted_content.decrypted_content_id
-    ).first()
+    from sqlalchemy import select
+    # Try to find an already created encrypted counterpart for this decrypted content
+    encrypted_content = (
+        await db_session.execute(
+            select(StoredContent).where(StoredContent.decrypted_content_id == decrypted_content.id)
+        )
+    ).scalars().first()
     if encrypted_content:
         make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): already exist (found by decrypted content)", level="debug")
         return encrypted_content
 
     encrypted_content = None
-    if decrypted_content.key is None:
+    # Avoid accessing relationship attributes in async context to prevent MissingGreenlet
+    if not decrypted_content.key_id:
         key = await create_new_encryption_key(db_session, user_id=decrypted_content.user_id)
         decrypted_content.key_id = key.id
-        db_session.commit()
-        decrypted_content = db_session.query(StoredContent).filter(
-            StoredContent.id == decrypted_content.id
-        ).first()
+        await db_session.commit()
         assert decrypted_content.key_id, "Key not assigned"
 
+    # Explicitly load the key to avoid lazy-loading via relationship in async mode
+    key = (
+        await db_session.execute(select(KnownKey).where(KnownKey.id == decrypted_content.key_id))
+    ).scalars().first()
+    # If the referenced key is missing or malformed, create a fresh one
+    if not key or not key.seed:
+        key = await create_new_encryption_key(db_session, user_id=decrypted_content.user_id)
+        decrypted_content.key_id = key.id
+        await db_session.commit()
+
     decrypted_path = os.path.join(UPLOADS_DIR, decrypted_content.hash)
     decrypted_bin = b58decode(decrypted_content.hash)
 
-    key = decrypted_content.key
     cipher = AESCipher(key.seed_bin)
 
     encrypted_bin = cipher.encrypt(decrypted_bin)
     encrypted_hash_bin = sha256(encrypted_bin).digest()
     encrypted_hash = b58encode(encrypted_hash_bin).decode()
-    encrypted_content = db_session.query(StoredContent).filter(
-        StoredContent.hash == encrypted_hash
-    ).first()
+    encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash))).scalars().first()
     if encrypted_content:
         make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): already exist (found by encrypted_hash)", level="debug")
         return encrypted_content
 
     encrypted_content = None
 
-    encrypted_meta = decrypted_content.meta
+    encrypted_meta = dict(decrypted_content.meta or {})
     encrypted_meta["encrypt_algo"] = "AES256"
 
     encrypted_content = StoredContent(
@@ -99,19 +109,15 @@ async def create_encrypted_content(
         created=datetime.now(),
     )
     db_session.add(encrypted_content)
-    db_session.commit()
+    await db_session.commit()
 
     encrypted_path = os.path.join(UPLOADS_DIR, encrypted_hash)
     async with aiofiles.open(encrypted_path, mode='wb') as file:
         await file.write(encrypted_bin)
 
-    encrypted_content = db_session.query(StoredContent).filter(
-        StoredContent.hash == encrypted_hash
-    ).first()
+    encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash))).scalars().first()
     assert encrypted_content, "Content not created"
     make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): created new content/bin", level="debug")
     return encrypted_content
 
 
-
-

app/core/_secrets.py

@@ -1,44 +1,115 @@
 from os import getenv, urandom
+import os
+import time
+import json
 
 from nacl.bindings import crypto_sign_seed_keypair
 from tonsdk.utils import Address
 
 from app.core._blockchain.ton.wallet_v3cr3 import WalletV3CR3
-from app.core.models._config import ServiceConfig
-from app.core.storage import db_session
 from app.core.logger import make_log
-import os
+from sqlalchemy import create_engine, inspect
+from sqlalchemy.orm import Session
+from typing import Optional
+from app.core.models._config import ServiceConfigValue
 
 
-def load_hot_pair():
-    with db_session() as session:
-        service_config = ServiceConfig(session)
-        hot_seed = service_config.get('private_key')
-        if hot_seed is None:
-            make_log("HotWallet", "No seed found, generating new one", level='info')
-            hot_seed = os.getenv("TON_INIT_HOT_SEED")
-            if not hot_seed:
-                hot_seed = urandom(32)
-                make_log("HotWallet", f"Generated random seed")
-            else:
-                hot_seed = bytes.fromhex(hot_seed)
-                make_log("HotWallet", f"Loaded seed from env")
+def _load_seed_from_env_or_generate() -> bytes:
+    seed_hex = os.getenv("TON_INIT_HOT_SEED")
+    if seed_hex:
+        make_log("HotWallet", "Loaded seed from env")
+        return bytes.fromhex(seed_hex)
+    make_log("HotWallet", "No seed provided; generating ephemeral seed", level='info')
+    return urandom(32)
 
-            service_config.set('private_key', hot_seed.hex())
-            return load_hot_pair()
 
-        hot_seed = bytes.fromhex(hot_seed)
-        public_key, private_key = crypto_sign_seed_keypair(hot_seed)
-        return hot_seed, public_key, private_key
+def _init_seed_via_db() -> bytes:
+    """Store and read hot seed from PostgreSQL service_config (key='private_key').
+    Primary node writes it once; workers wait until it appears.
+    """
+    from app.core._config import DATABASE_URL
+
+    engine = create_engine(DATABASE_URL, pool_pre_ping=True)
+    role = os.getenv("NODE_ROLE", "worker").lower()
+
+    def db_ready(conn) -> bool:
+        try:
+            inspector = inspect(conn)
+            return inspector.has_table('service_config')
+        except Exception:
+            return False
+
+    # Wait for table to exist
+    start = time.time()
+    # Wait for table existence, reconnecting to avoid stale transactions
+    while True:
+        with engine.connect() as conn:
+            if db_ready(conn):
+                break
+        time.sleep(0.5)
+        if time.time() - start > 120:
+            raise TimeoutError("service_config table not available")
+
+    def read_seed() -> Optional[bytes]:
+        # Use a fresh connection/session per read to avoid snapshot staleness
+        try:
+            with engine.connect() as rconn:
+                with Session(bind=rconn) as s:
+                    row = s.query(ServiceConfigValue).filter(ServiceConfigValue.key == 'private_key').first()
+                    if not row:
+                        return None
+                    packed = row.packed_value or {}
+                    if isinstance(packed, str):
+                        packed = json.loads(packed)
+                    seed_hex = packed.get('value')
+                    return bytes.fromhex(seed_hex) if seed_hex else None
+        except Exception:
+            return None
+
+    seed = read_seed()
+    if seed:
+        return seed
+
+    if role == "primary":
+        seed = _load_seed_from_env_or_generate()
+        # Try insert; if another primary raced, ignore
+        try:
+            with engine.connect() as wconn:
+                with Session(bind=wconn) as s:
+                    s.add(ServiceConfigValue(key='private_key', packed_value={"value": seed.hex()}))
+                    s.commit()
+            make_log("HotWallet", "Seed saved in service_config by primary", level='info')
+            return seed
+        except Exception:
+            # Read again in case of race
+            seed2 = read_seed()
+            if seed2:
+                return seed2
+            raise
+    else:
+        make_log("HotWallet", "Worker waiting for seed in service_config...", level='info')
+        while True:
+            seed = read_seed()
+            if seed:
+                return seed
+            time.sleep(0.5)
 
 
 _extra_ton_wallet_options = {}
 if getenv('TON_CUSTOM_WALLET_ADDRESS'):
     _extra_ton_wallet_options['address'] = Address(getenv('TON_CUSTOM_WALLET_ADDRESS'))
 
-hot_seed, hot_pubkey, hot_privkey = load_hot_pair()
-service_wallet = WalletV3CR3(
-    private_key=hot_privkey,
-    public_key=hot_pubkey,
-    **_extra_ton_wallet_options
-)
+
+def _init_wallet():
+    # Primary writes to DB; workers wait and read from DB
+    hot_seed_bytes = _init_seed_via_db()
+    pub, priv = crypto_sign_seed_keypair(hot_seed_bytes)
+    wallet = WalletV3CR3(
+        private_key=priv,
+        public_key=pub,
+        **_extra_ton_wallet_options
+    )
+    return hot_seed_bytes, pub, priv, wallet
+
+
+hot_seed, hot_pubkey, hot_privkey, service_wallet = _init_wallet()

app/core/_utils/create_maria_tables.py

@@ -1,10 +1,12 @@
-from app.core.models import Asset
+from sqlalchemy.ext.asyncio import AsyncEngine
+from app.core.models import BlockchainTask
 from app.core.models.base import AlchemyBase
 
 
-def create_maria_tables(engine):
-    """Create all tables in the database."""
-    Asset()
-    AlchemyBase.metadata.create_all(engine)
+async def create_db_tables(engine: AsyncEngine):
+    """Create all tables in the database (PostgreSQL, async)."""
+    # ensure model import side-effects initialize mappers
+    BlockchainTask()
+    async with engine.begin() as conn:
+        await conn.run_sync(AlchemyBase.metadata.create_all)
     
-

app/core/_utils/send_status.py

@@ -5,7 +5,6 @@ from httpx import AsyncClient
 
 from app.core._config import PROJECT_HOST
 from app.core._crypto.signer import Signer
-from app.core._secrets import hot_seed
 from app.core.logger import make_log
 
 
@@ -17,6 +16,8 @@ async def send_status(service: str, status: str):
             'status': status,
         }
         message_bytes = dumps(message).encode()
+        # Lazy import to avoid triggering _secrets before DB is ready
+        from app.core._secrets import hot_seed
         signer = Signer(hot_seed)
         message_signature = signer.sign(message_bytes)
         async with AsyncClient() as client:

app/core/auth_v1.py

@@ -56,9 +56,10 @@ class AuthenticationMixin:
             },
             created=datetime.fromtimestamp(init_ts)
         )
+        from sqlalchemy import select
         db_session.add(new_key)
-        db_session.commit()
-        new_key = db_session.query(KnownKey).filter(KnownKey.seed_hash == new_key.seed_hash).first()
+        await db_session.commit()
+        new_key = (await db_session.execute(select(KnownKey).where(KnownKey.seed_hash == new_key.seed_hash))).scalars().first()
         assert new_key, "Key not created"
         make_log("auth", f"[new-K] User {user_id} created new {token_type} key {new_key.id}")
         return {

app/core/background/convert_service.py

@@ -4,8 +4,9 @@ import os
 import uuid
 import json
 import shutil
+import magic  # python-magic for MIME detection
 from base58 import b58decode, b58encode
-from sqlalchemy import and_, or_
+from sqlalchemy import and_, or_, select
 from app.core.models.node_storage import StoredContent
 from app.core.models._telegram import Wrapped_CBotChat
 from app.core._utils.send_status import send_status
@@ -13,14 +14,14 @@ from app.core.logger import make_log
 from app.core.models.user import User
 from app.core.models import WalletConnection
 from app.core.storage import db_session
-from app.core._config import UPLOADS_DIR
+from app.core._config import UPLOADS_DIR, BACKEND_DATA_DIR_HOST, BACKEND_LOGS_DIR_HOST
 from app.core.content.content_id import ContentId
 
 
 async def convert_loop(memory):
-    with db_session() as session:
+    async with db_session() as session:
         # Query for unprocessed encrypted content
-        unprocessed_encrypted_content = session.query(StoredContent).filter(
+        unprocessed_encrypted_content = (await session.execute(select(StoredContent).where(
             and_(
                 StoredContent.type == "onchain/content",
                 or_(
@@ -28,66 +29,116 @@ async def convert_loop(memory):
                     StoredContent.ipfs_cid == None,
                 )
             )
-        ).first()
+        ))).scalars().first()
         if not unprocessed_encrypted_content:
             make_log("ConvertProcess", "No content to convert", level="debug")
             return
-        
+
+        # Достаем расшифрованный файл
+        decrypted_content = (await session.execute(select(StoredContent).where(
+            StoredContent.id == unprocessed_encrypted_content.decrypted_content_id
+        ))).scalars().first()
+        if not decrypted_content:
+            make_log("ConvertProcess", "Decrypted content not found", level="error")
+            return
+
+        # Определяем путь и расширение входного файла
+        # Путь внутри текущего контейнера (доступен Python процессу)
+        input_file_container = os.path.join(UPLOADS_DIR, decrypted_content.hash)
+        # Хостовый путь (нужен для docker -v маппинга при запуске конвертера)
+        input_file_host = os.path.join(BACKEND_DATA_DIR_HOST, decrypted_content.hash)
+        input_ext = (unprocessed_encrypted_content.filename.split('.')[-1]
+                     if '.' in unprocessed_encrypted_content.filename else "mp4")
+
+        # ==== Новая логика: определение MIME-тип через python-magic ====
+        try:
+            mime_type = magic.from_file(input_file_container, mime=True)
+        except Exception as e:
+            make_log("ConvertProcess", f"magic probe failed: {e}", level="warning")
+            mime_type = ""
+
+        if mime_type.startswith("video/"):
+            content_kind = "video"
+        elif mime_type.startswith("audio/"):
+            content_kind = "audio"
+        else:
+            content_kind = "other"
+
+        make_log("ConvertProcess", f"Detected content_kind={content_kind}, mime={mime_type}", level="info")
+
+        # Для прочих типов сохраняем raw копию и выходим
+        if content_kind == "other":
+            make_log("ConvertProcess", f"Content {unprocessed_encrypted_content.id} processed. Not audio/video, copy just", level="info")
+            unprocessed_encrypted_content.btfs_cid = ContentId(
+                version=2, content_hash=b58decode(decrypted_content.hash)
+            ).serialize_v2()
+            unprocessed_encrypted_content.ipfs_cid = ContentId(
+                version=2, content_hash=b58decode(decrypted_content.hash)
+            ).serialize_v2()
+            unprocessed_encrypted_content.meta = {
+                **unprocessed_encrypted_content.meta,
+                'converted_content': {
+                    option_name: decrypted_content.hash for option_name in ['high', 'low', 'low_preview']
+                }
+            }
+            await session.commit()
+            return
+
+        # ==== Конвертация для видео или аудио: оригинальная логика ====
         # Static preview interval in seconds
         preview_interval = [0, 30]
         if unprocessed_encrypted_content.onchain_index in [2]:
             preview_interval = [0, 60]
 
-        make_log("ConvertProcess", f"Processing content {unprocessed_encrypted_content.id} with preview interval {preview_interval}", level="info")
-        decrypted_content = session.query(StoredContent).filter(
-            StoredContent.id == unprocessed_encrypted_content.decrypted_content_id
-        ).first()
-        if not decrypted_content:
-            make_log("ConvertProcess", "Decrypted content not found", level="error")
-            return
+        make_log(
+            "ConvertProcess",
+            f"Processing content {unprocessed_encrypted_content.id} as {content_kind} with preview interval {preview_interval}",
+            level="info"
+        )
 
+        # Выбираем опции конвертации для видео и аудио
+        if content_kind == "video":
+            REQUIRED_CONVERT_OPTIONS = ['high', 'low', 'low_preview']
+        else:
+            REQUIRED_CONVERT_OPTIONS = ['high', 'low']  # no preview for audio
 
-        # List of conversion options to process
-        REQUIRED_CONVERT_OPTIONS = ['high', 'low', 'low_preview']
-        converted_content = {}  # Mapping: option -> sha256 hash of output file
+        converted_content = {}
+        # Директория логов на хосте для docker-контейнера конвертера
+        logs_dir_host = BACKEND_LOGS_DIR_HOST
 
-        # Define input file path and extract its extension from filename
-        input_file_path = f"/Storage/storedContent/{decrypted_content.hash}"
-
-
-        input_ext = unprocessed_encrypted_content.filename.split('.')[-1] if '.' in unprocessed_encrypted_content.filename else "mp4"
-
-        # Logs directory mapping
-        logs_dir = "/Storage/logs/converter"
-
-        # Process each conversion option in sequence
         for option in REQUIRED_CONVERT_OPTIONS:
             # Set quality parameter and trim option (only for preview)
             if option == "low_preview":
                 quality = "low"
                 trim_value = f"{preview_interval[0]}-{preview_interval[1]}"
             else:
-                quality = option  # 'high' or 'low'
+                quality = option
                 trim_value = None
 
             # Generate a unique output directory for docker container
             output_uuid = str(uuid.uuid4())
-            output_dir = f"/Storage/storedContent/converter-output/{output_uuid}"
+            # Директория вывода в текущем контейнере (та же что и в UPLOADS_DIR, смонтирована с хоста)
+            output_dir_container = os.path.join(UPLOADS_DIR, "converter-output", output_uuid)
+            os.makedirs(output_dir_container, exist_ok=True)
+            # Соответствующая директория на хосте — нужна для docker -v
+            output_dir_host = os.path.join(BACKEND_DATA_DIR_HOST, "converter-output", output_uuid)
 
-            # Build the docker command with appropriate volume mounts and parameters
+            # Build the docker command
             cmd = [
                 "docker", "run", "--rm",
-                "-v", f"{input_file_path}:/app/input",
-                "-v", f"{output_dir}:/app/output",
-                "-v", f"{logs_dir}:/app/logs",
+                # Важно: источники - это ХОСТОВЫЕ пути, так как docker демону они нужны на хосте
+                "-v", f"{input_file_host}:/app/input:ro",
+                "-v", f"{output_dir_host}:/app/output",
+                "-v", f"{logs_dir_host}:/app/logs",
                 "media_converter",
                 "--ext", input_ext,
                 "--quality", quality
             ]
             if trim_value:
                 cmd.extend(["--trim", trim_value])
+            if content_kind == "audio":
+                cmd.append("--audio-only")  # audio-only flag
 
-            # Run the docker container asynchronously
             process = await asyncio.create_subprocess_exec(
                 *cmd,
                 stdout=asyncio.subprocess.PIPE,
@@ -98,22 +149,21 @@ async def convert_loop(memory):
                 make_log("ConvertProcess", f"Docker conversion failed for option {option}: {stderr.decode()}", level="error")
                 return
 
-            # List files in the output directory
+            # List files in output dir
             try:
-                files = os.listdir(output_dir.replace("/Storage/storedContent", "/app/data"))
+                files = os.listdir(output_dir_container)
             except Exception as e:
                 make_log("ConvertProcess", f"Error reading output directory {output_dir}: {e}", level="error")
                 return
 
-            # Exclude 'output.json' and expect exactly one media output file
             media_files = [f for f in files if f != "output.json"]
             if len(media_files) != 1:
                 make_log("ConvertProcess", f"Expected one media file, found {len(media_files)} for option {option}", level="error")
                 return
 
-            output_file = os.path.join(output_dir.replace("/Storage/storedContent", "/app/data"), media_files[0])
+            output_file = os.path.join(output_dir_container, media_files[0])
 
-            # Compute SHA256 hash of the output file using async subprocess
+            # Compute SHA256 hash of the output file
             hash_process = await asyncio.create_subprocess_exec(
                 "sha256sum", output_file,
                 stdout=asyncio.subprocess.PIPE,
@@ -126,21 +176,18 @@ async def convert_loop(memory):
             file_hash = hash_stdout.decode().split()[0]
             file_hash = b58encode(bytes.fromhex(file_hash)).decode()
 
-            if not session.query(StoredContent).filter(
-                StoredContent.hash == file_hash
-            ).first():
+            # Save new StoredContent if not exists
+            if not (await session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first():
                 new_content = StoredContent(
                     type="local/content_bin",
                     hash=file_hash,
                     user_id=unprocessed_encrypted_content.user_id,
                     filename=media_files[0],
-                    meta={
-                        'encrypted_file_hash': unprocessed_encrypted_content.hash,
-                    },
+                    meta={'encrypted_file_hash': unprocessed_encrypted_content.hash},
                     created=datetime.now(),
                 )
                 session.add(new_content)
-                session.commit()
+                await session.commit()
 
             save_path = os.path.join(UPLOADS_DIR, file_hash)
             try:
@@ -156,41 +203,29 @@ async def convert_loop(memory):
 
             converted_content[option] = file_hash
 
-            # Process output.json: read its contents and update meta['ffprobe_meta']
-            output_json_path = os.path.join(output_dir.replace("/Storage/storedContent", "/app/data"), "output.json")
-            if os.path.exists(output_json_path):
-                if unprocessed_encrypted_content.meta.get('ffprobe_meta') is None:
-                    try:
-                        with open(output_json_path, "r") as f:
-                            output_json_content = f.read()
-                    except Exception as e:
-                        make_log("ConvertProcess", f"Error reading output.json for option {option}: {e}", level="error")
-                        return
-
-                    try:
-                        ffprobe_meta = json.loads(output_json_content)
-                    except Exception as e:
-                        make_log("ConvertProcess", f"Error parsing output.json for option {option}: {e}", level="error")
-                        return
-                    
+            # Process output.json for ffprobe_meta
+            output_json_path = os.path.join(output_dir_container, "output.json")
+            if os.path.exists(output_json_path) and unprocessed_encrypted_content.meta.get('ffprobe_meta') is None:
+                try:
+                    with open(output_json_path, "r") as f:
+                        ffprobe_meta = json.load(f)
                     unprocessed_encrypted_content.meta = {
                         **unprocessed_encrypted_content.meta,
                         'ffprobe_meta': ffprobe_meta
                     }
-            else:
-                make_log("ConvertProcess", f"output.json not found for option {option}", level="error")
+                except Exception as e:
+                    make_log("ConvertProcess", f"Error handling output.json for option {option}: {e}", level="error")
 
-            # Remove the output directory after processing
+            # Cleanup output directory
             try:
-                shutil.rmtree(output_dir.replace("/Storage/storedContent", "/app/data"))
+                shutil.rmtree(output_dir_container)
             except Exception as e:
-                make_log("ConvertProcess", f"Error removing output directory {output_dir}: {e}", level="error")
-                # Continue even if deletion fails
+                make_log("ConvertProcess", f"Error removing output dir {output_dir}: {e}", level="warning")
 
+        # Finalize original record
         make_log("ConvertProcess", f"Content {unprocessed_encrypted_content.id} processed. Converted content: {converted_content}", level="info")
-
         unprocessed_encrypted_content.btfs_cid = ContentId(
-            version=2, content_hash=b58decode(converted_content['high'])
+            version=2, content_hash=b58decode(converted_content['high' if content_kind=='video' else 'low'])
         ).serialize_v2()
         unprocessed_encrypted_content.ipfs_cid = ContentId(
             version=2, content_hash=b58decode(converted_content['low'])
@@ -199,32 +234,47 @@ async def convert_loop(memory):
             **unprocessed_encrypted_content.meta,
             'converted_content': converted_content
         }
+        await session.commit()
 
-        session.commit()
+        # Notify user if needed
         if not unprocessed_encrypted_content.meta.get('upload_notify_msg_id'):
-            wallet_owner_connection = session.query(WalletConnection).filter(
+            wallet_owner_connection = (await session.execute(select(WalletConnection).where(
                 WalletConnection.wallet_address == unprocessed_encrypted_content.owner_address
-            ).order_by(WalletConnection.id.desc()).first()
+            ).order_by(WalletConnection.id.desc()))).scalars().first()
             if wallet_owner_connection:
                 wallet_owner_user = wallet_owner_connection.user
-                wallet_owner_bot = Wrapped_CBotChat(memory._client_telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
-                unprocessed_encrypted_content.meta = {
-                    **unprocessed_encrypted_content.meta,
-                    'upload_notify_msg_id': await wallet_owner_bot.send_content(session, unprocessed_encrypted_content)
-                }
+                bot = Wrapped_CBotChat(
+                    memory._client_telegram_bot,
+                    chat_id=wallet_owner_user.telegram_id,
+                    user=wallet_owner_user,
+                    db_session=session
+                )
+                unprocessed_encrypted_content.meta['upload_notify_msg_id'] = await bot.send_content(session, unprocessed_encrypted_content)
+                await session.commit()
 
-                session.commit()
 
 async def main_fn(memory):
     make_log("ConvertProcess", "Service started", level="info")
     seqno = 0
     while True:
         try:
-            make_log("ConvertProcess", "Service running", level="debug")
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
+            make_log("ConvertProcess", "Service running", level="debug", rid=rid)
             await convert_loop(memory)
             await asyncio.sleep(5)
             await send_status("convert_service", f"working (seqno={seqno})")
             seqno += 1
         except BaseException as e:
-            make_log("ConvertProcess", f"Error: {e}", level="error")
+            make_log("ConvertProcess", f"Error: {e}", level="error", rid=locals().get('rid'))
             await asyncio.sleep(3)
+        finally:
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(None)
+            except BaseException:
+                pass

app/core/background/convert_v3_service.py

@@ -0,0 +1,314 @@
+import asyncio
+import os
+import json
+import shutil
+from datetime import datetime
+from typing import List, Tuple
+
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core._config import UPLOADS_DIR, BACKEND_LOGS_DIR_HOST
+from app.core.models.content_v3 import (
+    EncryptedContent,
+    ContentKey,
+    ContentDerivative,
+    UploadSession,
+)
+from app.core.models.node_storage import StoredContent
+from app.core.ipfs_client import cat_stream
+from app.core.crypto.encf_stream import decrypt_encf_auto
+from app.core.crypto.keywrap import unwrap_dek, wrap_dek, KeyWrapError
+from app.core.network.key_client import request_key_from_peer
+from app.core.models.my_network import KnownNode
+
+
+CONCURRENCY = int(os.getenv("CONVERT_V3_MAX_CONCURRENCY", "3"))
+
+
+def _ensure_dir(path: str):
+    try:
+        os.makedirs(path, exist_ok=True)
+    except Exception:
+        pass
+
+
+async def _sha256_b58(file_path: str) -> str:
+    import hashlib
+    import base58
+    h = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(2 * 1024 * 1024), b''):
+            h.update(chunk)
+    return base58.b58encode(h.digest()).decode()
+
+
+async def _save_derivative(file_path: str, filename: str) -> Tuple[str, int]:
+    """Move file into UPLOADS_DIR under sha256 b58 name; return (hash_b58, size)."""
+    file_hash = await _sha256_b58(file_path)
+    dst = os.path.join(UPLOADS_DIR, file_hash)
+    try:
+        os.remove(dst)
+    except FileNotFoundError:
+        pass
+    shutil.move(file_path, dst)
+    size = os.path.getsize(dst)
+    return file_hash, size
+
+
+async def _run_media_converter(input_host_path: str, input_ext: str, quality: str, trim_value: str | None, is_audio: bool) -> Tuple[str, dict]:
+    rid = __import__('uuid').uuid4().hex[:8]
+    output_dir_container = f"/tmp/conv_{rid}"
+    output_dir_host = f"/tmp/conv_{rid}"
+    _ensure_dir(output_dir_host)
+    logs_dir_host = BACKEND_LOGS_DIR_HOST
+    _ensure_dir(logs_dir_host)
+
+    cmd = [
+        "docker", "run", "--rm",
+        "-v", f"{input_host_path}:/app/input:ro",
+        "-v", f"{output_dir_host}:/app/output",
+        "-v", f"{logs_dir_host}:/app/logs",
+        "media_converter",
+        "--ext", input_ext,
+        "--quality", quality,
+    ]
+    if trim_value:
+        cmd.extend(["--trim", trim_value])
+    if is_audio:
+        cmd.append("--audio-only")
+
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        raise RuntimeError(f"media_converter failed: {stderr.decode()}")
+
+    # Find produced media file and optional output.json
+    try:
+        files = os.listdir(output_dir_host)
+    except Exception as e:
+        raise RuntimeError(f"Read output dir error: {e}")
+    media_files = [f for f in files if f != "output.json"]
+    if len(media_files) != 1:
+        raise RuntimeError(f"Expected one media file, found {len(media_files)}: {media_files}")
+    output_media = os.path.join(output_dir_host, media_files[0])
+    ffprobe_meta = {}
+    out_json = os.path.join(output_dir_host, "output.json")
+    if os.path.exists(out_json):
+        try:
+            with open(out_json, 'r') as f:
+                ffprobe_meta = json.load(f)
+        except Exception:
+            ffprobe_meta = {}
+    return output_media, ffprobe_meta
+
+
+async def _convert_content(ec: EncryptedContent, input_host_path: str):
+    content_kind = 'audio' if ec.content_type.startswith('audio/') else ('video' if ec.content_type.startswith('video/') else 'other')
+    if content_kind == 'other':
+        return
+
+    input_ext = (ec.content_type.split('/')[-1] or 'bin')
+    is_audio = content_kind == 'audio'
+    # Required outputs
+    required = ['high', 'low', 'low_preview']
+
+    # Preview interval
+    conf = ec.preview_conf or {}
+    intervals = conf.get('intervals') or [[0, int(conf.get('duration_ms', 30000))]]
+    main_interval = intervals[0]
+    trim_value = None
+    start_s = max(0, int(main_interval[0]) // 1000)
+    dur_s = max(1, int((main_interval[1] - main_interval[0]) // 1000) or 30)
+    trim_value = f"{start_s},{dur_s}"
+
+    qualities = {
+        'high': 'high',
+        'low': 'low',
+        'low_preview': 'low',
+    }
+
+    for opt in required:
+        try:
+            # Mark derivative processing
+            async with db_session() as session:
+                cd = ContentDerivative(
+                    content_id=ec.id,
+                    kind=f"decrypted_{opt if opt != 'low_preview' else 'preview'}",
+                    interval_start_ms=main_interval[0] if opt == 'low_preview' else None,
+                    interval_end_ms=main_interval[1] if opt == 'low_preview' else None,
+                    local_path="",
+                    status='processing',
+                )
+                session.add(cd)
+                await session.commit()
+
+            out_path, ffprobe = await _run_media_converter(
+                input_host_path=input_host_path,
+                input_ext=input_ext,
+                quality=qualities[opt],
+                trim_value=trim_value if opt == 'low_preview' else None,
+                is_audio=is_audio,
+            )
+
+            # Save into store and StoredContent
+            file_hash, size_bytes = await _save_derivative(out_path, os.path.basename(out_path))
+
+            async with db_session() as session:
+                sc = StoredContent(
+                    type="local/content_bin",
+                    hash=file_hash,
+                    user_id=None,
+                    filename=os.path.basename(out_path),
+                    meta={'encrypted_cid': ec.encrypted_cid, 'kind': opt, 'ffprobe_meta': ffprobe},
+                    created=datetime.utcnow(),
+                )
+                session.add(sc)
+                await session.flush()
+
+                # Update derivative record
+                cd = (await session.execute(select(ContentDerivative).where(
+                    ContentDerivative.content_id == ec.id,
+                    ContentDerivative.kind == (f"decrypted_{opt if opt != 'low_preview' else 'preview'}"),
+                    ContentDerivative.status == 'processing'
+                ))).scalars().first()
+                if cd:
+                    cd.local_path = os.path.join(UPLOADS_DIR, file_hash)
+                    cd.size_bytes = size_bytes
+                    cd.content_type = ('audio/mpeg' if is_audio else 'video/mp4') if opt != 'high' else ec.content_type
+                    cd.status = 'ready'
+                await session.commit()
+
+            make_log('convert_v3', f"Converted {ec.encrypted_cid} opt={opt} -> {file_hash}")
+        except Exception as e:
+            make_log('convert_v3', f"Convert error {ec.encrypted_cid} opt={opt}: {e}", level='error')
+            async with db_session() as session:
+                cd = ContentDerivative(
+                    content_id=ec.id,
+                    kind=f"decrypted_{opt if opt != 'low_preview' else 'preview'}",
+                    status='failed',
+                    error=str(e),
+                    local_path="",
+                )
+                session.add(cd)
+                await session.commit()
+
+
+async def _pick_pending(limit: int) -> List[Tuple[EncryptedContent, str]]:
+    async with db_session() as session:
+        # Find A/V contents with preview_enabled and no ready low/low_preview derivatives yet
+        ecs = (await session.execute(select(EncryptedContent).where(
+            EncryptedContent.preview_enabled == True
+        ).order_by(EncryptedContent.created_at.desc()))).scalars().all()
+
+        picked: List[Tuple[EncryptedContent, str]] = []
+        for ec in ecs:
+            # Check if derivatives already ready
+            rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id))).scalars().all()
+            kinds_ready = {r.kind for r in rows if r.status == 'ready'}
+            required = {'decrypted_low', 'decrypted_high'} if ec.content_type.startswith('audio/') else {'decrypted_low', 'decrypted_high', 'decrypted_preview'}
+            if required.issubset(kinds_ready):
+                continue
+            # Always decrypt from IPFS using local or remote key
+            storage_path: str | None = None
+            ck = (await session.execute(select(ContentKey).where(ContentKey.content_id == ec.id))).scalars().first()
+            if ck:
+                storage_path = await stage_plain_from_ipfs(ec, ck.key_ciphertext_b64)
+            if not storage_path:
+                peers = (await session.execute(select(KnownNode))).scalars().all()
+                for peer in peers:
+                    base_url = f"http://{peer.ip}:{peer.port}"
+                    dek = await request_key_from_peer(base_url, ec.encrypted_cid)
+                    if not dek:
+                        continue
+                    try:
+                        dek_b64 = wrap_dek(dek)
+                    except KeyWrapError as exc:
+                        make_log('convert_v3', f"wrap failed for peer DEK: {exc}", level='error')
+                        continue
+                    session_ck = ContentKey(
+                        content_id=ec.id,
+                        key_ciphertext_b64=dek_b64,
+                        key_fingerprint=peer.public_key,
+                        issuer_node_id=peer.public_key,
+                        allow_auto_grant=True,
+                    )
+                    session.add(session_ck)
+                    await session.commit()
+                    storage_path = await stage_plain_from_ipfs(ec, dek_b64)
+                    if storage_path:
+                        break
+            if not storage_path or not os.path.exists(storage_path):
+                continue
+            picked.append((ec, storage_path))
+            if len(picked) >= limit:
+                break
+        return picked
+
+
+async def worker_loop():
+    sem = asyncio.Semaphore(CONCURRENCY)
+
+    async def _run_one(ec: EncryptedContent, input_path: str):
+        async with sem:
+            try:
+                await _convert_content(ec, input_path)
+                # After successful conversion, attempt to remove staging file to avoid duplicates
+                try:
+                    if input_path and input_path.startswith("/data/") and os.path.exists(input_path):
+                        os.remove(input_path)
+                except Exception:
+                    pass
+            except Exception as e:
+                make_log('convert_v3', f"job error {ec.encrypted_cid}: {e}", level='error')
+
+    while True:
+        try:
+            batch = await _pick_pending(limit=CONCURRENCY * 2)
+            if not batch:
+                await asyncio.sleep(3)
+                continue
+            tasks = [asyncio.create_task(_run_one(ec, path)) for (ec, path) in batch]
+            await asyncio.gather(*tasks)
+        except Exception as e:
+            make_log('convert_v3', f"loop error: {e}", level='error')
+            await asyncio.sleep(2)
+
+
+async def main_fn(memory):
+    make_log('convert_v3', f"Service started with concurrency={CONCURRENCY}", level='info')
+    await worker_loop()
+
+
+async def stage_plain_from_ipfs(ec: EncryptedContent, dek_wrapped: str) -> str | None:
+    """Download encrypted ENCF stream from IPFS and decrypt on the fly into a temp file."""
+    import tempfile
+    try:
+        dek = unwrap_dek(dek_wrapped)
+    except KeyWrapError as exc:
+        make_log('convert_v3', f"unwrap failed for {ec.encrypted_cid}: {exc}", level='error')
+        return None
+    tmp = tempfile.NamedTemporaryFile(prefix=f"dec_{ec.encrypted_cid[:8]}_", delete=False)
+    tmp_path = tmp.name
+    tmp.close()
+    try:
+        async def _aiter():
+            async for ch in cat_stream(ec.encrypted_cid):
+                yield ch
+        await decrypt_encf_auto(_aiter(), dek, tmp_path)
+        return tmp_path
+    except Exception as e:
+        make_log('convert_v3', f"decrypt from ipfs failed: {e}", level='error')
+        try:
+            os.remove(tmp_path)
+        except Exception:
+            pass
+        return None
+
+
+    

app/core/background/derivative_cache_janitor.py

@@ -0,0 +1,93 @@
+import asyncio
+import os
+from datetime import datetime, timedelta
+
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core.models.content_v3 import ContentDerivative
+from app.core.models._config import ServiceConfig
+
+
+ENV_MAX_GB = float(os.getenv('DERIVATIVE_CACHE_MAX_GB', '50'))
+ENV_TTL_DAYS = int(os.getenv('DERIVATIVE_CACHE_TTL_DAYS', '0'))
+INTERVAL_SEC = int(os.getenv('DERIVATIVE_JANITOR_INTERVAL_SEC', '600'))
+
+
+async def _current_total_size() -> int:
+    async with db_session() as session:
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        return sum(int(r.size_bytes or 0) for r in rows)
+
+
+async def _evict_over_ttl(now: datetime) -> int:
+    removed = 0
+    # Pull TTL from ServiceConfig each time
+    async with db_session() as session:
+        ttl_days = await ServiceConfig(session).get('DERIVATIVE_CACHE_TTL_DAYS', ENV_TTL_DAYS)
+    if int(ttl_days) <= 0:
+        return 0
+    async with db_session() as session:
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        for r in rows:
+            la = r.last_access_at or r.created_at
+            if la and (now - la) > timedelta(days=TTL_DAYS):
+                try:
+                    if r.local_path and os.path.exists(r.local_path):
+                        os.remove(r.local_path)
+                except Exception:
+                    pass
+                r.status = 'pending'
+                r.local_path = None
+                r.size_bytes = None
+                r.last_access_at = None
+                removed += 1
+        await session.commit()
+    return removed
+
+
+async def _evict_to_fit():
+    async with db_session() as session:
+        max_gb = await ServiceConfig(session).get('DERIVATIVE_CACHE_MAX_GB', ENV_MAX_GB)
+    limit_bytes = int(float(max_gb) * (1024 ** 3))
+    total = await _current_total_size()
+    if total <= limit_bytes:
+        return 0
+    to_remove = total - limit_bytes
+    removed = 0
+    async with db_session() as session:
+        # Oldest first by last_access_at
+        rows = (await session.execute(select(ContentDerivative).where(ContentDerivative.status == 'ready'))).scalars().all()
+        rows.sort(key=lambda r: (r.last_access_at or r.created_at or datetime.utcfromtimestamp(0)))
+        for r in rows:
+            if to_remove <= 0:
+                break
+            size = int(r.size_bytes or 0)
+            try:
+                if r.local_path and os.path.exists(r.local_path):
+                    os.remove(r.local_path)
+            except Exception:
+                pass
+            r.status = 'pending'
+            r.local_path = None
+            r.last_access_at = None
+            r.size_bytes = None
+            await session.commit()
+            to_remove -= size
+            removed += 1
+    return removed
+
+
+async def main_fn(memory):
+    make_log('derivative_janitor', f"Started (MAX_GB={MAX_GB}, TTL_DAYS={TTL_DAYS})", level='info')
+    while True:
+        try:
+            now = datetime.utcnow()
+            r1 = await _evict_over_ttl(now)
+            r2 = await _evict_to_fit()
+            if r1 or r2:
+                make_log('derivative_janitor', f"Evicted: ttl={r1}, fit={r2}")
+        except Exception as e:
+            make_log('derivative_janitor', f"Error: {e}", level='error')
+        await asyncio.sleep(INTERVAL_SEC)

app/core/background/index_scout_v3.py

@@ -0,0 +1,184 @@
+import asyncio
+from typing import List
+
+import httpx
+import random
+import shutil
+from sqlalchemy import select
+
+from app.core.logger import make_log
+from app.core.storage import db_session
+from app.core.models.my_network import KnownNode
+from app.core.models.content_v3 import EncryptedContent, ContentDerivative
+from app.core.ipfs_client import pin_add, find_providers, swarm_connect
+
+
+INTERVAL_SEC = 60
+ENV_PIN_CONCURRENCY = int(os.getenv('SYNC_MAX_CONCURRENT_PINS', '4'))
+ENV_DISK_WATERMARK_PCT = int(os.getenv('SYNC_DISK_LOW_WATERMARK_PCT', '90'))
+
+
+async def fetch_index(base_url: str, etag: str | None, since: str | None) -> tuple[List[dict], str | None]:
+    try:
+        headers = {}
+        params = {}
+        if since:
+            params['since'] = since
+        url = f"{base_url.rstrip('/')}/api/v1/content.delta" if since else f"{base_url.rstrip('/')}/api/v1/content.index"
+        if etag:
+            headers['If-None-Match'] = etag
+        async with httpx.AsyncClient(timeout=20) as client:
+            r = await client.get(url, headers=headers, params=params)
+            if r.status_code != 200:
+                if r.status_code == 304:
+                    return [], etag
+                return [], etag
+            j = r.json()
+            new_etag = r.headers.get('ETag') or etag
+            return j.get('items') or [], (j.get('next_since') or new_etag or etag)
+    except Exception:
+        return [], etag
+
+
+async def upsert_content(item: dict):
+    cid = item.get('encrypted_cid')
+    if not cid:
+        return
+    async with db_session() as session:
+        row = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+        if not row:
+            row = EncryptedContent(
+                encrypted_cid=cid,
+                title=item.get('title') or cid,
+                description=item.get('description') or '',
+                content_type=item.get('content_type') or 'application/octet-stream',
+                enc_size_bytes=item.get('size_bytes'),
+                preview_enabled=bool(item.get('preview_enabled')),
+                preview_conf=item.get('preview_conf') or {},
+                salt_b64=item.get('salt_b64'),
+            )
+            session.add(row)
+        else:
+            row.title = item.get('title') or row.title
+            row.description = item.get('description') or row.description
+            row.content_type = item.get('content_type') or row.content_type
+            row.enc_size_bytes = item.get('size_bytes') or row.enc_size_bytes
+            row.preview_enabled = bool(item.get('preview_enabled')) if item.get('preview_enabled') is not None else row.preview_enabled
+            if item.get('preview_conf'):
+                row.preview_conf = item['preview_conf']
+            if item.get('salt_b64'):
+                row.salt_b64 = item['salt_b64']
+        await session.commit()
+
+    # Fetch thumbnail via HTTP if provided and not present locally
+    cover_url = item.get('cover_url')
+    if cover_url:
+        try:
+            async with db_session() as session:
+                ec = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == cid))).scalars().first()
+                have_thumb = (await session.execute(select(ContentDerivative).where(ContentDerivative.content_id == ec.id, ContentDerivative.kind == 'decrypted_thumbnail', ContentDerivative.status == 'ready'))).scalars().first()
+                if not have_thumb:
+                    import httpx, tempfile, os
+                    async with httpx.AsyncClient(timeout=30) as client:
+                        r = await client.get(cover_url)
+                        r.raise_for_status()
+                        tmp = tempfile.NamedTemporaryFile(delete=False)
+                        tmp.write(r.content)
+                        tmp.close()
+                    # Save into store
+                    from app.core.background.convert_v3_service import _save_derivative
+                    h, size = await _save_derivative(tmp.name, os.path.basename(cover_url) or 'thumb.jpg')
+                    cd = ContentDerivative(
+                        content_id=ec.id,
+                        kind='decrypted_thumbnail',
+                        local_path=os.path.join(os.getenv('UPLOADS_DIR', '/app/data'), h),
+                        content_type=r.headers.get('Content-Type') or 'image/jpeg',
+                        size_bytes=size,
+                        status='ready',
+                    )
+                    session.add(cd)
+                    await session.commit()
+        except Exception as e:
+            make_log('index_scout_v3', f"thumbnail fetch failed for {cid}: {e}", level='warning')
+
+
+async def main_fn(memory):
+    make_log('index_scout_v3', 'Service started', level='info')
+    sem = None
+    while True:
+        try:
+            # Read runtime config from ServiceConfig (fallback to env)
+            from app.core.models._config import ServiceConfig
+            async with db_session() as session:
+                max_pins = int(await ServiceConfig(session).get('SYNC_MAX_CONCURRENT_PINS', ENV_PIN_CONCURRENCY))
+                disk_pct = int(await ServiceConfig(session).get('SYNC_DISK_LOW_WATERMARK_PCT', ENV_DISK_WATERMARK_PCT))
+            if sem is None or sem._value != max_pins:
+                sem = asyncio.Semaphore(max_pins)
+            async with db_session() as session:
+                nodes = (await session.execute(select(KnownNode))).scalars().all()
+            for n in nodes:
+                base = f"http://{n.ip}:{n.port}"
+                # jitter 0..30s per node to reduce stampede
+                await asyncio.sleep(random.uniform(0, 30))
+                etag = (n.meta or {}).get('index_etag')
+                since = (n.meta or {}).get('index_since')
+                items, marker = await fetch_index(base, etag, since)
+                if not items and marker == etag:
+                    continue
+                # update node markers
+                try:
+                    async with db_session() as session:
+                        row = (await session.execute(select(KnownNode).where(KnownNode.id == n.id))).scalars().first()
+                        if row:
+                            meta = row.meta or {}
+                            meta['index_etag'] = marker
+                            meta['index_since'] = marker if (marker and 'T' in str(marker)) else meta.get('index_since')
+                            row.meta = meta
+                            await session.commit()
+                except Exception:
+                    pass
+
+                if not items:
+                    continue
+                make_log('index_scout_v3', f"Fetched {len(items)} from {base}")
+
+                # Check disk watermark
+                try:
+                    from app.core._config import UPLOADS_DIR
+                    du = shutil.disk_usage(UPLOADS_DIR)
+                    used_pct = int(100 * (1 - du.free / du.total))
+                    if used_pct >= disk_pct:
+                        make_log('index_scout_v3', f"Disk watermark reached ({used_pct}%), skipping pins")
+                        continue
+                except Exception:
+                    pass
+
+                async def _pin_one(cid: str):
+                    async with sem:
+                        try:
+                            # Try to pre-connect to discovered providers
+                            try:
+                                provs = await find_providers(cid, max_results=5)
+                                for p in provs:
+                                    for addr in (p.get('addrs') or [])[:2]:
+                                        try:
+                                            await swarm_connect(addr)
+                                        except Exception:
+                                            pass
+                            except Exception:
+                                pass
+                            await pin_add(cid, recursive=True)
+                        except Exception as e:
+                            make_log('index_scout_v3', f"pin {cid} failed: {e}", level='warning')
+
+                tasks = []
+                for it in items:
+                    await upsert_content(it)
+                    cid = it.get('encrypted_cid')
+                    if cid:
+                        tasks.append(asyncio.create_task(_pin_one(cid)))
+                if tasks:
+                    await asyncio.gather(*tasks)
+        except Exception as e:
+            make_log('index_scout_v3', f"loop error: {e}", level='error')
+        await asyncio.sleep(INTERVAL_SEC)

app/core/background/indexer_service.py

@@ -12,11 +12,13 @@ from app.core._blockchain.ton.toncenter import toncenter
 from app.core._utils.send_status import send_status
 from app.core.logger import make_log
 from app.core.models import UserContent, KnownTelegramMessage, ServiceConfig
+from app.core.models.user import User
 from app.core.models.node_storage import StoredContent
 from app.core._utils.resolve_content import resolve_content
 from app.core.models.wallet_connection import WalletConnection
 from app.core._keyboards import get_inline_keyboard
 from app.core.models._telegram import Wrapped_CBotChat
+from sqlalchemy import select, and_, desc
 from app.core.storage import db_session
 import os
 import traceback
@@ -33,7 +35,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
             platform_found = True
 
     make_log("Indexer", "Service running", level="debug")
-    with db_session() as session:
+    async with db_session() as session:
         try:
             result = await toncenter.run_get_method('EQD8TJ8xEWB1SpnRE4d89YO3jl0W0EiBnNS4IBaHaUmdfizE', 'get_pool_data')
             assert result['exit_code'] == 0, f"Error in get-method: {result}"
@@ -41,42 +43,46 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
             assert result['stack'][1][0] == 'num', f"get second element is not num"
             usdt_per_ton = (int(result['stack'][0][1], 16) * 1e3) / int(result['stack'][1][1], 16)
             ton_per_star = 0.014 / usdt_per_ton
-            ServiceConfig(session).set('live_tonPerStar', [ton_per_star, datetime.utcnow().timestamp()])
+            await ServiceConfig(session).set('live_tonPerStar', [ton_per_star, datetime.utcnow().timestamp()])
             make_log("TON_Daemon", f"TON per STAR price: {ton_per_star}", level="DEBUG")
         except BaseException as e:
             make_log("TON_Daemon", f"Error while saving TON per STAR price: {e}" + '\n' + traceback.format_exc(), level="ERROR")
 
-        new_licenses = session.query(UserContent).filter(
+        from sqlalchemy import cast
+        from sqlalchemy.dialects.postgresql import JSONB
+        new_licenses = (await session.execute(select(UserContent).where(
             and_(
-                ~UserContent.meta.contains({'notification_sent': True}),
+                ~(cast(UserContent.meta, JSONB).contains({'notification_sent': True})),
                 UserContent.type == 'nft/listen'
             )
-        ).all()
+        ))).scalars().all()
         for new_license in new_licenses:
-            licensed_content = session.query(StoredContent).filter(
+            licensed_content = (await session.execute(select(StoredContent).where(
                 StoredContent.id == new_license.content_id
-            ).first()
+            ))).scalars().first()
             if not licensed_content:
                 make_log("Indexer", f"Licensed content not found: {new_license.content_id}", level="error")
 
-            content_metadata = licensed_content.metadata_json(session)
+            content_metadata = await licensed_content.metadata_json_async(session)
             assert content_metadata, "No content metadata found"
 
             if not (licensed_content.owner_address == new_license.owner_address):
                 try:
-                    user = new_license.user
+                    user = await session.get(User, new_license.user_id)
                     if user.telegram_id and licensed_content:
                         await (Wrapped_CBotChat(memory._client_telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)).send_content(
                             session, licensed_content
                         )
 
-                    wallet_owner_connection = session.query(WalletConnection).filter_by(
-                        wallet_address=licensed_content.owner_address,
-                        invalidated=False
-                    ).order_by(desc(WalletConnection.id)).first()
-                    wallet_owner_user = wallet_owner_connection.user
+                    wallet_owner_connection = (await session.execute(
+                        select(WalletConnection).where(
+                            WalletConnection.wallet_address == licensed_content.owner_address,
+                            WalletConnection.invalidated == False
+                        ).order_by(desc(WalletConnection.id))
+                    )).scalars().first()
+                    wallet_owner_user = await session.get(User, wallet_owner_connection.user_id) if wallet_owner_connection else None
                     if wallet_owner_user.telegram_id:
-                        wallet_owner_bot = Wrapped_CBotChat(memory._client_telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
+                        wallet_owner_bot = Wrapped_CBotChat(memory._telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
                         await wallet_owner_bot.send_message(
                             user.translated('p_licenseWasBought').format(
                                 username=user.front_format(),
@@ -89,21 +95,19 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
                     make_log("IndexerSendNewLicense", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
 
             new_license.meta = {**new_license.meta, 'notification_sent': True}
-            session.commit()
+            await session.commit()
 
-        content_without_cid = session.query(StoredContent).filter(
-            StoredContent.content_id == None
-        )
+        content_without_cid = (await session.execute(select(StoredContent).where(StoredContent.content_id == None))).scalars().all()
         for target_content in content_without_cid:
             target_cid = target_content.cid.serialize_v2()
             make_log("Indexer", f"Content without CID: {target_content.hash}, setting CID: {target_cid}", level="debug")
             target_content.content_id = target_cid
 
-        session.commit()
+        await session.commit()
 
-        last_known_index_ = session.query(StoredContent).filter(
-            StoredContent.onchain_index != None
-        ).order_by(StoredContent.onchain_index.desc()).first()
+        last_known_index_ = (await session.execute(
+            select(StoredContent).where(StoredContent.onchain_index != None).order_by(StoredContent.onchain_index.desc())
+        )).scalars().first()
         last_known_index = last_known_index_.onchain_index if last_known_index_ else 0
         last_known_index = max(last_known_index, 0)
         make_log("Indexer", f"Last known index: {last_known_index}", level="debug")
@@ -196,14 +200,13 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
 
         user_wallet_connection = None
         if item_owner_address:
-            user_wallet_connection = session.query(WalletConnection).filter(
+            user_wallet_connection = (await session.execute(select(WalletConnection).where(
                 WalletConnection.wallet_address == item_owner_address.to_string(1, 1, 1)
-            ).first()
+            ))).scalars().first()
 
-        encrypted_stored_content = session.query(StoredContent).filter(
-            StoredContent.hash == item_content_hash_str,
-            # StoredContent.type.like("local%")
-        ).first()
+        encrypted_stored_content = (await session.execute(select(StoredContent).where(
+            StoredContent.hash == item_content_hash_str
+        ))).scalars().first()
         if encrypted_stored_content:
             is_duplicate = encrypted_stored_content.type.startswith("onchain") \
                            and encrypted_stored_content.onchain_index != item_index
@@ -215,7 +218,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
                     user = None
                     if user_wallet_connection:
                         encrypted_stored_content.user_id = user_wallet_connection.user_id
-                        user = user_wallet_connection.user
+                        user = await session.get(User, user_wallet_connection.user_id)
 
                     if user:
                         user_uploader_wrapper = Wrapped_CBotChat(memory._telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)
@@ -234,14 +237,15 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
                         )
 
                         try:
-                            for hint_message in session.query(KnownTelegramMessage).filter(
+                            result = await session.execute(select(KnownTelegramMessage).where(
                                 and_(
                                     KnownTelegramMessage.chat_id == user.telegram_id,
                                     KnownTelegramMessage.type == 'hint',
                                     cast(KnownTelegramMessage.meta['encrypted_content_hash'], String) == encrypted_stored_content.hash,
                                     KnownTelegramMessage.deleted == False
                                 )
-                            ).all():
+                            ))
+                            for hint_message in result.scalars().all():
                                 await user_uploader_wrapper.delete_message(hint_message.message_id)
                         except BaseException as e:
                             make_log("Indexer", f"Error while deleting hint messages: {e}" + '\n' + traceback.format_exc(), level="error")
@@ -260,7 +264,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
                     **item_metadata_packed
                 }
 
-                session.commit()
+                await session.commit()
                 return platform_found, seqno
             else:
                 item_metadata_packed['copied_from'] = encrypted_stored_content.id
@@ -282,7 +286,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
             updated=datetime.now()
         )
         session.add(onchain_stored_content)
-        session.commit()
+        await session.commit()
         make_log("Indexer", f"Item indexed: {item_content_hash_str}", level="info")
         last_known_index += 1
 
@@ -295,15 +299,27 @@ async def main_fn(memory, ):
     seqno = 0
     while True:
         try:
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
+            make_log("Indexer", f"Loop start", level="debug", rid=rid)
             platform_found, seqno = await indexer_loop(memory, platform_found, seqno)
         except BaseException as e:
-            make_log("Indexer", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
+            make_log("Indexer", f"Error: {e}" + '\n' + traceback.format_exc(), level="error", rid=locals().get('rid'))
 
         if platform_found:
             await send_status("indexer", f"working (seqno={seqno})")
 
         await asyncio.sleep(5)
         seqno += 1
+        try:
+            from app.core.log_context import ctx_rid
+            ctx_rid.set(None)
+        except BaseException:
+            pass
 
 
 

app/core/background/license_service.py

@@ -3,7 +3,7 @@ from base64 import b64decode
 from datetime import datetime, timedelta
 
 from base58 import b58encode
-from sqlalchemy import and_
+from sqlalchemy import and_, or_, select, desc
 from tonsdk.boc import Cell
 from tonsdk.utils import Address
 
@@ -27,7 +27,7 @@ import traceback
 
 async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
     make_log("LicenseIndex", "Service running", level="debug")
-    with db_session() as session:
+    async with db_session() as session:
         async def check_telegram_stars_transactions():
             # Проверка звездных telegram транзакций, обновление paid
             offset = {'desc': 'Статичное число заранее известного количества транзакций, которое даже не знает наш бот', 'value': 1}['value'] + \
@@ -45,19 +45,19 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
                         continue
 
                     try:
-                        existing_invoice = session.query(StarsInvoice).filter(
+                        existing_invoice = (await session.execute(select(StarsInvoice).where(
                             StarsInvoice.external_id == star_payment.source.invoice_payload
-                        ).first()
+                        ))).scalars().first()
                         if not existing_invoice:
                             continue
 
                         if star_payment.amount == existing_invoice.amount:
                             if not existing_invoice.paid:
                                 existing_invoice.paid = True
-                                session.commit()
+                                await session.commit()
 
-                                licensed_content = session.query(StoredContent).filter(StoredContent.hash == existing_invoice.content_hash).first()
-                                user = session.query(User).filter(User.id == existing_invoice.user_id).first()
+                                licensed_content = (await session.execute(select(StoredContent).where(StoredContent.hash == existing_invoice.content_hash))).scalars().first()
+                                user = (await session.execute(select(User).where(User.id == existing_invoice.user_id))).scalars().first()
 
                                 await (Wrapped_CBotChat(memory._client_telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)).send_content(
                                     session, licensed_content
@@ -73,33 +73,36 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
             make_log("StarsProcessing", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
 
         # Проверка кошельков пользователей на появление новых NFT, добавление их в базу как неопознанные
-        for user in session.query(User).filter(
-            User.last_use > datetime.now() - timedelta(minutes=10)
-        ).all():
-            if not user.wallet_address(session):
-                make_log("LicenseIndex", f"User {user.id} has no wallet address", level="info")
+        users = (await session.execute(select(User).where(
+            User.last_use > datetime.now() - timedelta(hours=4)
+        ).order_by(User.updated.asc()))).scalars().all()
+        for user in users:
+            user_wallet_address = await user.wallet_address_async(session)
+            if not user_wallet_address:
+                make_log("LicenseIndex", f"User {user.id} has no wallet address", level="debug")
                 continue
 
+            make_log("LicenseIndex", f"User {user.id} has wallet address {user_wallet_address}", level="debug")
             last_updated_licenses = user.meta.get('last_updated_licenses')
             must_skip = last_updated_licenses and (datetime.now() - datetime.fromisoformat(last_updated_licenses)) < timedelta(minutes=1)
-            make_log("LicenseIndex", f"User: {user.id}, last_updated_licenses: {last_updated_licenses}, must_skip: {must_skip}", level="info")
+            make_log("LicenseIndex", f"User: {user.id}, last_updated_licenses: {last_updated_licenses}, must_skip: {must_skip}", level="debug")
             if must_skip:
                 continue
 
             try:
                 await user.scan_owned_user_content(session)
                 user.meta = {**user.meta, 'last_updated_licenses': datetime.now().isoformat()}
-                session.commit()
+                await session.commit()
             except BaseException as e:
                 make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
-
+        
         # Проверка NFT на актуальность данных, в том числе уже проверенные
-        process_content = session.query(UserContent).filter(
+        process_content = (await session.execute(select(UserContent).where(
             and_(
                 UserContent.type.startswith('nft/'),
                 UserContent.updated < (datetime.now() - timedelta(minutes=60)),
             )
-        ).first()
+        ).order_by(UserContent.updated.asc()))).scalars().first()
         if process_content:
             make_log("LicenseIndex", f"Syncing content with blockchain: {process_content.id}", level="info")
             try:
@@ -108,7 +111,7 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
                 make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
             finally:
                 process_content.updated = datetime.now()
-                session.commit()
+                await session.commit()
 
     return platform_found, seqno
 
@@ -119,14 +122,26 @@ async def main_fn(memory, ):
     seqno = 0
     while True:
         try:
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
+            make_log("LicenseIndex", f"Loop start", level="debug", rid=rid)
             platform_found, seqno = await license_index_loop(memory, platform_found, seqno)
             if platform_found:
                 await send_status("licenses", f"working (seqno={seqno})")
         except BaseException as e:
-            make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
+            make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error", rid=locals().get('rid'))
 
         await asyncio.sleep(1)
         seqno += 1
+        try:
+            from app.core.log_context import ctx_rid
+            ctx_rid.set(None)
+        except BaseException:
+            pass
 
 # if __name__ == '__main__':
 #     loop = asyncio.get_event_loop()

app/core/background/ton_service.py

@@ -1,13 +1,22 @@
 import asyncio
+from base64 import b64decode
 import os
+import traceback
+import httpx
 
-from tonsdk.boc import begin_cell
+from sqlalchemy import and_, func
+from tonsdk.boc import begin_cell, Cell
+from tonsdk.contract.wallet import Wallets
+from tonsdk.utils import HighloadQueryId
+from datetime import datetime, timedelta
 
 from app.core._blockchain.ton.platform import platform
 from app.core._blockchain.ton.toncenter import toncenter
+from app.core.models.tasks import BlockchainTask
 from app.core._config import MY_FUND_ADDRESS
 from app.core._secrets import service_wallet
 from app.core._utils.send_status import send_status
+from app.core.storage import db_session
 from app.core.logger import make_log
 
 
@@ -74,27 +83,220 @@ async def main_fn(memory):
         await send_status("ton_daemon", "working: deploying platform")
         await asyncio.sleep(15)
         return await main_fn(memory)
+    
+    highload_wallet = Wallets.ALL['hv3'](
+        private_key=service_wallet.options['private_key'],
+        public_key=service_wallet.options['public_key'],
+        wc=0
+    )
+    make_log("TON", f"Highload wallet address: {highload_wallet.address.to_string(1, 1, 1)}", level="info")
+    highload_state = await toncenter.get_account(highload_wallet.address.to_string(1, 1, 1))
+    if int(highload_state.get('balance', '0')) / 1e9 < 0.05:
+        make_log("TON", "Highload wallet balance is less than 0.05, send topup transaction..", level="info")
+        await toncenter.send_boc(
+            service_wallet.create_transfer_message(
+                [{
+                    'address': highload_wallet.address.to_string(1, 1, 0),
+                    'amount': int(0.02 * 10 ** 9),
+                    'send_mode': 1,
+                    'payload': begin_cell().store_uint(0, 32).end_cell()
+                }], sw_seqno_value
+            )['message'].to_boc(False)
+        )
+        await send_status("ton_daemon", "working: topup highload wallet")
+        await asyncio.sleep(15)
+        return await main_fn(memory)
+    
+    if not highload_state.get('code'):
+        make_log("TON", "Highload wallet contract is not deployed, send deploy transaction..", level="info")
+        created_at_ts = int(datetime.utcnow().timestamp()) - 60
+        await toncenter.send_boc(
+            highload_wallet.create_transfer_message(
+                service_wallet.address.to_string(1, 1, 1),
+                1, HighloadQueryId.from_seqno(0), created_at_ts, send_mode=1, payload="hello world", need_deploy=True
+            )['message'].to_boc(False)
+        )
+        await send_status("ton_daemon", "working: deploying highload wallet")
+        await asyncio.sleep(15)
+        return await main_fn(memory)
 
     while True:
         try:
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
             sw_seqno_value = await get_sw_seqno()
-            make_log("TON", f"Service running ({sw_seqno_value})", level="debug")
+            make_log("TON", f"Service running ({sw_seqno_value})", level="debug", rid=rid)
 
-            # with db_session() as session:
-            #     for stored_content in session.query(StoredContent).filter(StoredContent.uploaded == False).all():
-            #         pass
+            async with db_session() as session:
+                # Проверка отправленных сообщений
+                await send_status("ton_daemon", f"working: processing in-txs (seqno={sw_seqno_value})")
+                async def process_incoming_transaction(transaction: dict):
+                    transaction_hash = transaction['transaction_id']['hash']
+                    transaction_lt = str(transaction['transaction_id']['lt'])
+                    # transaction_success = bool(transaction['success'])
 
-            await asyncio.sleep(5)
+                    async def process_incoming_message(blockchain_message: dict):
+                        in_msg_cell = Cell.one_from_boc(b64decode(blockchain_message['msg_data']['body']))
+                        in_msg_slice = in_msg_cell.refs[0].begin_parse()
+                        in_msg_slice.read_uint(32)
+                        in_msg_slice.read_uint(8)
+                        in_msg_query_id = in_msg_slice.read_uint(23)
+                        in_msg_created_at = in_msg_slice.read_uint(64)
+                        in_msg_epoch = int(in_msg_created_at // (60 * 60))
+                        in_msg_seqno = HighloadQueryId.from_query_id(in_msg_query_id).to_seqno()
+                        from sqlalchemy import select
+                        in_msg_blockchain_task = (
+                            await session.execute(
+                                select(BlockchainTask).where(
+                                    and_(
+                                        BlockchainTask.seqno == in_msg_seqno,
+                                        BlockchainTask.epoch == in_msg_epoch,
+                                    )
+                                )
+                            )
+                        ).scalars().first()
+                        if not in_msg_blockchain_task:
+                            return
+
+                        if not (in_msg_blockchain_task.status in ['done']) or in_msg_blockchain_task.transaction_hash != transaction_hash:
+                            in_msg_blockchain_task.status = 'done'
+                            in_msg_blockchain_task.transaction_hash = transaction_hash
+                            in_msg_blockchain_task.transaction_lt = transaction_lt
+                            await session.commit()
+
+                    for blockchain_message in [transaction['in_msg']]:
+                        try:
+                            await process_incoming_message(blockchain_message)
+                        except BaseException as e:
+                            pass # make_log("TON_Daemon", f"Error while processing incoming message: {e}" + '\n' + traceback.format_exc(), level='debug', rid=rid)
+
+                try:
+                    sw_transactions = await toncenter.get_transactions(highload_wallet.address.to_string(1, 1, 1), limit=100)
+                    for sw_transaction in sw_transactions:
+                        try:
+                            await process_incoming_transaction(sw_transaction)
+                        except BaseException as e:
+                            make_log("TON_Daemon", f"Error while processing incoming transaction: {e}", level="debug", rid=rid)
+                except BaseException as e:
+                    make_log("TON_Daemon", f"Error while getting service wallet transactions: {e}", level="ERROR", rid=rid)
+
+                await send_status("ton_daemon", f"working: processing out-txs (seqno={sw_seqno_value})")
+                # Отправка подписанных сообщений
+                from sqlalchemy import select
+                _processing = (await session.execute(select(BlockchainTask).where(
+                    BlockchainTask.status == 'processing'
+                ).order_by(BlockchainTask.updated.asc()))).scalars().all()
+                for blockchain_task in _processing:
+                    make_log("TON_Daemon", f"Processing task (processing) {blockchain_task.id}", rid=rid)
+                    query_boc = bytes.fromhex(blockchain_task.meta['signed_message'])
+                    errors_list = []
+
+                    try:
+                        await toncenter.send_boc(query_boc)
+                    except BaseException as e:
+                        errors_list.append(f"{e}")
+
+                    try:
+                        make_log("TON_Daemon", str(
+                            httpx.post(
+                                'https://tonapi.io/v2/blockchain/message',
+                                json={
+                                    'boc': query_boc.hex()
+                                }
+                            ).text
+                        ))
+                    except BaseException as e:
+                        make_log("TON_Daemon", f"Error while pushing task to tonkeeper ({blockchain_task.id}): {e}", level="ERROR")
+                        errors_list.append(f"{e}")
+
+                    blockchain_task.updated = datetime.utcnow()
+
+                    if blockchain_task.meta['sign_created'] + 10 * 60 < datetime.utcnow().timestamp():
+                            # or sum([int("terminating vm with exit code 36" in e) for e in errors_list]) > 0:
+                        make_log("TON_Daemon", f"Task {blockchain_task.id} done", level="DEBUG")
+                        blockchain_task.status = 'done'
+                        await session.commit()
+                        continue
+
+                    await asyncio.sleep(0.5)
+
+                await send_status("ton_daemon", f"working: creating new messages (seqno={sw_seqno_value})")
+                # Создание новых подписей
+                _waiting = (await session.execute(select(BlockchainTask).where(BlockchainTask.status == 'wait'))).scalars().all()
+                for blockchain_task in _waiting:
+                    try:
+                        # Check processing tasks in current epoch < 3_000_000
+                        from sqlalchemy import func
+                        _cnt = (await session.execute(select(func.count()).select_from(BlockchainTask).where(
+                            BlockchainTask.epoch == blockchain_task.epoch
+                        ))).scalar() or 0
+                        if _cnt > 3_000_000:
+                            make_log("TON", f"Too many processing tasks in epoch {blockchain_task.epoch}", level="error")
+                            await send_status("ton_daemon", f"working: too many tasks in epoch {blockchain_task.epoch}")
+                            await asyncio.sleep(5)
+                            continue
+
+                        sign_created = int(datetime.utcnow().timestamp()) - 60
+                        try:
+                            current_epoch = int(datetime.utcnow().timestamp() // (60 * 60))
+                            from sqlalchemy import func
+                            max_epoch_seqno = (
+                                (await session.execute(select(func.max(BlockchainTask.seqno)).where(
+                                    BlockchainTask.epoch == current_epoch
+                                ))).scalar() or 0
+                            )
+                            current_epoch_shift = 3_000_000 if current_epoch % 2 == 0 else 0
+                            current_seqno = max_epoch_seqno + 1 + (current_epoch_shift if max_epoch_seqno == 0 else 0)
+                        except BaseException as e:
+                            make_log("CRITICAL", f"Error calculating epoch,seqno: {e}", level="error")
+                            current_epoch = 0
+                            current_seqno = 0
+
+                        blockchain_task.seqno = current_seqno
+                        blockchain_task.epoch = current_epoch
+                        blockchain_task.status = 'processing'
+                        try:
+                            query = highload_wallet.create_transfer_message(
+                                blockchain_task.destination, int(blockchain_task.amount), HighloadQueryId.from_seqno(current_seqno),
+                                sign_created, send_mode=1,
+                                payload=Cell.one_from_boc(b64decode(blockchain_task.payload))
+                            )
+                            query_boc = query['message'].to_boc(False)
+                        except BaseException as e:
+                            make_log("TON", f"Error creating transfer message: {e}", level="error", rid=rid)
+                            query_boc = begin_cell().end_cell().to_boc(False)
+                        
+                        blockchain_task.meta = {
+                            **blockchain_task.meta,
+                            'sign_created': sign_created,
+                            'signed_message': query_boc.hex(),
+                        }
+                        await session.commit()
+                        make_log("TON", f"Created signed message for task {blockchain_task.id}" + '\n' + traceback.format_exc(), level="info", rid=rid)
+                    except BaseException as e:
+                        make_log("TON", f"Error processing task {blockchain_task.id}: {e}" + '\n' + traceback.format_exc(), level="error", rid=rid)
+                        continue
+
+                    await asyncio.sleep(1)
+
+            await asyncio.sleep(1)
             await send_status("ton_daemon", f"working (seqno={sw_seqno_value})")
         except BaseException as e:
-            make_log("TON", f"Error: {e}", level="error")
+            make_log("TON", f"Error: {e}", level="error", rid=locals().get('rid'))
             await asyncio.sleep(3)
+        finally:
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(None)
+            except BaseException:
+                pass
 
 # if __name__ == '__main__':
 #     loop = asyncio.get_event_loop()
 #     loop.run_until_complete(main())
 #     loop.close()
 
-
-
-

app/core/background/uploader_service.py

@@ -13,14 +13,26 @@ async def main_fn(memory):
     seqno = 0
     while True:
         try:
-            make_log("Uploader", "Service running", level="debug")
+            rid = __import__('uuid').uuid4().hex[:8]
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(rid)
+            except BaseException:
+                pass
+            make_log("Uploader", f"Service running", level="debug", rid=rid)
             await uploader_loop()
             await asyncio.sleep(5)
             await send_status("uploader_daemon", f"working (seqno={seqno})")
             seqno += 1
         except BaseException as e:
-            make_log("Uploader", f"Error: {e}", level="error")
+            make_log("Uploader", f"Error: {e}", level="error", rid=locals().get('rid'))
             await asyncio.sleep(3)
+        finally:
+            try:
+                from app.core.log_context import ctx_rid
+                ctx_rid.set(None)
+            except BaseException:
+                pass
 
 # if __name__ == '__main__':
 #     loop = asyncio.get_event_loop()
@@ -28,5 +40,3 @@ async def main_fn(memory):
 #     loop.close()
 
 
-
-

app/core/content/utils.py

@@ -1,8 +1,11 @@
 import json
 import asyncio
 import os
+import string
 import aiofiles
 from hashlib import sha256
+import re # Added import
+import unicodedata # Added import
 
 from base58 import b58encode
 from datetime import datetime, timedelta
@@ -23,7 +26,9 @@ async def create_new_content(
     content_hash_bin = sha256(content_bin).digest()
     content_hash_b58 = b58encode(content_hash_bin).decode()
 
-    new_content = db_session.query(StoredContent).filter(StoredContent.hash == content_hash_b58).first()
+    from sqlalchemy import select
+    result = await db_session.execute(select(StoredContent).where(StoredContent.hash == content_hash_b58))
+    new_content = result.scalars().first()
     if new_content:
         return new_content, False
 
@@ -35,8 +40,9 @@ async def create_new_content(
 
     )
     db_session.add(new_content)
-    db_session.commit()
-    new_content = db_session.query(StoredContent).filter(StoredContent.hash == content_hash_b58).first()
+    await db_session.commit()
+    result = await db_session.execute(select(StoredContent).where(StoredContent.hash == content_hash_b58))
+    new_content = result.scalars().first()
     assert new_content, "Content not created (through utils)"
     content_filepath = os.path.join(UPLOADS_DIR, content_hash_b58)
     async with aiofiles.open(content_filepath, 'wb') as file:
@@ -45,34 +51,125 @@ async def create_new_content(
     return new_content, True
 
 
+# New helper functions for string cleaning
+def _remove_emojis(text: str) -> str:
+    """Removes common emoji characters from a string."""
+    # This regex covers many common emojis but might not be exhaustive.
+    emoji_pattern = re.compile(
+        "["
+        "\U0001F600-\U0001F64F"  # emoticons
+        "\U0001F300-\U0001F5FF"  # symbols & pictographs
+        "\U0001F680-\U0001F6FF"  # transport & map symbols
+        "\U0001F1E0-\U0001F1FF"  # flags (iOS)
+        "\U00002702-\U000027B0"  # Dingbats
+        "\U000024C2-\U0001F251"  # Various symbols
+        "]+",
+        flags=re.UNICODE,
+    )
+    return emoji_pattern.sub(r'', text)
+
+def _clean_text_content(text: str, is_hashtag: bool = False) -> str:
+    """
+    Cleans a string by removing emojis and unusual characters.
+    Level 1: Emoji removal.
+    Level 2: Unusual character cleaning (specific logic for hashtags).
+    """
+    if not isinstance(text, str):
+        return ""
+
+    # Level 1: Remove emojis
+    text_no_emojis = _remove_emojis(text)
+
+    # Level 2: Clean unusual characters
+    if is_hashtag:
+        # Convert to lowercase
+        processed_text = text_no_emojis.lower()
+        # Replace hyphens, dots, spaces (and sequences) with a single underscore
+        processed_text = re.sub(r'[\s.-]+', '_', processed_text)
+        # Keep only lowercase letters (a-z), digits (0-9), and underscores
+        cleaned_text = re.sub(r'[^a-z0-9_]', '', processed_text)
+        # Remove leading/trailing underscores
+        cleaned_text = cleaned_text.strip('_')
+        # Consolidate multiple underscores into one
+        cleaned_text = re.sub(r'_+', '_', cleaned_text)
+        return cleaned_text
+    else: # For title, authors, or general text
+        # Normalize Unicode characters (e.g., NFKD form)
+        nfkd_form = unicodedata.normalize('NFKD', text_no_emojis)
+        # Keep letters (Unicode), numbers (Unicode), spaces, and basic punctuation
+        # This allows for a wider range of characters suitable for titles/names.
+        cleaned_text_chars = []
+        for char_in_nfkd in nfkd_form:
+            if not unicodedata.combining(char_in_nfkd): # remove combining diacritics
+                # Keep letters, numbers, spaces, and specific punctuation
+                cat = unicodedata.category(char_in_nfkd)
+                if cat.startswith('L') or cat.startswith('N') or cat.startswith('Z') or char_in_nfkd in '.,!?-':
+                    cleaned_text_chars.append(char_in_nfkd)
+        
+        cleaned_text = "".join(cleaned_text_chars)
+        # Normalize multiple spaces to a single space and strip leading/trailing spaces
+        cleaned_text = re.sub(r'\s+', ' ', cleaned_text).strip()
+        return cleaned_text
+
+
 async def create_metadata_for_item(
     db_session,
     title: str = None,
     cover_url: str = None,
     authors: list = None,
     hashtags: list = [],
+    downloadable: bool = False,
 ) -> StoredContent:
     assert title, "No title provided"
-    # assert cover_url, "No cover_url provided"
-    assert len(title) > 3, "Title too short"
-    title = title[:100].strip()
+    # assert cover_url, "No cover_url provided" # Original comment, kept as is
+
+    # Clean title using the new helper function
+    cleaned_title = _clean_text_content(title, is_hashtag=False)
+    cleaned_title = cleaned_title[:100].strip() # Truncate and strip after cleaning
+    assert len(cleaned_title) > 3, f"Cleaned title '{cleaned_title}' (from original '{title}') is too short or became empty after cleaning."
+
+    # Process and clean hashtags
+    processed_hashtags = []
+    if hashtags and isinstance(hashtags, list):
+        for _h_tag_text in hashtags:
+            if isinstance(_h_tag_text, str):
+                cleaned_h = _clean_text_content(_h_tag_text, is_hashtag=True)
+                if cleaned_h:  # Add only if not empty after cleaning
+                    processed_hashtags.append(cleaned_h)
+    
+    # Ensure uniqueness of hashtags and limit their count (e.g., to first 10 unique)
+    # Using dict.fromkeys to preserve order while ensuring uniqueness
+    processed_hashtags = list(dict.fromkeys(processed_hashtags))[:10] 
 
     item_metadata = {
-        'name': title,
-        'description': ' '.join([f"#{_h.replace(' ', '_')}" for _h in hashtags]),
+        'name': cleaned_title,
         'attributes': [
             # {
             #     'trait_type': 'Artist',
             #     'value': 'Unknown'
             # },
         ],
+        'downloadable': downloadable,
+        'tags': processed_hashtags,  # New field for storing the list of cleaned hashtags
     }
+
+    # Generate description from the processed hashtags
+    item_metadata['description'] = ' '.join([f"#{h}" for h in processed_hashtags if h])
+
     if cover_url:
         item_metadata['image'] = cover_url
 
-    item_metadata['authors'] = [
-        ''.join([_a_ch for _a_ch in _a if len(_a_ch.encode()) == 1]) for _a in (authors or [])[:500]
-    ]
+    # Clean authors
+    cleaned_authors = []
+    if authors and isinstance(authors, list):
+        for author_name in (authors or [])[:500]: # Limit number of authors
+            if isinstance(author_name, str):
+                # Apply general cleaning to author names
+                # This replaces the old logic: ''.join([_a_ch for _a_ch in _a if len(_a_ch.encode()) == 1])
+                cleaned_author = _clean_text_content(author_name, is_hashtag=False)
+                if cleaned_author.strip(): # Ensure not empty
+                     cleaned_authors.append(cleaned_author.strip()[:100]) # Limit length of each author name
+    item_metadata['authors'] = cleaned_authors
 
     # Upload file
     metadata_bin = json.dumps(item_metadata).encode()

app/core/crypto/aes_gcm_siv_stream.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import hmac
+import hashlib
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+from cryptography.hazmat.primitives.ciphers.aead import AESGCMSIV
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+SCHEME_AES_GCM_SIV = 0x01
+
+
+def _derive_nonce(salt: bytes, idx: int) -> bytes:
+    b = idx.to_bytes(8, 'big')
+    return hmac.new(salt, b, hashlib.sha256).digest()[:12]
+
+
+def build_header(chunk_bytes: int, salt: bytes) -> bytes:
+    assert 0 < chunk_bytes <= (1 << 31)
+    assert 1 <= len(salt) <= 255
+    # MAGIC(4) | ver(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt | reserved(5)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr += bytes([VERSION])
+    hdr += bytes([SCHEME_AES_GCM_SIV])
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr += bytes([len(salt)])
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """
+    Yield ENCF v1 stream using AES-GCM-SIV per chunk with deterministic nonces.
+    Frame: [p_len:4][cipher][tag(16)].
+    """
+    yield build_header(chunk_bytes, salt)
+    idx = 0
+    cipher = AESGCMSIV(key)
+
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        ct_and_tag = cipher.encrypt(nonce, block, associated_data=None)
+        # Split tag
+        tag = ct_and_tag[-16:]
+        ct = ct_and_tag[:-16]
+        yield struct.pack(">I", len(block))
+        yield ct
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """Parse ENCF v1 (AES-GCM-SIV) and write plaintext to out_path."""
+    import aiofiles
+    buf = bytearray()
+
+    async def _fill(n: int):
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # header minimal
+    await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    if version != 1 or scheme != SCHEME_AES_GCM_SIV:
+        raise ValueError("unsupported encf header")
+    chunk_bytes = struct.unpack(">I", bytes(buf[6:10]))[0]
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    salt = bytes(buf[11:11 + salt_len])
+    del buf[:hdr_len]
+
+    cipher = AESGCMSIV(key)
+
+    async with aiofiles.open(out_path, 'wb') as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            await _fill(4)
+            if len(buf) == 0:
+                break
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            ct = bytes(buf[:p_len])
+            tag = bytes(buf[p_len:p_len+TAG_LEN])
+            del buf[:p_len+TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            pt = cipher.decrypt(nonce, ct + tag, associated_data=None)
+            await out.write(pt)
+            idx += 1

app/core/crypto/aes_gcm_stream.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import hmac
+import hashlib
+import os
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+import aiofiles
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+SCHEME_AES_GCM = 0x03
+
+CHUNK_BYTES = int(os.getenv("CRYPTO_CHUNK_BYTES", "1048576"))
+
+
+def _derive_nonce(salt: bytes, idx: int) -> bytes:
+    """Derive a deterministic 12-byte nonce from salt and chunk index."""
+    if len(salt) < 12:
+        raise ValueError("salt must be at least 12 bytes")
+    idx_bytes = idx.to_bytes(8, "big")
+    return hmac.new(salt, idx_bytes, hashlib.sha256).digest()[:12]
+
+
+def build_header(chunk_bytes: int, salt: bytes) -> bytes:
+    if not (0 < chunk_bytes <= (1 << 31)):
+        raise ValueError("chunk_bytes must be between 1 and 2^31")
+    if not (1 <= len(salt) <= 255):
+        raise ValueError("salt length must be 1..255 bytes")
+    # MAGIC(4) | ver(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt | reserved(5 zeros)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr.append(VERSION)
+    hdr.append(SCHEME_AES_GCM)
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr.append(len(salt))
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """Yield ENCF v1 frames encrypted with AES-GCM."""
+    if len(key) not in (16, 24, 32):
+        raise ValueError("AES-GCM key must be 128, 192 or 256 bits long")
+    cipher = AESGCM(key)
+    yield build_header(chunk_bytes, salt)
+    idx = 0
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        ct = cipher.encrypt(nonce, block, associated_data=None)
+        tag = ct[-16:]
+        data = ct[:-16]
+        yield struct.pack(">I", len(block))
+        yield data
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """Parse ENCF v1 (AES-GCM) stream and write plaintext to `out_path`."""
+    if len(key) not in (16, 24, 32):
+        raise ValueError("AES-GCM key must be 128, 192 or 256 bits long")
+    cipher = AESGCM(key)
+    buf = bytearray()
+
+    async def _fill(n: int) -> None:
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # Parse header
+    await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    if version != VERSION or scheme != SCHEME_AES_GCM:
+        raise ValueError("unsupported ENCF header")
+    chunk_bytes = struct.unpack(">I", bytes(buf[6:10]))[0]
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    salt = bytes(buf[11:11 + salt_len])
+    del buf[:hdr_len]
+
+    async with aiofiles.open(out_path, "wb") as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            await _fill(4)
+            if len(buf) == 0:
+                break
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            ct = bytes(buf[:p_len])
+            tag = bytes(buf[p_len:p_len + TAG_LEN])
+            del buf[:p_len + TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            pt = cipher.decrypt(nonce, ct + tag, associated_data=None)
+            await out.write(pt)
+            idx += 1

app/core/crypto/aes_siv_stream.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import os
+import struct
+from typing import BinaryIO, Iterator, AsyncIterator
+
+from Crypto.Cipher import SIV
+from Crypto.Cipher import AES
+
+
+MAGIC = b"ENCF"
+VERSION = 1
+
+# Scheme codes
+SCHEME_AES_SIV = 0x02  # RFC5297 AES-SIV (CMAC-based)
+
+
+def build_header(chunk_bytes: int, salt: bytes, scheme: int = SCHEME_AES_SIV) -> bytes:
+    assert 0 < chunk_bytes <= (1 << 31)
+    assert 1 <= len(salt) <= 255
+    # Layout: MAGIC(4) | version(1) | scheme(1) | chunk_bytes(4,BE) | salt_len(1) | salt(N) | reserved(5 zeros)
+    hdr = bytearray()
+    hdr += MAGIC
+    hdr += bytes([VERSION])
+    hdr += bytes([scheme])
+    hdr += struct.pack(">I", int(chunk_bytes))
+    hdr += bytes([len(salt)])
+    hdr += salt
+    hdr += b"\x00" * 5
+    return bytes(hdr)
+
+
+def parse_header(buf: bytes) -> tuple[int, int, int, bytes, int]:
+    if len(buf) < 4 + 1 + 1 + 4 + 1:
+        raise ValueError("header too short")
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    version = buf[4]
+    scheme = buf[5]
+    chunk_bytes = struct.unpack(">I", buf[6:10])[0]
+    salt_len = buf[10]
+    needed = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    if len(buf) < needed:
+        raise ValueError("incomplete header")
+    salt = buf[11:11 + salt_len]
+    # reserved 5 bytes at the end ignored
+    return version, scheme, chunk_bytes, salt, needed
+
+
+def _ad(salt: bytes, idx: int) -> bytes:
+    return salt + struct.pack(">Q", idx)
+
+
+def encrypt_file_to_encf(src: BinaryIO, key: bytes, chunk_bytes: int, salt: bytes) -> Iterator[bytes]:
+    """
+    Yield ENCF v1 stream bytes: [header] then for each chunk: [p_len:4][cipher][tag(16)].
+    Uses AES-SIV (RFC5297) with per-chunk associated data salt||index.
+    """
+    yield build_header(chunk_bytes, salt, SCHEME_AES_SIV)
+    idx = 0
+    while True:
+        block = src.read(chunk_bytes)
+        if not block:
+            break
+        siv = SIV.new(key=key, ciphermod=AES)  # new object per message
+        siv.update(_ad(salt, idx))
+        ciph, tag = siv.encrypt_and_digest(block)
+        yield struct.pack(">I", len(block))
+        yield ciph
+        yield tag
+        idx += 1
+
+
+async def decrypt_encf_to_file(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """
+    Parse ENCF v1 stream from async byte iterator and write plaintext to out_path.
+    """
+    import aiofiles
+    from Crypto.Cipher import SIV as _SIV
+    from Crypto.Cipher import AES as _AES
+
+    buf = bytearray()
+
+    async def _fill(n: int):
+        """Ensure at least n bytes in buffer (or EOF)."""
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                chunk = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if chunk:
+                buf.extend(chunk)
+
+    # Read and parse header
+    await _fill(4 + 1 + 1 + 4 + 1)  # minimal header
+    # Might still be incomplete if salt_len > 0; keep filling progressively
+    # First, get preliminary to know salt_len
+    if len(buf) < 11:
+        await _fill(11)
+    if buf[:4] != MAGIC:
+        raise ValueError("bad magic")
+    salt_len = buf[10]
+    hdr_len = 4 + 1 + 1 + 4 + 1 + salt_len + 5
+    await _fill(hdr_len)
+    version, scheme, chunk_bytes, salt, consumed = parse_header(bytes(buf))
+    del buf[:consumed]
+    if version != 1:
+        raise ValueError("unsupported ENCF version")
+    if scheme != SCHEME_AES_SIV:
+        raise ValueError("unsupported scheme")
+
+    async with aiofiles.open(out_path, 'wb') as out:
+        idx = 0
+        TAG_LEN = 16
+        while True:
+            # Need at least 4 bytes for p_len
+            await _fill(4)
+            if len(buf) == 0:
+                break  # EOF exactly on boundary
+            if len(buf) < 4:
+                raise ValueError("truncated frame length")
+            p_len = struct.unpack(">I", bytes(buf[:4]))[0]
+            del buf[:4]
+            # Now need p_len + 16 bytes
+            await _fill(p_len + TAG_LEN)
+            if len(buf) < p_len + TAG_LEN:
+                raise ValueError("truncated cipher/tag")
+            c = bytes(buf[:p_len])
+            t = bytes(buf[p_len:p_len+TAG_LEN])
+            del buf[:p_len+TAG_LEN]
+            siv = _SIV.new(key=key, ciphermod=_AES)
+            siv.update(_ad(salt, idx))
+            p = siv.decrypt_and_verify(c, t)
+            await out.write(p)
+            idx += 1
+

app/core/crypto/aesgcm_stream.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import os
+import hmac
+import hashlib
+from typing import BinaryIO, Iterator
+
+from Crypto.Cipher import AES
+
+
+CHUNK_BYTES = int(os.getenv("CRYPTO_CHUNK_BYTES", "1048576"))  # 1 MiB
+
+
+def _derive_nonce(salt: bytes, chunk_index: int) -> bytes:
+    """Derive a 12-byte GCM nonce deterministically from per-file salt and chunk index."""
+    idx = chunk_index.to_bytes(8, 'big')
+    digest = hmac.new(salt, idx, hashlib.sha256).digest()
+    return digest[:12]
+
+
+def encrypt_stream_aesgcm(src: BinaryIO, key: bytes, salt: bytes) -> Iterator[bytes]:
+    """
+    Read plaintext from src by CHUNK_BYTES, encrypt each chunk with AES-GCM using a
+    deterministic nonce derived from (salt, index). Yields bytes in framing: [C_i][TAG_i]...
+    Ciphertext length equals plaintext chunk length. Tag is 16 bytes.
+    """
+    assert len(key) in (16, 24, 32)
+    assert len(salt) >= 12
+    idx = 0
+    while True:
+        block = src.read(CHUNK_BYTES)
+        if not block:
+            break
+        nonce = _derive_nonce(salt, idx)
+        cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+        ciphertext, tag = cipher.encrypt_and_digest(block)
+        yield ciphertext
+        yield tag
+        idx += 1
+
+
+def decrypt_stream_aesgcm_iter(byte_iter: Iterator[bytes], key: bytes, salt: bytes) -> Iterator[bytes]:
+    """
+    Decrypt a stream that was produced by encrypt_stream_aesgcm.
+    Frame format: concatenation of [C_i][TAG_i] for each i, where |C_i| = CHUNK_BYTES and |TAG_i|=16.
+    We accept arbitrary chunking from the underlying iterator and reframe accordingly.
+    """
+    assert len(key) in (16, 24, 32)
+    buf = bytearray()
+    idx = 0
+    TAG_LEN = 16
+    def _try_yield():
+        nonlocal idx
+        out = []
+        while len(buf) >= CHUNK_BYTES + TAG_LEN:
+            c = bytes(buf[:CHUNK_BYTES])
+            t = bytes(buf[CHUNK_BYTES:CHUNK_BYTES+TAG_LEN])
+            del buf[:CHUNK_BYTES+TAG_LEN]
+            nonce = _derive_nonce(salt, idx)
+            cipher = AES.new(key, AES.MODE_GCM, nonce=nonce)
+            try:
+                p = cipher.decrypt_and_verify(c, t)
+            except Exception as e:
+                raise ValueError(f"Decrypt failed at chunk {idx}: {e}")
+            out.append(p)
+            idx += 1
+        return out
+    for chunk in byte_iter:
+        if not chunk:
+            continue
+        buf.extend(chunk)
+        for p in _try_yield():
+            yield p
+    # At end, buffer must be empty
+    if len(buf) != 0:
+        raise ValueError("Trailing bytes in encrypted stream (incomplete frame)")
+

app/core/crypto/cli.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+import sys
+from typing import Optional
+
+from .aes_gcm_stream import CHUNK_BYTES, encrypt_file_to_encf
+from .encf_stream import decrypt_encf_auto
+from .keywrap import unwrap_dek, KeyWrapError
+
+
+def _normalize_base64(value: str) -> str:
+    padding = (-len(value)) % 4
+    if padding:
+        return value + "=" * padding
+    return value
+
+
+def _decode_key(value: str, fmt: str) -> bytes:
+    if fmt == "base64":
+        return base64.b64decode(_normalize_base64(value))
+    if fmt == "hex":
+        cleaned = value[2:] if value.lower().startswith("0x") else value
+        return bytes.fromhex(cleaned)
+    if fmt == "raw":
+        return value.encode()
+    raise ValueError(f"unsupported key format: {fmt}")
+
+
+def _decode_salt(value: str, fmt: str) -> bytes:
+    if fmt == "base64":
+        return base64.b64decode(_normalize_base64(value))
+    if fmt == "hex":
+        cleaned = value[2:] if value.lower().startswith("0x") else value
+        return bytes.fromhex(cleaned)
+    raise ValueError(f"unsupported salt format: {fmt}")
+
+
+async def _decrypt_file(input_path: str, key: bytes, output_path: str) -> None:
+    async def _aiter():
+        with open(input_path, "rb") as src:
+            while True:
+                chunk = src.read(65536)
+                if not chunk:
+                    break
+                yield chunk
+
+    await decrypt_encf_auto(_aiter(), key, output_path)
+
+
+def cmd_encrypt(args: argparse.Namespace) -> int:
+    key = _decode_key(args.key, args.key_format)
+    salt = _decode_salt(args.salt, args.salt_format) if args.salt else os.urandom(args.salt_bytes)
+    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+    with open(args.input, "rb") as src, open(args.output, "wb") as dst:
+        for chunk in encrypt_file_to_encf(src, key, args.chunk_bytes, salt):
+            dst.write(chunk)
+    # Emit JSON metadata with salt for convenience
+    meta = {
+        "salt_b64": base64.b64encode(salt).decode(),
+        "chunk_bytes": args.chunk_bytes,
+        "aead_scheme": "AES_GCM",
+    }
+    print(json.dumps(meta), file=sys.stdout)
+    return 0
+
+
+def cmd_decrypt(args: argparse.Namespace) -> int:
+    if bool(args.key) == bool(args.wrapped_key):
+        raise SystemExit("Provide exactly one of --key or --wrapped-key")
+    if args.wrapped_key:
+        try:
+            key = unwrap_dek(args.wrapped_key)
+        except KeyWrapError as exc:
+            raise SystemExit(f"Failed to unwrap key: {exc}") from exc
+    else:
+        key = _decode_key(args.key, args.key_format)
+    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+    asyncio.run(_decrypt_file(args.input, key, args.output))
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="python -m app.core.crypto.cli", description="ENCF AES-GCM helper")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    enc = sub.add_parser("encrypt", help="Encrypt file into ENCF v1 stream (AES-256-GCM)")
+    enc.add_argument("--input", required=True, help="Path to plaintext input file")
+    enc.add_argument("--output", required=True, help="Destination path for ENCF output")
+    enc.add_argument("--key", required=True, help="Encryption key")
+    enc.add_argument("--key-format", choices=["base64", "hex", "raw"], default="base64")
+    enc.add_argument("--salt", help="Salt in specified format; generates random if omitted")
+    enc.add_argument("--salt-format", choices=["base64", "hex"], default="base64")
+    enc.add_argument("--salt-bytes", type=int, default=16, help="Salt length when generated (default: 16)")
+    enc.add_argument("--chunk-bytes", type=int, default=CHUNK_BYTES, help="Plaintext chunk size (default from env)")
+    enc.set_defaults(func=cmd_encrypt)
+
+    dec = sub.add_parser("decrypt", help="Decrypt ENCF stream to plaintext")
+    dec.add_argument("--input", required=True, help="Path to ENCF input file")
+    dec.add_argument("--output", required=True, help="Destination path for decrypted file")
+    dec.add_argument("--key", help="Plaintext key")
+    dec.add_argument("--wrapped-key", help="Wrapped key produced by the backend")
+    dec.add_argument("--key-format", choices=["base64", "hex", "raw"], default="base64")
+    dec.set_defaults(func=cmd_decrypt)
+
+    return parser
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

app/core/crypto/encf_stream.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from typing import AsyncIterator
+
+from .aes_gcm_siv_stream import MAGIC as _MAGIC, VERSION as _VER, SCHEME_AES_GCM_SIV
+from .aes_gcm_siv_stream import decrypt_encf_to_file as _dec_gcmsiv
+from .aes_gcm_stream import SCHEME_AES_GCM, decrypt_encf_to_file as _dec_gcm
+from .aes_siv_stream import decrypt_encf_to_file as _dec_siv
+
+
+async def decrypt_encf_auto(byte_iter: AsyncIterator[bytes], key: bytes, out_path: str) -> None:
+    """
+    Detect scheme by peeking header, then delegate to proper decrypter.
+    Re-feeds the peeked bytes back to the chosen decoder.
+    """
+    buf = bytearray()
+
+    async def _fill(n: int):
+        nonlocal buf
+        while len(buf) < n:
+            try:
+                ch = await byte_iter.__anext__()
+            except StopAsyncIteration:
+                break
+            if ch:
+                buf.extend(ch)
+
+    await _fill(11)
+    if buf[:4] != _MAGIC:
+        raise ValueError("bad magic")
+    scheme = buf[5]
+
+    async def _prepend_iter():
+        nonlocal buf
+        if buf:
+            yield bytes(buf)
+        async for ch in byte_iter:
+            yield ch
+
+    if scheme == SCHEME_AES_GCM_SIV:
+        await _dec_gcmsiv(_prepend_iter(), key, out_path)
+    elif scheme == SCHEME_AES_GCM:
+        await _dec_gcm(_prepend_iter(), key, out_path)
+    else:
+        await _dec_siv(_prepend_iter(), key, out_path)

app/core/crypto/keywrap.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import base64
+import os
+import threading
+from typing import Optional
+
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+
+_VERSION = 1
+_PREFIX_LEN = 1  # version byte
+_NONCE_LEN = 12
+_TAG_LEN = 16
+_valid_key_lengths = {16, 24, 32}
+_kek_lock = threading.Lock()
+_cached_kek: Optional[bytes] = None
+
+
+class KeyWrapError(RuntimeError):
+    """Raised when KEK configuration or unwrap operations fail."""
+
+
+def _normalize_base64(value: str) -> str:
+    v = value.strip()
+    missing = (-len(v)) % 4
+    if missing:
+        v += "=" * missing
+    return v
+
+
+def _decode_key_material(value: str) -> bytes:
+    v = value.strip()
+    if v.startswith("0x") or v.startswith("0X"):
+        v = v[2:]
+    try:
+        raw = bytes.fromhex(v)
+        if len(raw) in _valid_key_lengths:
+            return raw
+    except ValueError:
+        pass
+    try:
+        raw = base64.b64decode(_normalize_base64(value), validate=False)
+        if len(raw) in _valid_key_lengths:
+            return raw
+    except Exception as exc:  # noqa: BLE001 - we want to re-raise as KeyWrapError
+        raise KeyWrapError(f"invalid KEK encoding: {exc}") from exc
+    raise KeyWrapError("KEK must decode to 16/24/32 bytes")
+
+
+def _load_kek() -> bytes:
+    global _cached_kek
+    if _cached_kek is not None:
+        return _cached_kek
+    with _kek_lock:
+        if _cached_kek is not None:
+            return _cached_kek
+        env = os.getenv("CONTENT_KEY_KEK_B64") or os.getenv("CONTENT_KEY_KEK_HEX")
+        if not env:
+            raise KeyWrapError("CONTENT_KEY_KEK_B64 or CONTENT_KEY_KEK_HEX must be set")
+        kek = _decode_key_material(env)
+        if len(kek) != 32:
+            # Force 256-bit KEK for uniform security properties
+            raise KeyWrapError("KEK must be 32 bytes (256-bit) for AES-256-GCM")
+        _cached_kek = kek
+        return _cached_kek
+
+
+def wrap_dek(plaintext: bytes) -> str:
+    """Wrap a DEK (plaintext bytes) with AES-256-GCM; return base64 string."""
+    if not isinstance(plaintext, (bytes, bytearray)):
+        raise TypeError("plaintext must be bytes")
+    kek = _load_kek()
+    nonce = os.urandom(_NONCE_LEN)
+    cipher = AESGCM(kek)
+    ct = cipher.encrypt(nonce, bytes(plaintext), associated_data=None)
+    blob = bytes([_VERSION]) + nonce + ct
+    return base64.b64encode(blob).decode()
+
+
+def unwrap_dek(encoded: str) -> bytes:
+    """Unwrap DEK from base64 string. Supports legacy (raw base64 key) values."""
+    if not encoded:
+        raise KeyWrapError("empty key payload")
+    try:
+        raw = base64.b64decode(_normalize_base64(encoded), validate=False)
+    except Exception as exc:  # noqa: BLE001
+        raise KeyWrapError(f"invalid base64 payload: {exc}") from exc
+    if not raw:
+        raise KeyWrapError("decoded payload is empty")
+    version = raw[0]
+    if version == _VERSION:
+        if len(raw) < _PREFIX_LEN + _NONCE_LEN + _TAG_LEN + 1:
+            raise KeyWrapError("wrapped payload too short")
+        nonce = raw[_PREFIX_LEN:_PREFIX_LEN + _NONCE_LEN]
+        ciphertext = raw[_PREFIX_LEN + _NONCE_LEN:]
+        kek = _load_kek()
+        cipher = AESGCM(kek)
+        try:
+            return cipher.decrypt(nonce, ciphertext, associated_data=None)
+        except Exception as exc:  # noqa: BLE001
+            raise KeyWrapError(f"unwrap failed: {exc}") from exc
+    # Legacy fallback: value is raw DEK (no version prefix)
+    if len(raw) in {16, 24, 32}:
+        return raw
+    raise KeyWrapError("unknown key payload format")

app/core/crypto/x25519.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import base64
+from typing import Tuple
+
+from nacl import public, signing, bindings
+
+
+def ed25519_to_x25519(ed_seed: bytes) -> Tuple[public.PrivateKey, public.PublicKey]:
+    """Convert Ed25519 seed (32 bytes) to X25519 key pair using libsodium conversion."""
+    if len(ed_seed) != 32:
+        raise ValueError("ed25519 seed must be 32 bytes")
+    sk_ed = signing.SigningKey(ed_seed)
+    sk_ed_bytes = sk_ed._seed + sk_ed.verify_key._key  # 64-byte expanded sk (seed||pub)
+    sk_x_bytes = bindings.crypto_sign_ed25519_sk_to_curve25519(sk_ed_bytes)
+    pk_x_bytes = bindings.crypto_sign_ed25519_pk_to_curve25519(bytes(sk_ed.verify_key))
+    sk_x = public.PrivateKey(sk_x_bytes)
+    pk_x = public.PublicKey(pk_x_bytes)
+    return sk_x, pk_x
+
+
+def x25519_pub_b64_from_ed_seed(ed_seed: bytes) -> str:
+    _, pk = ed25519_to_x25519(ed_seed)
+    return base64.b64encode(bytes(pk)).decode()
+

app/core/ipfs_client.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import os
+from typing import AsyncIterator, Dict, Any, Iterable, Optional
+
+import httpx
+
+IPFS_API_URL = os.getenv("IPFS_API_URL", "http://ipfs:5001")
+IPFS_GATEWAY_URL = os.getenv("IPFS_GATEWAY_URL", "http://ipfs:8080")
+
+
+async def add_streamed_file(stream_iter: Iterable[bytes], filename: str = "file.bin", params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """
+    Stream-encrypt pipeline can pass a generator of bytes here. We stream to /api/v0/add as multipart.
+    Returns dict with fields from IPFS: { Name, Hash, Size }.
+    """
+    params = params or {}
+    # Ensure deterministic chunking and CIDv1
+    default_params = {
+        "cid-version": 1,
+        "raw-leaves": "true",
+        "chunker": f"size-{int(os.getenv('CRYPTO_CHUNK_BYTES', '1048576'))}",
+        "pin": "true",
+        "wrap-with-directory": "false",
+        "progress": "true",
+    }
+    q = {**default_params, **params}
+
+    async with httpx.AsyncClient(timeout=None) as client:
+        files = {"file": (filename, stream_iter, "application/octet-stream")}
+        r = await client.post(f"{IPFS_API_URL}/api/v0/add", params=q, files=files)
+        r.raise_for_status()
+        # /add may emit NDJSON lines; most often single JSON
+        try:
+            data = r.json()
+        except Exception:
+            # Fallback: last non-empty line
+            last = [ln for ln in r.text.splitlines() if ln.strip()][-1]
+            import json as _json
+            data = _json.loads(last)
+        return data
+
+
+async def pin_add(cid: str, recursive: bool = True) -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=None) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/pin/add", params={"arg": cid, "recursive": str(recursive).lower(), "progress": "true"})
+        r.raise_for_status()
+        return r.json()
+
+
+async def pin_ls(cid: str) -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=30) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/pin/ls", params={"arg": cid})
+        r.raise_for_status()
+        return r.json()
+
+
+async def swarm_connect(multiaddr: str) -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=10) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/swarm/connect", params={"arg": multiaddr})
+        r.raise_for_status()
+        return r.json()
+
+
+async def cat_stream(cid: str):
+    client = httpx.AsyncClient(timeout=None)
+    try:
+        async with client.stream("POST", f"{IPFS_API_URL}/api/v0/cat", params={"arg": cid}) as r:
+            r.raise_for_status()
+            async for chunk in r.aiter_bytes():
+                if chunk:
+                    yield chunk
+    finally:
+        await client.aclose()
+
+
+async def find_providers(cid: str, max_results: int = 8):
+    """Query DHT for providers of a CID and return a list of {peer, addrs[]}.
+    Uses /api/v0/dht/findprovs and parses NDJSON stream.
+    """
+    out = []
+    async with httpx.AsyncClient(timeout=30) as client:
+        async with client.stream("POST", f"{IPFS_API_URL}/api/v0/dht/findprovs", params={"arg": cid}) as r:
+            r.raise_for_status()
+            async for line in r.aiter_lines():
+                if not line:
+                    continue
+                try:
+                    j = httpx.Response(200, text=line).json()
+                except Exception:
+                    import json as _json
+                    try:
+                        j = _json.loads(line)
+                    except Exception:
+                        continue
+                # Entries can include 'Extra' or 'Responses'
+                resps = j.get('Responses') or []
+                for resp in resps:
+                    peer = resp.get('ID') or resp.get('ID', '')
+                    addrs = resp.get('Addrs') or []
+                    if peer:
+                        out.append({"peer": peer, "addrs": addrs})
+                        if len(out) >= max_results:
+                            return out
+    return out
+
+
+async def bitswap_stat() -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=10) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/bitswap/stat")
+        r.raise_for_status()
+        return r.json()
+
+
+async def repo_stat() -> Dict[str, Any]:
+    async with httpx.AsyncClient(timeout=10) as client:
+        r = await client.post(f"{IPFS_API_URL}/api/v0/repo/stat")
+        r.raise_for_status()
+        return r.json()

app/core/log_context.py

@@ -0,0 +1,11 @@
+from contextvars import ContextVar
+
+# Correlation for HTTP requests
+ctx_session_id = ContextVar('ctx_session_id', default=None)
+ctx_user_id = ContextVar('ctx_user_id', default=None)
+ctx_method = ContextVar('ctx_method', default=None)
+ctx_path = ContextVar('ctx_path', default=None)
+ctx_remote = ContextVar('ctx_remote', default=None)
+
+# Correlation for background loop iterations
+ctx_rid = ContextVar('ctx_rid', default=None)

app/core/logger.py

@@ -1,5 +1,6 @@
 import os
 from app.core.projscale_logger import logger
+from app.core.log_context import ctx_session_id, ctx_user_id, ctx_method, ctx_path, ctx_remote, ctx_rid
 import logging
 
 LOG_LEVELS = {
@@ -17,6 +18,23 @@ def make_log(issuer, message, *args, level='INFO', **kwargs):
 
     assert level.upper() in LOG_LEVELS.keys(), f"Unknown log level"
     _log = getattr(logger, level.lower())
+    # Merge context variables if not explicitly provided
+    context_fields = {
+        'sid': kwargs.get('sid') or kwargs.get('session_id') or ctx_session_id.get(),
+        'user_id': kwargs.get('user_id') or ctx_user_id.get(),
+        'method': kwargs.get('method') or ctx_method.get(),
+        'path': kwargs.get('path') or ctx_path.get(),
+        'remote': kwargs.get('remote') or ctx_remote.get(),
+        'rid': kwargs.get('rid') or ctx_rid.get(),
+    }
+    # Only include non-empty context
+    for k, v in list(context_fields.items()):
+        if v is None:
+            context_fields.pop(k)
+    # Do not override provided kwargs; merge missing only
+    for k, v in context_fields.items():
+        kwargs.setdefault(k, v)
+
     log_buffer = f"[{issuer if not (issuer is None) else 'System'}] {message}"
     if args:
         log_buffer += f" | {args}"

app/core/models/__init__.py

@@ -11,3 +11,13 @@ from app.core.models.content.user_content import UserContent, UserAction
 from app.core.models._config import ServiceConfigValue, ServiceConfig
 from app.core.models.asset import Asset
 from app.core.models.my_network import KnownNode, KnownNodeIncident, RemoteContentIndex
+from app.core.models.promo import PromoAction
+from app.core.models.tasks import BlockchainTask
+from app.core.models.content_v3 import (
+    EncryptedContent,
+    ContentKey,
+    IpfsSync,
+    ContentDerivative,
+    ContentIndexItem,
+    KeyGrant,
+)

app/core/models/_config.py

@@ -1,6 +1,6 @@
 
 from app.core.models.base import AlchemyBase
-from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
+from sqlalchemy import Column, Integer, String, JSON, select
 
 
 class ServiceConfigValue(AlchemyBase):
@@ -8,7 +8,7 @@ class ServiceConfigValue(AlchemyBase):
 
     id = Column(Integer, autoincrement=True, primary_key=True)
     key = Column(String(128), nullable=False, unique=True)
-    packed_value = Column(JSON, nullable=False, default={})
+    packed_value = Column(JSON, nullable=False, default=dict)
 
     @property
     def value(self):
@@ -19,20 +19,18 @@ class ServiceConfig:
     def __init__(self, session):
         self.session = session
 
-    def get(self, key, default=None):
-        result = self.session.query(ServiceConfigValue).filter(ServiceConfigValue.key == key).first()
+    async def get(self, key, default=None):
+        result = (await self.session.execute(select(ServiceConfigValue).where(ServiceConfigValue.key == key))).scalars().first()
         return (result.value if result else None) or default
 
-    def set(self, key, value):
-        config_value = self.session.query(ServiceConfigValue).filter(
-            ServiceConfigValue.key == key
-        ).first()
-        if not config_value:
-            config_value = ServiceConfigValue(key=key)
-            self.session.add(config_value)
-            self.session.commit()
-            return self.set(key, value)
+    async def set(self, key, value):
+        result = (await self.session.execute(select(ServiceConfigValue).where(ServiceConfigValue.key == key))).scalars().first()
+        if not result:
+            result = ServiceConfigValue(key=key)
+            self.session.add(result)
+            await self.session.commit()
+            return await self.set(key, value)
 
-        config_value.packed_value = {'value': value}
-        self.session.commit()
+        result.packed_value = {'value': value}
+        await self.session.commit()
         return

app/core/models/_telegram/templates/player.py

@@ -1,4 +1,4 @@
-from sqlalchemy import and_
+from sqlalchemy import and_, select
 from app.core.models.node_storage import StoredContent
 from app.core.models.content.user_content import UserContent, UserAction
 from app.core.logger import make_log
@@ -27,17 +27,17 @@ class PlayerTemplates:
             if not content.encrypted:
                 local_content = content
             else:
-                local_content = db_session.query(StoredContent).filter_by(
-                    id=content.decrypted_content_id
-                ).first()
+                local_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == content.decrypted_content_id))).scalars().first()
                 # TODO: add check decrypted_content by .format_json()['content_cid']
                 if local_content:
                     cd_log += f"Decrypted: {local_content.hash}. "
                 else:
                     cd_log += "Can't decrypt content. "
 
-        user_wallet_address = self.user.wallet_address(self.db_session)
-        user_existing_license = self.db_session.query(UserContent).filter_by(user_id=self.user.id, content_id=content.id).first()
+        user_wallet_address = await self.user.wallet_address_async(self.db_session)
+        user_existing_license = (await self.db_session.execute(select(UserContent).where(
+            and_(UserContent.user_id == self.user.id, UserContent.content_id == content.id)
+        ))).scalars().first()
 
         if local_content:
             content_meta = content.json_format()
@@ -48,29 +48,16 @@ class PlayerTemplates:
             except:
                 content_type, content_encoding = 'application', 'x-binary'
 
-            content_metadata = StoredContent.from_cid(db_session, content_meta.get('metadata_cid') or None)
+            content_metadata = await StoredContent.from_cid_async(db_session, content_meta.get('metadata_cid') or None)
             with open(content_metadata.filepath, 'r') as f:
                 content_metadata_json = json.loads(f.read())
 
             try:
-                cover_content = StoredContent.from_cid(self.db_session, content_meta.get('cover_cid') or None)
+                cover_content = await StoredContent.from_cid_async(self.db_session, content_meta.get('cover_cid') or None)
                 cd_log += f"Cover content: {cover_content.cid.serialize_v2()}. "
             except BaseException as e:
                 cd_log += f"Can't get cover content: {e}. "
                 cover_content = None
-
-            local_content.meta['cover_cid'] = cover_content.cid.serialize_v2() if cover_content else None
-
-            local_content_cid = local_content.cid
-            local_content_url = f"{PROJECT_HOST}/api/v1.5/storage/{local_content_cid.serialize_v2()}"
-            converted_content = content.meta.get('converted_content')
-            if not converted_content:
-                r = await tg_process_template(
-                    self, self.user.translated('p_playerContext_contentNotReady'),
-                    message_id=message_id,
-                    message_type='common'
-                )
-                return r
             
             content_share_link = {
                 'text': self.user.translated('p_shareLinkContext').format(title=content_metadata_json.get('name', "")),
@@ -78,88 +65,9 @@ class PlayerTemplates:
             }
             if user_existing_license:
                 content_share_link['url'] = f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}/content?startapp={user_existing_license.onchain_address}"
-            
-            preview_content = db_session.query(StoredContent).filter(
-                StoredContent.hash == converted_content['low_preview']
-            ).first()
-            if preview_content.filename.split('.')[-1] in ['mov', 'mp4']:
-                content_type = 'video'
 
-            local_content_preview_url = preview_content.web_url
-            if content_type == 'audio':
-                audio_title = content_metadata_json.get('name', "").split(' - ')
-                if len(audio_title) > 1:
-                    template_kwargs['performer'] = audio_title[0].strip()
-                    audio_title = audio_title[1:]
-
-                template_kwargs['title'] = audio_title[0].strip()
-                template_kwargs['protect_content'] = True
-                template_kwargs['audio'] = URLInputFile(local_content_preview_url)
-                if cover_content:
-                    template_kwargs['thumbnail'] = URLInputFile(cover_content.web_url)
-
-                if self.bot_id == 1:
-                    inline_keyboard_array.append([
-                        {
-                            'text': self.user.translated('shareTrack_button'),
-                            'switch_inline_query': f"Q{user_existing_license.onchain_address}" if user_existing_license else f"C{content.cid.serialize_v2()}",
-                        },
-                        {
-                            'text': self.user.translated('shareLink_button'),
-                            'url': f"https://t.me/share/url?text={urllib.parse.quote(content_share_link['text'])}&url={urllib.parse.quote(content_share_link['url'])}"
-                        }
-                    ])
-                    inline_keyboard_array.append([{
-                        'text': self.user.translated('openTrackInApp_button'),
-                        'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}/content?startapp={content.cid.serialize_v2()}"
-                    }])
-                else:
-                    inline_keyboard_array.append([{
-                        'text': self.user.translated('viewTrackAsClient_button'),
-                        'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}?start=C{content.cid.serialize_v2()}"
-                    }])
-                    inline_keyboard_array.append([{
-                        'text': self.user.translated('openContractPage_button'),
-                        'url': f"https://tonviewer.com/{content_meta['item_address']}"
-                    }])
-            elif content_type == 'video':
-                # Processing video
-                video_title = content_metadata_json.get('name', "")
-                template_kwargs['video'] = URLInputFile(local_content_preview_url)
-                template_kwargs['protect_content'] = True
-
-                if cover_content:
-                    # Add thumbnail if cover content is available
-                    template_kwargs['thumbnail'] = URLInputFile(cover_content.web_url)
-
-                if self.bot_id == 1:
-                    # Buttons for sharing and opening in app
-                    inline_keyboard_array.append([
-                        {
-                            'text': self.user.translated('shareVideo_button'),
-                            'switch_inline_query': f"Q{user_existing_license.onchain_address}" if user_existing_license else f"C{content.cid.serialize_v2()}",
-                        },
-                        {
-                            'text': self.user.translated('shareLink_button'),
-                            'url': f"https://t.me/share/url?text={urllib.parse.quote(content_share_link['text'])}&url={urllib.parse.quote(content_share_link['url'])}"
-                        }
-                    ])
-                    inline_keyboard_array.append([{
-                        'text': self.user.translated('openTrackInApp_button'),
-                        'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}/content?startapp={content.cid.serialize_v2()}"
-                    }])
-                else:
-                    # Buttons for viewing as a client and opening contract
-                    inline_keyboard_array.append([{
-                        'text': self.user.translated('viewTrackAsClient_button'),
-                        'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}?start=C{content.cid.serialize_v2()}"
-                    }])
-                    inline_keyboard_array.append([{
-                        'text': self.user.translated('openContractPage_button'),
-                        'url': f"https://tonviewer.com/address/{content_meta['item_address']}"
-                    }])
-            else:
-                local_content = None
+            if cover_content:
+                template_kwargs['photo'] = URLInputFile(cover_content.web_url)
 
         if not local_content:
             text = self.user.translated('p_playerContext_unsupportedContent').format(
@@ -169,37 +77,26 @@ class PlayerTemplates:
             inline_keyboard_array = []
             extra_buttons = []
         else:
-            text = content_metadata_json.get('description').strip()
+            content_hashtags = content_metadata_json.get('description').strip()
+            if content_hashtags:
+                content_hashtags += '\n'
 
-            have_access = (
-                    (content.owner_address == user_wallet_address)
-                    or bool(user_existing_license)
-                    or bool(self.db_session.query(StarsInvoice).filter(
-                        and_(
-                            StarsInvoice.user_id == self.user.id,
-                            StarsInvoice.content_hash == content.hash,
-                            StarsInvoice.paid == True
-                        )
-                ).first())
-            )
-            if have_access:
-                full_content = self.db_session.query(StoredContent).filter_by(
-                    hash=content.meta.get('converted_content', {}).get('low')  # TODO: support high quality
-                ).first()
-                if content_type == 'audio':
-                    # Restrict audio to 30 seconds if user does not have access
-                    template_kwargs['audio'] = URLInputFile(full_content.web_url)
-                elif content_type == 'video':
-                    # Restrict video to 30 seconds if user does not have access
-                    template_kwargs['video'] = URLInputFile(full_content.web_url)
+            text = f"""<b>{content_metadata_json.get('name', 'Unnamed')}</b>
+{content_hashtags}
+Этот контент был загружен в MY
+\t/ p2p content market /
+<blockquote><a href="{content_share_link['url']}">🔴 «открыть в MY»</a></blockquote>"""
 
         make_log("TG-Player", f"Send content {content_type} ({content_encoding}) to chat {self._chat_id}. {cd_log}")
-        for kmsg in self.db_session.query(KnownTelegramMessage).filter_by(
-            content_id=content.id,
-            chat_id=self._chat_id,
-            type=f'content/{content_type}',
-            deleted=False
-        ).all():
+        kmsgs = (await self.db_session.execute(select(KnownTelegramMessage).where(
+            and_(
+                KnownTelegramMessage.content_id == content.id,
+                KnownTelegramMessage.chat_id == self._chat_id,
+                KnownTelegramMessage.type == f'content/{content_type}',
+                KnownTelegramMessage.deleted == False
+            )
+        ))).scalars().all()
+        for kmsg in kmsgs:
             await self.delete_message(kmsg.message_id)
 
         r = await tg_process_template(
@@ -210,19 +107,4 @@ class PlayerTemplates:
             content_id=content.id if content else None
         )
 
-        if self.bot_id == 1:
-            if content.type == 'onchain/content':
-                if content_type == 'audio':
-                    content.meta = {
-                        **content.meta,
-                        'telegram_file_cache': r.audio.file_id,
-                    }
-                elif content_type == 'video':
-                    content.meta = {
-                        **content.meta,
-                        'telegram_file_cache': r.video.file_id,
-                    }
-
-                self.db_session.commit()
-
         return r

app/core/models/_telegram/wrapped_bot.py

@@ -1,7 +1,7 @@
 from aiogram import Bot, types
 from datetime import datetime, timedelta
 
-from sqlalchemy import and_
+from sqlalchemy import and_, select
 
 from app.core.logger import make_log
 from app.core.models.messages import KnownTelegramMessage
@@ -46,14 +46,15 @@ class Wrapped_CBotChat(T, PlayerTemplates):
         if self.db_session:
             if message_type == 'common':
                 ci = 0
-                for oc_msg in self.db_session.query(KnownTelegramMessage).filter(
+                result = await self.db_session.execute(select(KnownTelegramMessage).where(
                     and_(
                         KnownTelegramMessage.type == 'common',
                         KnownTelegramMessage.bot_id == self.bot_id,
                         KnownTelegramMessage.chat_id == self._chat_id,
                         KnownTelegramMessage.deleted == False
                     )
-                ).all():
+                ))
+                for oc_msg in result.scalars().all():
                     make_log(self, f"Delete old message {oc_msg.message_id} {oc_msg.type} {oc_msg.bot_id} {oc_msg.chat_id}")
                     await self.delete_message(oc_msg.message_id)
                     ci += 1
@@ -75,7 +76,7 @@ class Wrapped_CBotChat(T, PlayerTemplates):
                         content_id=content_id
                     )
                 )
-                self.db_session.commit()
+                await self.db_session.commit()
             else:
                 make_log(self, f"Unknown result type: {type(result)}", level='warning')
 
@@ -127,14 +128,16 @@ class Wrapped_CBotChat(T, PlayerTemplates):
                 message_id
             )):
                 if self.db_session:
-                    known_message = self.db_session.query(KnownTelegramMessage).filter(
-                        KnownTelegramMessage.bot_id == self.bot_id,
-                        KnownTelegramMessage.chat_id == self._chat_id,
-                        KnownTelegramMessage.message_id == message_id
-                    ).first()
+                    known_message = (await self.db_session.execute(select(KnownTelegramMessage).where(
+                        and_(
+                            KnownTelegramMessage.bot_id == self.bot_id,
+                            KnownTelegramMessage.chat_id == self._chat_id,
+                            KnownTelegramMessage.message_id == message_id
+                        )
+                    ))).scalars().first()
                     if known_message:
                         known_message.deleted = True
-                        self.db_session.commit()
+                        await self.db_session.commit()
         except Exception as e:
             make_log(self, f"Error deleting message {self._chat_id}/{message_id}. Error: {e}", level='warning')
             return None

app/core/models/asset.py

@@ -1,5 +1,6 @@
 from sqlalchemy import Column, Integer, String, DateTime, JSON, Boolean
 from sqlalchemy.orm import relationship
+from datetime import datetime
 
 from app.core._defaults import DEFAULT_ASSET_INITOBJ
 from app.core.models.base import AlchemyBase
@@ -15,10 +16,10 @@ class Asset(AlchemyBase):
 
     network = Column(String(32), nullable=True)
     address = Column(String(1024), nullable=True)
-    meta = Column(JSON, nullable=False, default={})
-    rates = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
+    rates = Column(JSON, nullable=False, default=dict)
 
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
     is_active = Column(Boolean, nullable=False, default=True)
 
     balances = relationship('UserBalance', back_populates='asset')
@@ -29,22 +30,21 @@ class Asset(AlchemyBase):
         AlchemyBase.metadata.create_all(engine)
 
     @classmethod
-    def find(cls, session, **kwargs):
+    async def find_async(cls, session, **kwargs):
+        from sqlalchemy import select, func
         if 'symbol' in kwargs:
             kwargs['symbol'] = kwargs['symbol'].upper()
 
-        result = session.query(cls).filter_by(**kwargs)
-        results_count = result.count()
-        if results_count == 0:
-            any_count = session.query(cls).count()
-            if any_count == 0:
-                init_asset = cls(**DEFAULT_ASSET_INITOBJ)
-                session.add(init_asset)
-                session.commit()
-                return cls.find(session, **kwargs)
+        result = await session.execute(select(cls).filter_by(**kwargs))
+        row = result.scalars().first()
+        if row:
+            return row
 
-            raise Exception(f"Asset not found: {kwargs}")
-        elif results_count == 1:
-            return result.first()
-        else:
-            raise Exception(f"Multiple assets found: {results_count}")
+        any_count = (await session.execute(select(func.count()).select_from(cls))).scalar() or 0
+        if any_count == 0:
+            init_asset = cls(**DEFAULT_ASSET_INITOBJ)
+            session.add(init_asset)
+            await session.commit()
+            return await cls.find_async(session, **kwargs)
+
+        raise Exception(f"Asset not found: {kwargs}")

app/core/models/content/indexation_mixins.py

@@ -1,7 +1,7 @@
 import traceback
 
 import base58
-from sqlalchemy import and_
+from sqlalchemy import and_, select
 
 from app.core.logger import make_log
 from app.core.models import StoredContent
@@ -57,13 +57,9 @@ class UserContentIndexationMixin:
                 values_slice = cc_indexator_data['values'].begin_parse()
                 content_hash_b58 = base58.b58encode(bytes.fromhex(hex(values_slice.read_uint(256))[2:])).decode()
                 make_log("UserContent", f"License ({self.onchain_address}) content hash: {content_hash_b58}", level="info")
-                stored_content = db_session.query(StoredContent).filter(
-                    and_(
-                        StoredContent.type == 'onchain/content',
-                        StoredContent.hash == content_hash_b58,
-
-                    )
-                ).first()
+                stored_content = (await db_session.execute(select(StoredContent).where(
+                    and_(StoredContent.type == 'onchain/content', StoredContent.hash == content_hash_b58)
+                ))).scalars().first()
                 trusted_cop_address_result = await toncenter.run_get_method(stored_content.meta['item_address'], 'get_nft_address_by_index', [['num', cc_indexator_data['index']]])
                 assert trusted_cop_address_result.get('exit_code', -1) == 0, "Trusted cop address error"
                 trusted_cop_address = Cell.one_from_boc(b64decode(trusted_cop_address_result['stack'][0][1]['bytes'])).begin_parse().read_msg_addr().to_string(1, 1, 1)
@@ -72,7 +68,7 @@ class UserContentIndexationMixin:
                 self.owner_address = cc_indexator_data['owner_address']
                 self.type = 'nft/listen'
                 self.content_id = stored_content.id
-                db_session.commit()
+                await db_session.commit()
             except BaseException as e:
                 errored = True
                 make_log("UserContent", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
@@ -80,7 +76,6 @@ class UserContentIndexationMixin:
         if errored is True:
             self.type = 'nft/unknown'
             self.content_id = None
-            db_session.commit()
-
+            await db_session.commit()
 
 

app/core/models/content/user_content.py

@@ -3,6 +3,7 @@ from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime
 from sqlalchemy.orm import relationship
 from app.core.models.base import AlchemyBase
 from app.core.models.content.indexation_mixins import UserContentIndexationMixin
+from datetime import datetime
 
 
 class UserContent(AlchemyBase, UserContentIndexationMixin):
@@ -14,12 +15,12 @@ class UserContent(AlchemyBase, UserContentIndexationMixin):
     owner_address = Column(String(1024), nullable=True)
     code_hash = Column(String(128), nullable=True)
     data_hash = Column(String(128), nullable=True)
-    updated = Column(DateTime, nullable=False, default=0)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
 
     content_id = Column(Integer, ForeignKey('node_storage.id'), nullable=True)
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
     user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
     wallet_connection_id = Column(Integer, ForeignKey('wallet_connections.id'), nullable=True)
     status = Column(String(64), nullable=False, default='active')  # 'transaction_requested'
@@ -41,9 +42,8 @@ class UserAction(AlchemyBase):
     to_address = Column(String(1024), nullable=True)
     from_address = Column(String(1024), nullable=True)
     status = Column(String(128), nullable=True)
-    meta = Column(JSON, nullable=False, default={})
-    created = Column(DateTime, nullable=False, default=0)
+    meta = Column(JSON, nullable=False, default=dict)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     user = relationship('User', uselist=False, foreign_keys=[user_id])
     content = relationship('StoredContent', uselist=False, foreign_keys=[content_id])
-

app/core/models/content_v3.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+from datetime import datetime
+from sqlalchemy import Column, BigInteger, Integer, String, DateTime, JSON, Boolean, ForeignKey
+from sqlalchemy.orm import relationship
+
+from .base import AlchemyBase
+
+
+class EncryptedContent(AlchemyBase):
+    __tablename__ = 'encrypted_contents'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    # CID of encrypted source stored in IPFS (CIDv1 base32)
+    encrypted_cid = Column(String(128), nullable=False, unique=True)
+
+    # Public metadata
+    title = Column(String(512), nullable=False)
+    description = Column(String(4096), nullable=True)
+    content_type = Column(String(64), nullable=False)  # e.g. audio/flac, video/mp4, application/octet-stream
+
+    # Sizes
+    enc_size_bytes = Column(BigInteger, nullable=True)
+    plain_size_bytes = Column(BigInteger, nullable=True)
+
+    # Preview flags and config (all preview params live here, not in derivatives)
+    preview_enabled = Column(Boolean, nullable=False, default=False)
+    preview_conf = Column(JSON, nullable=False, default=dict)
+
+    # Crypto parameters (fixed per network)
+    aead_scheme = Column(String(32), nullable=False, default='AES_GCM')
+    chunk_bytes = Column(Integer, nullable=False, default=1048576)
+    salt_b64 = Column(String(64), nullable=True)  # per-content salt used for nonce derivation
+
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class ContentKey(AlchemyBase):
+    __tablename__ = 'content_keys'
+
+    content_id = Column(Integer, ForeignKey('encrypted_contents.id'), primary_key=True)
+    key_ciphertext_b64 = Column(String(512), nullable=False)
+    key_fingerprint = Column(String(128), nullable=False)
+    issuer_node_id = Column(String(128), nullable=False)
+    allow_auto_grant = Column(Boolean, nullable=False, default=True)
+    lease_expires_at = Column(DateTime, nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+
+    content = relationship('EncryptedContent', uselist=False, foreign_keys=[content_id])
+
+
+class IpfsSync(AlchemyBase):
+    __tablename__ = 'ipfs_sync'
+
+    content_id = Column(Integer, ForeignKey('encrypted_contents.id'), primary_key=True)
+    pin_state = Column(String(32), nullable=False, default='pinned')  # not_pinned|queued|pinning|pinned|failed
+    pin_error = Column(String(1024), nullable=True)
+    bytes_total = Column(BigInteger, nullable=True)
+    bytes_fetched = Column(BigInteger, nullable=True)
+    providers_cache = Column(JSON, nullable=False, default=list)
+    first_seen_at = Column(DateTime, nullable=True)
+    pinned_at = Column(DateTime, nullable=True)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    content = relationship('EncryptedContent', uselist=False, foreign_keys=[content_id])
+
+
+class ContentDerivative(AlchemyBase):
+    __tablename__ = 'content_derivatives'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    content_id = Column(Integer, ForeignKey('encrypted_contents.id'), nullable=False)
+    kind = Column(String(64), nullable=False)  # decrypted_high|decrypted_low|decrypted_thumbnail|decrypted_preview
+    interval_start_ms = Column(Integer, nullable=True)
+    interval_end_ms = Column(Integer, nullable=True)
+    local_path = Column(String(1024), nullable=False)
+    content_type = Column(String(64), nullable=True)
+    size_bytes = Column(BigInteger, nullable=True)
+    status = Column(String(32), nullable=False, default='pending')  # pending|processing|ready|failed
+    error = Column(String(1024), nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    last_access_at = Column(DateTime, nullable=True)
+
+    content = relationship('EncryptedContent', uselist=False, foreign_keys=[content_id])
+
+
+class ContentIndexItem(AlchemyBase):
+    __tablename__ = 'content_index_items'
+
+    encrypted_cid = Column(String(128), primary_key=True)
+    payload = Column(JSON, nullable=False, default=dict)
+    sig = Column(String(512), nullable=False)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+
+class KeyGrant(AlchemyBase):
+    __tablename__ = 'key_grants'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    encrypted_cid = Column(String(128), nullable=False)
+    issuer_node_id = Column(String(128), nullable=False)
+    to_node_id = Column(String(128), nullable=False)
+    sealed_key_b64 = Column(String(1024), nullable=False)
+    aead_scheme = Column(String(32), nullable=False)
+    chunk_bytes = Column(Integer, nullable=False)
+    constraints = Column(JSON, nullable=False, default=dict)
+    issued_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    sig = Column(String(512), nullable=False)
+
+
+class UploadSession(AlchemyBase):
+    __tablename__ = 'upload_sessions'
+
+    id = Column(String(128), primary_key=True)  # tus Upload.ID
+    filename = Column(String(512), nullable=True)
+    size_bytes = Column(BigInteger, nullable=True)
+    state = Column(String(32), nullable=False, default='uploading')  # uploading|processing|pinned|failed
+    encrypted_cid = Column(String(128), nullable=True)
+    storage_path = Column(String(1024), nullable=True)
+    error = Column(String(1024), nullable=True)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)

app/core/models/keys.py

@@ -1,5 +1,6 @@
 from base58 import b58decode
 from sqlalchemy import Column, Integer, String, DateTime, JSON
+from datetime import datetime
 
 from .base import AlchemyBase
 
@@ -15,12 +16,12 @@ class KnownKey(AlchemyBase):
     public_key_hash = Column(String(64), nullable=False, unique=True)  # base58
 
     algo = Column(String(32), nullable=True, default=None)
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
     # {
     #   "I_user_id": TRUSTED_USER_ID,
     # }
 
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     # stored_content = relationship('StoredContent', back_populates='key')
 

app/core/models/memory.py

@@ -42,6 +42,10 @@ class Memory:
         self._telegram_bot = Bot(TELEGRAM_API_KEY)
         self._client_telegram_bot = Bot(CLIENT_TELEGRAM_API_KEY)
 
+        # Network handshake guards
+        self._handshake_rl = {"minute": 0, "counts": {}}
+        self._handshake_nonces = {}
+
     @asynccontextmanager
     async def transaction(self, desc=""):
         make_log("Memory.transaction", f"Starting transaction; {desc}", level='debug')
@@ -77,4 +81,3 @@ class Memory:
 
             self._execute_queue.append([_fn, args, kwargs])
 
-

app/core/models/my_network.py

@@ -12,9 +12,9 @@ class KnownNode(AlchemyBase):
     public_key = Column(String(256), nullable=False)
     codebase_hash = Column(String(512), nullable=True)  # Node software version
     reputation = Column(Integer, nullable=False, default=0)
-    last_sync = Column(DateTime, nullable=False, default=datetime.now)
-    meta = Column(JSON, nullable=False, default={})
-    located_at = Column(DateTime, nullable=False, default=datetime.now)
+    last_sync = Column(DateTime, nullable=False, default=datetime.utcnow)
+    meta = Column(JSON, nullable=False, default=dict)
+    located_at = Column(DateTime, nullable=False, default=datetime.utcnow)
 
 
 class KnownNodeIncident(AlchemyBase):
@@ -28,7 +28,7 @@ class KnownNodeIncident(AlchemyBase):
     severity = Column(Integer, nullable=False, default=1)  # Severity level (1-low to 5-critical)
     resolved = Column(Boolean, nullable=False, default=False)  # Whether the incident has been resolved
     resolved_at = Column(DateTime, nullable=True)  # Timestamp when the incident was resolved
-    meta = Column(JSON, nullable=False, default={})  # Additional metadata if needed
+    meta = Column(JSON, nullable=False, default=dict)  # Additional metadata if needed
 
 
 class RemoteContentIndex(AlchemyBase):
@@ -41,7 +41,6 @@ class RemoteContentIndex(AlchemyBase):
     decrypted_hash = Column(String(128), nullable=True)  # Decrypted content hash, available once permission is granted
     ton_address = Column(String(128), nullable=True)  # TON network address for the content
     onchain_index = Column(Integer, nullable=True)  # Onchain index or reference on a blockchain
-    meta = Column(JSON, nullable=False, default={})  # Additional metadata for flexible content description
+    meta = Column(JSON, nullable=False, default=dict)  # Additional metadata for flexible content description
     last_updated = Column(DateTime, nullable=False, default=datetime.utcnow)  # Timestamp of the last update
     created_at = Column(DateTime, nullable=False, default=datetime.utcnow)  # Record creation timestamp
-

app/core/models/node_storage.py

@@ -25,7 +25,8 @@ class StoredContent(AlchemyBase, AudioContentMixin):
 
     status = Column(String(32), nullable=True)
     filename = Column(String(1024), nullable=False)
-    meta = Column(JSON, nullable=False, default={})
+    # Use a factory for JSON default to avoid shared mutable dict
+    meta = Column(JSON, nullable=False, default=dict)
 
     user_id = Column(Integer, ForeignKey('users.id'), nullable=True)
     owner_address = Column(String(1024), nullable=True)
@@ -35,9 +36,11 @@ class StoredContent(AlchemyBase, AudioContentMixin):
     telegram_cid = Column(String(1024), nullable=True)
 
     codebase_version = Column(Integer, nullable=True)
-    created = Column(DateTime, nullable=False, default=0)
-    updated = Column(DateTime, nullable=False, default=0)
-    disabled = Column(DateTime, nullable=False, default=0)
+    # Use proper datetime defaults; updated also auto-updates on change
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+    # Timestamp of when content was disabled; None means active
+    disabled = Column(DateTime, nullable=True, default=None)
     disabled_by = Column(Integer, ForeignKey('users.id'), nullable=True, default=None)
 
     encrypted = Column(Boolean, nullable=False, default=False)
@@ -96,6 +99,30 @@ class StoredContent(AlchemyBase, AudioContentMixin):
             make_log("NodeStorage.open_content", f"Can't open content: {self.id} {e}", level='warning')
             raise e
 
+    async def open_content_async(self, db_session, content_type=None):
+        from sqlalchemy import select
+        try:
+            decrypted_content = self if not self.encrypted else None
+            encrypted_content = self if self.encrypted else None
+            if not decrypted_content:
+                decrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == self.decrypted_content_id))).scalars().first()
+            else:
+                encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.decrypted_content_id == self.id))).scalars().first()
+
+            assert decrypted_content, "Can't get decrypted content"
+            assert encrypted_content, "Can't get encrypted content"
+            _ct = content_type or decrypted_content.json_format()['content_type']
+            content_type = _ct.split('/')[0] if _ct else 'application'
+
+            return {
+                'encrypted_content': encrypted_content,
+                'decrypted_content': decrypted_content,
+                'content_type': content_type or 'application/x-binary'
+            }
+        except BaseException as e:
+            make_log("NodeStorage.open_content_async", f"Can't open content: {self.id} {e}", level='warning')
+            raise e
+
     def json_format(self):
         extra_fields = {}
         if self.type.startswith('local'):
@@ -144,6 +171,16 @@ class StoredContent(AlchemyBase, AudioContentMixin):
             with open(metadata_content.filepath, 'r') as f:
                 return json.loads(f.read())
 
+    async def metadata_json_async(self, db_session):
+        metadata_cid = self.meta.get('metadata_cid')
+        if not metadata_cid:
+            return None
+        metadata_content = await StoredContent.from_cid_async(db_session, metadata_cid)
+        import aiofiles
+        async with aiofiles.open(metadata_content.filepath, 'r') as f:
+            data = await f.read()
+        return json.loads(data)
+
     @classmethod
     def from_cid(cls, db_session, content_id):
         if isinstance(content_id, str):
@@ -155,3 +192,15 @@ class StoredContent(AlchemyBase, AudioContentMixin):
         assert content, "Content not found"
         return content
 
+    @classmethod
+    async def from_cid_async(cls, db_session, content_id):
+        from sqlalchemy import select
+        if isinstance(content_id, str):
+            cid = ContentId.deserialize(content_id)
+        else:
+            cid = content_id
+
+        result = await db_session.execute(select(StoredContent).where(StoredContent.hash == cid.content_hash_b58))
+        content = result.scalars().first()
+        assert content, "Content not found"
+        return content

app/core/models/promo.py

@@ -0,0 +1,16 @@
+from .base import AlchemyBase
+from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
+from datetime import datetime
+
+
+class PromoAction(AlchemyBase):
+    __tablename__ = 'promo_actions'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    user_id = Column(String(512), nullable=False)
+    user_internal_id = Column(Integer, ForeignKey('users.id'), nullable=True)
+
+    action_type = Column(String(64), nullable=False)  # Type of action, e.g., 'referral', 'discount'
+    action_ref = Column(String(512), nullable=False)  # Reference to the action, e.g., promo code
+
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)

app/core/models/tasks.py

@@ -0,0 +1,26 @@
+from .base import AlchemyBase
+from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
+from datetime import datetime
+
+
+class BlockchainTask(AlchemyBase):
+    __tablename__ = 'blockchain_tasks'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    
+    destination = Column(String(1024), nullable=False)
+    amount = Column(String(256), nullable=False)
+    payload = Column(String(4096), nullable=False)
+
+    epoch = Column(Integer, nullable=True)
+    seqno = Column(Integer, nullable=True)
+
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    user_id = Column(Integer, ForeignKey('users.id'), nullable=True)
+    meta = Column(JSON, nullable=False, default=dict)
+    status = Column(String(256), nullable=False)
+
+    transaction_hash = Column(String(1024), nullable=True)
+    transaction_lt = Column(String(1024), nullable=True)

app/core/models/transaction.py

@@ -13,8 +13,8 @@ class UserBalance(AlchemyBase):
     asset_id = Column(Integer, ForeignKey('assets.id'), nullable=False)
     balance = Column(Float, nullable=False, default=0)
 
-    updated = Column(DateTime, nullable=False, default=0)
-    created = Column(DateTime, nullable=False, default=0)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     user = relationship('User', uselist=False, foreign_keys=[user_id], back_populates='balances')
     asset = relationship('Asset', uselist=False, foreign_keys=[asset_id], back_populates='balances')
@@ -32,7 +32,7 @@ class InternalTransaction(AlchemyBase):
     spent_transaction_id = Column(Integer, ForeignKey('internal_transactions.id'), nullable=True)
     type = Column(String(256), nullable=False, default="NOT_SPECIFIED")
 
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     user = relationship('User', uselist=False, back_populates='internal_transactions', foreign_keys=[user_id])
     asset = relationship('Asset', uselist=False, foreign_keys=[asset_id])

app/core/models/user/__init__.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from sqlalchemy import Column, Integer, String, BigInteger, DateTime, JSON
 from sqlalchemy.orm import relationship
 
@@ -17,10 +18,11 @@ class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1, W
 
     username = Column(String(512), nullable=True)
     lang_code = Column(String(8), nullable=False, default="en")
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
 
-    last_use = Column(DateTime, nullable=False, default=0)
-    created = Column(DateTime, nullable=False, default=0)
+    last_use = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     balances = relationship('UserBalance', back_populates='user')
     internal_transactions = relationship('InternalTransaction', back_populates='user')
@@ -30,4 +32,3 @@ class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1, W
     def __str__(self):
         return f"User, {self.id}_{self.telegram_id} | Username: {self.username} " + '\\'
 
-

app/core/models/user/wallet_mixin.py

@@ -10,18 +10,21 @@ from httpx import AsyncClient
 
 
 class WalletMixin:
-    def wallet_connection(self, db_session):
-        return db_session.query(WalletConnection).filter(
-            WalletConnection.user_id == self.id,
-            WalletConnection.invalidated == False
-        ).order_by(WalletConnection.created.desc()).first()
+    async def wallet_connection_async(self, db_session):
+        from sqlalchemy import select, and_, desc
+        result = await db_session.execute(
+            select(WalletConnection)
+            .where(and_(WalletConnection.user_id == self.id, WalletConnection.invalidated == False))
+            .order_by(WalletConnection.created.desc())
+        )
+        return result.scalars().first()
 
-    def wallet_address(self, db_session):
-        wallet_connection = self.wallet_connection(db_session)
-        return wallet_connection.wallet_address if wallet_connection else None
+    async def wallet_address_async(self, db_session):
+        wc = await self.wallet_connection_async(db_session)
+        return wc.wallet_address if wc else None
 
     async def scan_owned_user_content(self, db_session):
-        user_wallet_address = self.wallet_address(db_session)
+        user_wallet_address = await self.wallet_address_async(db_session)
 
         async def get_nft_items_list():
             try:
@@ -40,9 +43,8 @@ class WalletMixin:
             item_address = Address(nft_item['address']).to_string(1, 1, 1)
             owner_address = Address(nft_item['owner']['address']).to_string(1, 1, 1)
 
-            user_content = db_session.query(UserContent).filter(
-                UserContent.onchain_address == item_address
-            ).first()
+            from sqlalchemy import select
+            user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
             if user_content:
                 continue
 
@@ -57,18 +59,18 @@ class WalletMixin:
                 created=datetime.now(),
                 meta={},
                 user_id=self.id,
-                wallet_connection_id=self.wallet_connection(db_session).id,
+                wallet_connection_id=(await self.wallet_connection_async(db_session)).id,
                 status="active"
             )
             db_session.add(user_content)
-            db_session.commit()
+            await db_session.commit()
             make_log(self, f"New onchain NFT found: {item_address}", level='info')
 
     async def ____scan_owned_user_content(self, db_session):
         page_id = -1
         page_size = 100
         have_next_page = True
-        user_wallet_address = self.wallet_address(db_session)
+        user_wallet_address = await self.wallet_address_async(db_session)
         while have_next_page:
             page_id += 1
             nfts_list = await toncenter.get_nft_items(limit=100, offset=page_id * page_size, owner_address=user_wallet_address)
@@ -81,9 +83,8 @@ class WalletMixin:
                     item_address = Address(nft_item['address']).to_string(1, 1, 1)
                     owner_address = Address(nft_item['owner_address']).to_string(1, 1, 1)
 
-                    user_content = db_session.query(UserContent).filter(
-                        UserContent.onchain_address == item_address
-                    ).first()
+                    from sqlalchemy import select
+                    user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
                     if user_content:
                         continue
 
@@ -105,11 +106,11 @@ class WalletMixin:
                             'metadata_uri': nft_content,
                         },
                         user_id=self.id,
-                        wallet_connection_id=self.wallet_connection(db_session).id,
+                        wallet_connection_id=(await self.wallet_connection_async(db_session)).id,
                         status="active"
                     )
                     db_session.add(user_content)
-                    db_session.commit()
+                    await db_session.commit()
 
                     make_log(self, f"New onchain NFT found: {item_address}", level='info')
                 except BaseException as e:
@@ -122,6 +123,6 @@ class WalletMixin:
         except BaseException as e:
             make_log(self, f"Error while scanning user content: {e}", level='error')
 
-        return self.db_session.query(UserContent).filter(
-            UserContent.user_id == self.id
-        ).offset(offset).limit(limit).all()
+        from sqlalchemy import select
+        result = await db_session.execute(select(UserContent).where(UserContent.user_id == self.id).offset(offset).limit(limit))
+        return result.scalars().all()

app/core/models/user_activity.py

@@ -2,6 +2,7 @@
 from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
 from sqlalchemy.orm import relationship
 from .base import AlchemyBase
+from datetime import datetime
 
 
 class UserActivity(AlchemyBase):
@@ -9,10 +10,10 @@ class UserActivity(AlchemyBase):
 
     id = Column(Integer, autoincrement=True, primary_key=True)
     type = Column(String(64), nullable=False)
-    meta = Column(JSON, nullable=False, default={})
+    meta = Column(JSON, nullable=False, default=dict)
 
     user_id = Column(Integer, ForeignKey('users.id'), nullable=True)
     user_ip = Column(String(64), nullable=True)
-    created = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
 
     user = relationship('User', uselist=False, foreign_keys=[user_id])

app/core/models/wallet_connection.py

@@ -1,5 +1,6 @@
 from sqlalchemy import Column, Integer, String, ForeignKey, DateTime, JSON, Boolean
 from sqlalchemy.orm import relationship
+from datetime import datetime
 
 from .base import AlchemyBase
 
@@ -15,11 +16,11 @@ class WalletConnection(AlchemyBase):
 
     wallet_address = Column(String(1024), nullable=False)
 
-    keys = Column(JSON, nullable=False, default={})
-    meta = Column(JSON, nullable=False, default={})
+    keys = Column(JSON, nullable=False, default=dict)
+    meta = Column(JSON, nullable=False, default=dict)
 
-    created = Column(DateTime, nullable=False, default=0)
-    updated = Column(DateTime, nullable=False, default=0)
+    created = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)
     invalidated = Column(Boolean, nullable=False, default=True)
     without_pk = Column(Boolean, nullable=False, default=False)
 

app/core/network/__init__.py

@@ -0,0 +1,2 @@
+# Network package for MY nodes
+

app/core/network/config.py

@@ -0,0 +1,38 @@
+import os
+from typing import List
+
+from app.core._config import PROJECT_HOST
+from .constants import NODE_TYPE_PUBLIC, NODE_TYPE_PRIVATE
+
+
+def _csv_list(val: str) -> List[str]:
+    return [x.strip() for x in (val or "").split(",") if x.strip()]
+
+
+# Handshake / network config driven by env
+NODE_PRIVACY = os.getenv("NODE_PRIVACY", NODE_TYPE_PUBLIC).strip().lower()
+if NODE_PRIVACY not in (NODE_TYPE_PUBLIC, NODE_TYPE_PRIVATE):
+    NODE_PRIVACY = NODE_TYPE_PUBLIC
+
+# Public endpoint for network (can be empty for private nodes)
+_env_public_host = os.getenv("PUBLIC_HOST")
+PUBLIC_HOST = _env_public_host if (_env_public_host is not None and _env_public_host.strip() != "") else None
+
+HANDSHAKE_INTERVAL_SEC = int(os.getenv("HANDSHAKE_INTERVAL_SEC", "5"))
+UNSUPPORTED_RECHECK_INTERVAL_SEC = int(os.getenv("UNSUPPORTED_RECHECK_INTERVAL_SEC", str(24 * 3600)))
+
+BOOTSTRAP_SEEDS = _csv_list(os.getenv("BOOTSTRAP_SEEDS", ""))
+BOOTSTRAP_REQUIRED = int(os.getenv("BOOTSTRAP_REQUIRED", "1")) == 1
+BOOTSTRAP_TIMEOUT_SEC = int(os.getenv("BOOTSTRAP_TIMEOUT_SEC", "20"))
+
+# Security knobs
+NETWORK_TLS_VERIFY = int(os.getenv("NETWORK_TLS_VERIFY", "1")) == 1
+HANDSHAKE_TS_TOLERANCE_SEC = int(os.getenv("HANDSHAKE_TS_TOLERANCE_SEC", "300"))
+HANDSHAKE_RATE_LIMIT_PER_MIN = int(os.getenv("HANDSHAKE_RATE_LIMIT_PER_MIN", "60"))
+
+# Capabilities
+NODE_IS_BOOTSTRAP = int(os.getenv("NODE_IS_BOOTSTRAP", "0")) == 1
+MAX_CONTENT_SIZE_MB = int(os.getenv("MAX_CONTENT_SIZE_MB", "512"))
+
+# Privacy allowlist (for NODE_PRIVACY=private)
+PRIVATE_ALLOWLIST = _csv_list(os.getenv("PRIVATE_ALLOWLIST", "/api/system.version"))

app/core/network/constants.py

@@ -0,0 +1,6 @@
+CURRENT_PROTOCOL_VERSION = "3.0.0"
+
+# Node roles/types
+NODE_TYPE_PUBLIC = "public"
+NODE_TYPE_PRIVATE = "private"
+

app/core/network/guard.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import time
+from typing import Dict, Set
+
+from app.core.network.config import HANDSHAKE_RATE_LIMIT_PER_MIN, HANDSHAKE_TS_TOLERANCE_SEC
+
+
+def check_rate_limit(memory, remote_ip: str) -> bool:
+    """Simple per-IP rate limit within current minute window.
+    Returns True if allowed, False if limited.
+    """
+    now = int(time.time())
+    minute = now // 60
+    rl = getattr(memory, "_handshake_rl", None)
+    if rl is None or rl.get("minute") != minute:
+        rl = {"minute": minute, "counts": {}}
+        memory._handshake_rl = rl
+    counts = rl["counts"]
+    cnt = counts.get(remote_ip, 0)
+    if cnt >= HANDSHAKE_RATE_LIMIT_PER_MIN:
+        return False
+    counts[remote_ip] = cnt + 1
+    return True
+
+
+def check_timestamp_fresh(ts: int) -> bool:
+    now = int(time.time())
+    return abs(now - int(ts)) <= HANDSHAKE_TS_TOLERANCE_SEC
+
+
+def check_and_remember_nonce(memory, pubkey_b58: str, nonce: str) -> bool:
+    """Return True if nonce is new; remember nonce with TTL ~ tolerance window.
+    We keep a compact in-memory set per pubkey.
+    """
+    now = int(time.time())
+    store = getattr(memory, "_handshake_nonces", None)
+    if store is None:
+        store = {}
+        memory._handshake_nonces = store
+
+    entry = store.get(pubkey_b58)
+    if entry is None:
+        entry = {"nonces": {}, "updated": now}
+        store[pubkey_b58] = entry
+
+    nonces: Dict[str, int] = entry["nonces"]
+    # prune old nonces
+    to_delete = [k for k, t in nonces.items() if now - int(t) > HANDSHAKE_TS_TOLERANCE_SEC]
+    for k in to_delete:
+        nonces.pop(k, None)
+
+    if nonce in nonces:
+        return False
+    # prevent unbounded growth
+    if len(nonces) > 2048:
+        # drop half oldest
+        for k, _ in sorted(nonces.items(), key=lambda kv: kv[1])[:1024]:
+            nonces.pop(k, None)
+    nonces[nonce] = now
+    entry["updated"] = now
+    return True
+

app/core/network/handshake.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+import os
+import time
+import shutil
+import secrets
+from typing import Dict, Any
+
+from base58 import b58encode
+from sqlalchemy import select
+
+from app.core._secrets import hot_pubkey, hot_seed
+from app.core._crypto.signer import Signer
+from app.core.logger import make_log
+from app.core.models.my_network import KnownNode
+from app.core.models.node_storage import StoredContent
+from app.core.storage import db_session
+from .constants import CURRENT_PROTOCOL_VERSION
+from .nodes import list_known_public_nodes
+from .config import PUBLIC_HOST, NODE_PRIVACY, NODE_IS_BOOTSTRAP, MAX_CONTENT_SIZE_MB
+from app.core._config import ALLOWED_CONTENT_TYPES
+from .constants import NODE_TYPE_PUBLIC
+
+
+START_TS = time.time()
+
+
+async def _metrics(session) -> Dict[str, Any]:
+    # Lightweight metrics for handshake
+    # Count total content (any type)
+    total_contents = (await session.execute(select(StoredContent))).scalars().all()
+    content_count = len(total_contents)
+    # Basic system metrics
+    try:
+        load1, load5, load15 = os.getloadavg()
+    except Exception:
+        load1 = load5 = load15 = 0.0
+    try:
+        from app.core._config import UPLOADS_DIR
+        du = shutil.disk_usage(UPLOADS_DIR)
+        disk_total_gb = round(du.total / (1024 ** 3), 2)
+        disk_free_gb = round(du.free / (1024 ** 3), 2)
+    except Exception:
+        disk_total_gb = disk_free_gb = -1
+    uptime_sec = int(time.time() - START_TS)
+    return {
+        "content_count": content_count,
+        "uptime_sec": uptime_sec,
+        "loadavg": [load1, load5, load15],
+        "disk_total_gb": disk_total_gb,
+        "disk_free_gb": disk_free_gb,
+    }
+
+
+def _sign(obj: Dict[str, Any]) -> str:
+    signer = Signer(hot_seed)
+    blob = json.dumps(obj, sort_keys=True, separators=(",", ":")).encode()
+    return signer.sign(blob)
+
+
+async def build_handshake_payload(session) -> Dict[str, Any]:
+    payload = {
+        "version": CURRENT_PROTOCOL_VERSION,
+        "public_key": b58encode(hot_pubkey).decode(),
+        # public_host is optional for private nodes
+        **({"public_host": PUBLIC_HOST} if PUBLIC_HOST else {}),
+        "node_type": NODE_PRIVACY if NODE_PRIVACY != NODE_TYPE_PUBLIC else NODE_TYPE_PUBLIC,
+        "metrics": await _metrics(session),
+        "capabilities": {
+            "accepts_inbound": NODE_PRIVACY == NODE_TYPE_PUBLIC,
+            "is_bootstrap": NODE_IS_BOOTSTRAP,
+            "supported_types": ALLOWED_CONTENT_TYPES,
+            "max_content_size_mb": MAX_CONTENT_SIZE_MB,
+        },
+        "timestamp": int(datetime.utcnow().timestamp()),
+        "nonce": secrets.token_hex(16),
+    }
+    try:
+        payload["known_public_nodes"] = await list_known_public_nodes(session)
+    except Exception:
+        payload["known_public_nodes"] = []
+    payload["signature"] = _sign(payload)
+    return payload
+
+
+async def compute_node_info(session) -> Dict[str, Any]:
+    node_info = {
+        "id": b58encode(hot_pubkey).decode(),
+        "public_key": b58encode(hot_pubkey).decode(),
+        **({"public_host": PUBLIC_HOST} if PUBLIC_HOST else {}),
+        "version": CURRENT_PROTOCOL_VERSION,
+        "node_type": NODE_PRIVACY,
+        "metrics": await _metrics(session),
+        "capabilities": {
+            "accepts_inbound": NODE_PRIVACY == NODE_TYPE_PUBLIC,
+            "is_bootstrap": NODE_IS_BOOTSTRAP,
+            "supported_types": ALLOWED_CONTENT_TYPES,
+            "max_content_size_mb": MAX_CONTENT_SIZE_MB,
+        },
+    }
+    return node_info
+
+def sign_response(data: Dict[str, Any]) -> Dict[str, Any]:
+    body = {
+        **data,
+        "timestamp": int(datetime.utcnow().timestamp()),
+    }
+    sig = _sign(body)
+    body["server_public_key"] = b58encode(hot_pubkey).decode()
+    body["server_signature"] = sig
+    return body

app/core/network/key_client.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import base64
+from typing import Optional
+
+import httpx
+from base58 import b58encode
+
+from app.core._secrets import hot_seed, hot_pubkey
+from app.core.crypto.x25519 import ed25519_to_x25519
+from app.core.logger import make_log
+from app.core.network.nodesig import sign_headers
+
+
+async def request_key_from_peer(base_url: str, encrypted_cid: str) -> Optional[bytes]:
+    """
+    Request a sealed key from peer and decrypt it using our X25519 private key.
+    Returns plaintext DEK bytes or None on failure.
+    """
+    try:
+        sk_x, pk_x = ed25519_to_x25519(hot_seed)
+        node_id = b58encode(hot_pubkey).decode()
+        body = {
+            "encrypted_cid": encrypted_cid,
+            "requestor_node_id": node_id,
+            "recipient_box_pub": base64.b64encode(bytes(pk_x)).decode(),
+        }
+        path = "/api/v1/keys.request"
+        headers = sign_headers("POST", path, json.dumps(body).encode(), hot_seed, b58encode(hot_pubkey).decode())
+        async with httpx.AsyncClient(timeout=15) as client:
+            r = await client.post(f"{base_url.rstrip('/')}{path}", json=body, headers=headers)
+            if r.status_code != 200:
+                make_log('key_client', f"{base_url} returned {r.status_code}: {r.text}", level='warning')
+                return None
+            j = r.json()
+        sealed_b64 = j.get('sealed_key_b64')
+        if not sealed_b64:
+            return None
+        sealed = base64.b64decode(sealed_b64)
+        from nacl.public import SealedBox
+        sb = SealedBox(sk_x)
+        dek = sb.decrypt(sealed)
+        return dek
+    except Exception as e:
+        make_log('key_client', f"request/decrypt failed: {e}", level='error')
+        return None

app/core/network/nodes.py

@@ -0,0 +1,261 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timedelta
+import json
+from typing import Dict, Any, Optional, List
+
+import httpx
+from base58 import b58encode
+from sqlalchemy import select, update
+
+from app.core.logger import make_log
+from app.core.models.my_network import KnownNode
+from app.core.storage import db_session
+from app.core._secrets import hot_pubkey
+from .config import (
+    HANDSHAKE_INTERVAL_SEC,
+    UNSUPPORTED_RECHECK_INTERVAL_SEC,
+    BOOTSTRAP_SEEDS,
+    BOOTSTRAP_REQUIRED,
+    BOOTSTRAP_TIMEOUT_SEC,
+    NODE_PRIVACY,
+    NETWORK_TLS_VERIFY,
+)
+from .constants import NODE_TYPE_PRIVATE
+from .semver import compatibility
+from .constants import CURRENT_PROTOCOL_VERSION
+
+
+def _now() -> datetime:
+    return datetime.utcnow()
+
+
+async def upsert_known_node(session, host: str, port: int, public_key: str, meta: Dict[str, Any]) -> KnownNode:
+    # Host can be full URL; normalize host/ip and port if available
+    host = (host or "").replace("http://", "").replace("https://", "").strip("/")
+    h_only = host
+    if ":" in host:
+        h_only, port_str = host.rsplit(":", 1)
+        try:
+            port = int(port_str)
+        except Exception:
+            pass
+    # Prefer match by public_key (stable identity)
+    if public_key:
+        result = await session.execute(select(KnownNode).where(KnownNode.public_key == public_key))
+        row = result.scalars().first()
+        if row:
+            row.ip = h_only or row.ip
+            row.port = port or row.port
+            row.public_key = public_key or row.public_key
+            row.meta = {**(row.meta or {}), **(meta or {})}
+            row.last_sync = _now()
+            await session.commit()
+            return row
+    # Fallback by IP/host
+    result = await session.execute(select(KnownNode).where(KnownNode.ip == h_only))
+    row = result.scalars().first()
+    if row:
+        row.port = port or row.port
+        row.public_key = public_key or row.public_key
+        row.meta = {**(row.meta or {}), **(meta or {})}
+        row.last_sync = _now()
+        await session.commit()
+        return row
+    node = KnownNode(
+        ip=h_only,
+        port=port or 80,
+        public_key=public_key,
+        reputation=0,
+        last_sync=_now(),
+        meta=meta or {},
+        located_at=_now(),
+    )
+    session.add(node)
+    await session.commit()
+    return node
+
+
+def _compatibility_for_meta(remote_version: str) -> str:
+    if not remote_version or remote_version == "0.0.0":
+        return "warning"
+    return compatibility(remote_version, CURRENT_PROTOCOL_VERSION)
+
+
+async def list_known_public_nodes(session) -> List[Dict[str, Any]]:
+    rows = (await session.execute(select(KnownNode))).scalars().all()
+    result = []
+    for r in rows:
+        meta = r.meta or {}
+        if not meta.get("is_public", True):
+            continue
+        result.append({
+            "host": r.ip,
+            "port": r.port,
+            "public_key": r.public_key,
+            "version": meta.get("version"),
+            "compatibility": _compatibility_for_meta(meta.get("version", "0.0.0")),
+            "last_seen": (r.last_sync.isoformat() + "Z") if r.last_sync else None,
+            "public_host": meta.get("public_host"),
+            "capabilities": meta.get("capabilities") or {},
+        })
+    return result
+
+
+async def _handshake_with(session, base_url: str) -> Optional[Dict[str, Any]]:
+    url = base_url.rstrip("/") + "/api/v1/network.handshake"
+    from .handshake import build_handshake_payload
+    payload = await build_handshake_payload(session)
+    timeout = httpx.Timeout(5.0, read=10.0)
+    async with httpx.AsyncClient(timeout=timeout, verify=NETWORK_TLS_VERIFY) as client:
+        r = await client.post(url, json=payload)
+        if r.status_code == 403 and NODE_PRIVACY == NODE_TYPE_PRIVATE:
+            # We are private; outbound is allowed, inbound denied by peers is fine
+            pass
+        r.raise_for_status()
+        data = r.json()
+        # Verify server signature if present
+        try:
+            import nacl.signing
+            from base58 import b58decode
+            required = ["server_signature", "server_public_key", "timestamp"]
+            if all(k in data for k in required):
+                signed_fields = {k: data[k] for k in data if k not in ("server_signature", "server_public_key")}
+                blob = json.dumps(signed_fields, sort_keys=True, separators=(",", ":")).encode()
+                vk = nacl.signing.VerifyKey(b58decode(data["server_public_key"]))
+                vk.verify(blob, b58decode(data["server_signature"]))
+        except Exception as e:
+            make_log("Handshake", f"Server signature verification failed for {base_url}: {e}", level='warning')
+        return data
+
+
+async def pick_next_node(session) -> Optional[KnownNode]:
+    rows = (await session.execute(select(KnownNode))).scalars().all()
+    if not rows:
+        return None
+    # Prefer nodes with oldest last_sync
+    rows.sort(key=lambda r: (r.last_sync or datetime.fromtimestamp(0)))
+    now = _now()
+    for r in rows:
+        meta = r.meta or {}
+        compat = _compatibility_for_meta(meta.get("version", "0.0.0"))
+        if compat == "blocked":
+            last = datetime.fromisoformat(meta.get("unsupported_last_checked_at")) if meta.get("unsupported_last_checked_at") else None
+            if last and (now - last) < timedelta(seconds=UNSUPPORTED_RECHECK_INTERVAL_SEC):
+                continue
+        # Backoff after failures
+        if meta.get("last_failure_at"):
+            try:
+                last_fail = datetime.fromisoformat(meta.get("last_failure_at"))
+                fail_count = int(meta.get("fail_count", 1))
+                # Exponential backoff: 30s * 2^fail_count, capped 2h
+                wait = min(7200, 30 * (2 ** max(0, fail_count)))
+                if (now - last_fail) < timedelta(seconds=wait):
+                    continue
+            except Exception:
+                pass
+        return r
+    # If we only have unsupported nodes and all are within cooldown, skip this round
+    return None
+
+
+async def perform_handshake_round():
+    async with db_session(auto_commit=True) as session:
+        # Private nodes still do outbound handshakes; inbound typically unreachable without public endpoint
+        node = await pick_next_node(session)
+        if not node:
+            return
+        base_url = node.meta.get("public_host") or f"http://{node.ip}:{node.port}"
+        try:
+            resp = await _handshake_with(session, base_url)
+            # Merge known nodes received
+            for peer in (resp or {}).get("known_public_nodes", []):
+                try:
+                    await upsert_known_node(
+                        session,
+                        host=peer.get("host") or peer.get("public_host") or "",
+                        port=int(peer.get("port") or 80),
+                        public_key=peer.get("public_key") or "",
+                        meta={
+                            "is_public": True,
+                            "version": peer.get("version") or "0.0.0",
+                            "public_host": peer.get("public_host") or (f"http://{peer.get('host')}:{peer.get('port')}" if peer.get('host') else None),
+                        }
+                    )
+                except Exception as e:
+                    make_log("Handshake", f"Ignore bad peer from {base_url}: {e}", level='warning')
+            # Update last_sync and meta for node
+            node.last_sync = _now()
+            node.meta = {**(node.meta or {}), "last_response": resp, "fail_count": 0}
+            await session.commit()
+            make_log("Handshake", f"Handshake OK with {base_url}")
+        except Exception as e:
+            make_log("Handshake", f"Handshake failed with {base_url}: {e}", level='warning')
+            # Record incident-lite in meta
+            meta = node.meta or {}
+            meta["last_error"] = str(e)
+            meta["last_failure_at"] = _now().isoformat()
+            meta["fail_count"] = int(meta.get("fail_count", 0)) + 1
+            node.meta = meta
+            await session.commit()
+
+
+async def network_handshake_daemon(app):
+    # Stagger start a bit to allow HTTP server to come up
+    await asyncio.sleep(3)
+    make_log("Handshake", f"Daemon started; interval={HANDSHAKE_INTERVAL_SEC}s")
+    while True:
+        try:
+            await perform_handshake_round()
+        except Exception as e:
+            make_log("Handshake", f"Round error: {e}", level='error')
+        await asyncio.sleep(HANDSHAKE_INTERVAL_SEC)
+
+
+async def bootstrap_once_and_exit_if_failed():
+    # Do not try to bootstrap private nodes as inbound is blocked, but outbound required for seeds discovery
+    seeds = BOOTSTRAP_SEEDS or []
+    if not seeds:
+        return  # Nothing to do
+    async with db_session(auto_commit=True) as session:
+        # If we already know nodes, skip bootstrap
+        have_any = (await session.execute(select(KnownNode))).scalars().first()
+        if have_any:
+            return
+    make_log("Bootstrap", f"Starting bootstrap with seeds={seeds}; required={BOOTSTRAP_REQUIRED}")
+
+    deadline = _now() + timedelta(seconds=BOOTSTRAP_TIMEOUT_SEC)
+    ok = False
+    for seed in seeds:
+        try:
+            async with db_session(auto_commit=True) as session:
+                resp = await _handshake_with(session, seed)
+                if resp:
+                    ok = True
+                    # Seed itself gets inserted by handshake handling route; also insert it explicitly
+                    try:
+                        await upsert_known_node(
+                            session,
+                            host=seed,
+                            port=80,
+                            public_key=resp.get("node", {}).get("public_key", ""),
+                            meta={
+                                "is_public": True,
+                                "version": resp.get("node", {}).get("version", "0.0.0"),
+                                "public_host": resp.get("node", {}).get("public_host") or seed,
+                            }
+                        )
+                    except Exception:
+                        pass
+                    break
+        except Exception as e:
+            make_log("Bootstrap", f"Seed failed {seed}: {e}", level='warning')
+        if _now() > deadline:
+            break
+
+    if BOOTSTRAP_REQUIRED and not ok:
+        make_log("Bootstrap", "Failed to reach any bootstrap seeds; exiting", level='error')
+        # Hard exit; Sanic won't stop otherwise
+        import os
+        os._exit(2)

app/core/network/nodesig.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import secrets
+import time
+from typing import Dict, Tuple
+
+from base58 import b58decode, b58encode
+
+from app.core.network.guard import check_timestamp_fresh, check_and_remember_nonce
+
+
+def _body_sha256(body: bytes) -> str:
+    h = hashlib.sha256()
+    h.update(body or b"")
+    return h.hexdigest()
+
+
+def canonical_string(method: str, path: str, body: bytes, ts: int, nonce: str, node_id: str) -> bytes:
+    parts = [
+        method.upper(),
+        path,
+        _body_sha256(body),
+        str(int(ts)),
+        str(nonce),
+        node_id,
+    ]
+    return ("\n".join(parts)).encode()
+
+
+def sign_headers(method: str, path: str, body: bytes, sk_bytes: bytes, pk_b58: str) -> Dict[str, str]:
+    import nacl.signing
+    ts = int(time.time())
+    nonce = secrets.token_hex(16)
+    msg = canonical_string(method, path, body, ts, nonce, pk_b58)
+    sig = nacl.signing.SigningKey(sk_bytes).sign(msg).signature
+    return {
+        "X-Node-Id": pk_b58,
+        "X-Node-Ts": str(ts),
+        "X-Node-Nonce": nonce,
+        "X-Node-Sig": b58encode(sig).decode(),
+    }
+
+
+def verify_request(request, memory) -> Tuple[bool, str, str]:
+    """Verify NodeSig headers of an incoming Sanic request.
+    Returns (ok, node_id, error). ok==True if signature valid, timestamp fresh, nonce unused.
+    """
+    try:
+        node_id = request.headers.get("X-Node-Id", "").strip()
+        ts = int(request.headers.get("X-Node-Ts", "0").strip() or 0)
+        nonce = request.headers.get("X-Node-Nonce", "").strip()
+        sig_b58 = request.headers.get("X-Node-Sig", "").strip()
+        if not node_id or not ts or not nonce or not sig_b58:
+            return False, "", "MISSING_HEADERS"
+        if not check_timestamp_fresh(ts):
+            return False, node_id, "STALE_TS"
+        if not check_and_remember_nonce(memory, node_id, nonce):
+            return False, node_id, "NONCE_REPLAY"
+        import nacl.signing
+        vk = nacl.signing.VerifyKey(b58decode(node_id))
+        sig = b58decode(sig_b58)
+        msg = canonical_string(request.method, request.path, request.body or b"", ts, nonce, node_id)
+        vk.verify(msg, sig)
+        return True, node_id, ""
+    except Exception as e:
+        return False, "", f"BAD_SIGNATURE: {e}"
+

app/core/network/semver.py

@@ -0,0 +1,26 @@
+from typing import Tuple
+
+
+def parse_semver(v: str) -> Tuple[int, int, int]:
+    try:
+        parts = v.split(".")
+        major = int(parts[0])
+        minor = int(parts[1]) if len(parts) > 1 else 0
+        patch = int(parts[2]) if len(parts) > 2 else 0
+        return major, minor, patch
+    except Exception:
+        return 0, 0, 0
+
+
+def compatibility(peer: str, current: str) -> str:
+    """Return one of: compatible, warning, blocked"""
+    pM, pm, pp = parse_semver(peer)
+    cM, cm, cp = parse_semver(current)
+    if pM != cM:
+        return "blocked"
+    # Same major
+    if pm == cm:
+        return "compatible"
+    # Different minor within same major => warning
+    return "warning"
+

app/core/projscale_logger.py

@@ -68,7 +68,7 @@ file_handler.setLevel(logging.DEBUG)
 file_handler.setFormatter(logging.Formatter(FORMAT_STRING))
 logger.addHandler(file_handler)
 
-if os.getenv('APP_ENABLE_STDOUT_LOGS', '0') == '1':
+if int(os.getenv('APP_ENABLE_STDOUT_LOGS', '0')) == 1:
     stdout_handler = logging.StreamHandler()
     stdout_handler.setLevel(logging.DEBUG)
     stdout_handler.setFormatter(logging.Formatter(FORMAT_STRING))

app/core/storage.py

@@ -1,45 +1,57 @@
 import time
-from contextlib import contextmanager
+from contextlib import asynccontextmanager
 
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.sql import text
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
 
-from app.core._config import MYSQL_URI, MYSQL_DATABASE
+from app.core._config import DATABASE_URL
 from app.core.logger import make_log
-from sqlalchemy.pool import NullPool
-
-engine = create_engine(MYSQL_URI, poolclass=NullPool)  #, echo=True)
-Session = sessionmaker(bind=engine)
 
 
-database_initialized = False
-while not database_initialized:
+def _to_async_dsn(url: str) -> str:
+    # Convert psycopg2 DSN to asyncpg DSN
+    # postgresql+psycopg2://user:pass@host:5432/db -> postgresql+asyncpg://user:pass@host:5432/db
+    return url.replace("+psycopg2", "+asyncpg")
+
+
+# Async engine for PostgreSQL
+engine = create_async_engine(
+    _to_async_dsn(DATABASE_URL),
+    pool_size=10,
+    max_overflow=20,
+    pool_timeout=30,
+    pool_recycle=1800,
+    pool_pre_ping=True,
+)
+
+AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
+
+
+async def wait_db_ready():
+    ready = False
+    while not ready:
+        try:
+            async with engine.connect() as conn:
+                await conn.execute(text("SELECT 1"))
+                ready = True
+        except Exception as e:
+            make_log("SQL", 'PostgreSQL is not ready yet: ' + str(e), level='debug')
+            time.sleep(1)
+
+
+@asynccontextmanager
+async def db_session(auto_commit: bool = False):
+    session: AsyncSession = AsyncSessionLocal()
     try:
-        with Session() as session:
-            databases_list = session.execute(text("SHOW DATABASES;"))
-            databases_list = [row[0] for row in databases_list]
-            make_log("SQL", 'Database list: ' + str(databases_list), level='debug')
-            assert MYSQL_DATABASE in databases_list, 'Database not found'
-            database_initialized = True
-    except Exception as e:
-        make_log("SQL", 'MariaDB is not ready yet: ' + str(e), level='debug')
-        time.sleep(1)
-
-engine = create_engine(f"{MYSQL_URI}/{MYSQL_DATABASE}", poolclass=NullPool)
-Session = sessionmaker(bind=engine)
-
-
-@contextmanager
-def db_session(auto_commit=False):
-    _session = Session()
-    try:
-        yield _session
-        if auto_commit is True:
-            _session.commit()
+        yield session
+        if auto_commit:
+            await session.commit()
     except BaseException as e:
-        _session.rollback()
+        await session.rollback()
         raise e
     finally:
-        _session.close()
+        await session.close()
 
+
+def new_session() -> AsyncSession:
+    return AsyncSessionLocal()

app/core/transactions.py

@@ -1,19 +1,19 @@
 from datetime import datetime
 
+from sqlalchemy import select, and_, func
 from app.core.logger import make_log
 from app.core.models import Memory, User, UserBalance, Asset, InternalTransaction
 from app.core.storage import db_session
 
 
-def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
+async def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
     assert user, "No user"
     assert asset, "No asset"
-    result = session.query(UserBalance).filter(
-        UserBalance.user_id == user.id,
-        UserBalance.asset_id == asset.id
-    )
-    results_count = result.count()
-    if results_count == 0:
+    result = await session.execute(select(UserBalance).where(
+        and_(UserBalance.user_id == user.id, UserBalance.asset_id == asset.id)
+    ))
+    row = result.scalars().first()
+    if not row:
         user_balance = UserBalance(
             user_id=user.id,
             asset_id=asset.id,
@@ -21,12 +21,9 @@ def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
             created=datetime.now(),
         )
         session.add(user_balance)
-        session.commit()
-        return get_user_balance(session, user, asset)
-    elif results_count == 1:
-        return result.first()
-    else:
-        raise Exception(f"Multiple user balances found: {results_count}")
+        await session.commit()
+        return await get_user_balance(session, user, asset)
+    return row
 
 
 async def make_internal_transaction(
@@ -46,13 +43,13 @@ async def make_internal_transaction(
         raise Exception(f"Invalid amount: {amount}")
 
     abs_amount = abs(amount)
-    with db_session(auto_commit=False) as session:
+    async with db_session(auto_commit=False) as session:
         async with memory.transaction():
-            user = session.query(User).filter_by(id=user_id).first()
+            user = (await session.execute(select(User).where(User.id == user_id))).scalars().first()
             assert user, "No user"
-            asset = session.query(Asset).filter_by(id=asset_id).first()
+            asset = (await session.execute(select(Asset).where(Asset.id == asset_id))).scalars().first()
             assert asset, "No asset"
-            user_balance = get_user_balance(session, user, asset)
+            user_balance = await get_user_balance(session, user, asset)
             assert user_balance, "No user balance"
             if is_spent is True:
                 if abs_amount > user_balance.balance:
@@ -71,6 +68,6 @@ async def make_internal_transaction(
                 created=datetime.now(),
             )
             session.add(internal_transaction)
-            session.commit()
+            await session.commit()
 
         make_log(user, f"Made internal transaction: {'-' if is_spent else ''}{abs_amount} {asset.symbol}, type: {type}")

docker-compose.yml

@@ -1,99 +0,0 @@
-version: '3'
-services:
-  maria_db:
-    image: mariadb:11.2
-    ports:
-      - "3307:3306"
-    env_file:
-      - .env
-    volumes:
-      - /Storage/sqlStorage:/var/lib/mysql
-    healthcheck:
-      test: [ "CMD", "healthcheck.sh", "--connect", "--innodb_initialized" ]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-
-  app:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: python -m app
-    env_file:
-      - .env
-    links:
-      - maria_db
-    ports:
-      - "15100:15100"
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-    depends_on:
-      maria_db:
-        condition: service_healthy
-
-  indexer:  # Отправка уведомления о появлении новой NFT-listen. Установка CID поля у всего контента. Проверка следующего за последним индексом item коллекции и поиск нового контента, отправка информации о том что контент найден его загружателю. Присваивание encrypted_content onchain_index
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: python -m app indexer
-    env_file:
-      - .env
-    links:
-      - maria_db
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-    depends_on:
-      maria_db:
-        condition: service_healthy
-
-  ton_daemon: # Работа с TON-сетью. Задачи сервисного кошелька и деплой контрактов
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: python -m app ton_daemon
-    env_file:
-      - .env
-    links:
-      - maria_db
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-    depends_on:
-      maria_db:
-        condition: service_healthy
-
-  license_index: # Проверка кошельков пользователей на новые NFT. Опрос этих NFT на определяемый GET-метод по которому мы определяем что это определенная лицензия и сохранение информации по ней
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: python -m app license_index
-    env_file:
-      - .env
-    links:
-      - maria_db
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-    depends_on:
-      maria_db:
-        condition: service_healthy
-
-  convert_process:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    command: python -m app convert_process
-    env_file:
-      - .env
-    links:
-      - maria_db
-    volumes:
-      - /Storage/logs:/app/logs
-      - /Storage/storedContent:/app/data
-      - /var/run/docker.sock:/var/run/docker.sock
-    depends_on:
-      maria_db:
-        condition: service_healthy
-

docs/indexation.md

@@ -38,4 +38,73 @@ values:^[
 2. User uploads content cover to server (/api/v1/storage)
 3. User send /api/v1/blockchain.sendNewContentMessage to server and accept the transaction in wallet
 4. Indexer receives the transaction and indexes the content. And send telegram notification to user.
+# Network Index & Sync (v3)
 
+This document describes the simplified, production‑ready stack for content discovery and sync:
+
+- Upload via tus → stream encrypt (ENCF v1, AES‑256‑GCM, 1 MiB chunks) → `ipfs add --cid-version=1 --raw-leaves --chunker=size-1048576 --pin`.
+- Public index exposes only encrypted sources (CID) and safe metadata; no plaintext ids.
+- Nodes full‑sync by pinning encrypted CIDs; keys are auto‑granted to trusted peers for preview/full access.
+
+## ENCF v1 (Encrypted Content Format)
+
+Unencrypted header and framed body; same bytes on all nodes ⇒ stable CID.
+
+Header (all big endian):
+
+```
+MAGIC(4):  'ENCF'
+VER(1):    0x01
+SCHEME(1): 0x03 = AES_GCM (0x01 AES_GCM_SIV legacy, 0x02 AES_SIV legacy)
+CHUNK(4):  plaintext chunk bytes (1048576)
+SALT_LEN(1)
+SALT(N)
+RESERVED(5): zeros
+```
+
+Body: repeated frames `[p_len:4][cipher][tag(16)]` where `p_len <= CHUNK` for last frame.
+
+AES‑GCM (scheme `0x03`) encrypts each frame with deterministic `nonce = HMAC_SHA256(salt, u64(frame_idx))[:12]`. Legacy scheme `0x01` keeps AES‑GCM‑SIV with the same nonce derivation.
+
+For new uploads (v2025-09), the pipeline defaults to AES‑256‑GCM. Legacy AES‑GCM‑SIV/AES‑SIV content is still readable — the decoder auto-detects the scheme byte.
+
+### Local encryption/decryption helpers
+
+```
+python -m app.core.crypto.cli encrypt --input demo.wav --output demo.encf \
+  --key AAAAEyHSVws5O8JGrg3kUSVtk5dQSc5x5e7jh0S2WGE= --salt-bytes 16
+
+python -m app.core.crypto.cli decrypt --input demo.encf --output demo.wav \
+  --wrapped-key <ContentKey.key_ciphertext_b64>
+```
+
+Because we use standard AES‑GCM, you can also re-hydrate frames manually with tools like `openssl aes-256-gcm`. The header exposes `chunk_bytes` and salt; derive the per-frame nonce via `HMAC_SHA256(salt, idx)` where `idx` is the frame number (0-based) and feed the 12-byte prefix as IV.
+
+## API
+
+- `GET /api/v1/content.index` → `{ items:[...], schema, ETag }` with signed items.
+- `GET /api/v1/content.delta?since=ISO8601` → `{ items:[...], next_since, schema }` with ETag.
+- `POST /api/v1/sync.pin` (NodeSig required) → queue/pin CID.
+- `POST /api/v1/keys.request` (NodeSig required) → sealed DEK for trusted peers.
+- `GET /api/v1/content.derivatives?cid=` → local ready derivatives (low/high/preview).
+
+## NodeSig
+
+Canonical string:
+
+```
+METHOD\nPATH\nSHA256(body)\nTS\nNONCE\nNODE_ID
+```
+
+Headers: `X-Node-Id`, `X-Node-Ts`, `X-Node-Nonce`, `X-Node-Sig`.
+Window ±120s, nonce cache ~10min; replay → 401.
+
+## Sync daemon
+
+- Jitter 0–30s per peer; uses ETag/`since`.
+- Disk watermark (`SYNC_DISK_LOW_WATERMARK_PCT`) stops pin burst.
+- Pinned concurrently (`SYNC_MAX_CONCURRENT_PINS`) with pre‑`findprovs` `swarm/connect`.
+
+## Keys policy
+
+`KEY_AUTO_GRANT_TRUSTED_ONLY=1` — only KnownNode.meta.role=='trusted' gets DEK automatically. Preview lease TTL via `KEY_GRANT_PREVIEW_TTL_SEC`.

env.example

@@ -3,3 +3,4 @@ TELEGRAM_API_KEY=Paste your telegram api key from @BotFather here
 MYSQL_URI=mysql+pymysql://user:password@maria_db:3306
 MYSQL_ROOT_PASSWORD=playground
 MYSQL_DATABASE=bot_database
+CONTENT_KEY_KEK_B64=Paste base64-encoded 32-byte key for wrapping DEKs

locale/en/LC_MESSAGES/sanic_telegram_bot.po

@@ -148,8 +148,11 @@ msgid "buyTrackListenLicense_button"
 msgstr "💶 Купить лицензию ({price} TON)"
 
 #: app/core/models/_telegram/templates/player.py:117
-msgid "p_playerContext_preview"
-msgstr "<i>Это демонстрационная версия трека. Чтобы прослушать полную версию, необходимо приобрести лицензию 🎧.</i>"
+msgid "p_playerAudioContext_preview"
+msgstr "<i>Превью аудиозаписи. Полная версия доступна только по лицензии 🎧</i>"
+
+msgid "p_playerVideoContext_preview"
+msgstr "<i>Превью видео. Полная версия доступна только по лицензии 🎬</i>"
 
 #: app/client_bot/routers/content.py:32
 msgid "error_contentNotFound"
@@ -194,3 +197,12 @@ msgstr "🔗 Поделиться"
 
 msgid "p_shareLinkContext"
 msgstr "🎉 Наслаждайтесь {title} на MY!"
+
+msgid "p_uploadContentTxPromo"
+msgstr ""
+"🎉 Вам доступно ещё <b>{free_count}</b> бесплатных приветственных загрузок контента! "
+"Контент <b>{title}</b> уже находится в процессе загрузки. Как только блокчейн обработает транзакцию, "
+"вы получите NFT-лицензию."
+
+msgid "p_playerContext_contentNotReady"
+msgstr "⚠️ Контент, который вы хотите просмотреть, ещё не готов. Пожалуйста, попробуйте позже."

requirements.txt

@@ -2,11 +2,12 @@ sanic==21.9.1
 websockets==10.0
 sqlalchemy==2.0.23
 python-dotenv==1.0.0
-pymysql==1.1.0
+psycopg2-binary==2.9.9
+asyncpg==0.29.0
 aiogram==3.13.0
 pytonconnect==0.3.0
 base58==2.1.1
-tonsdk==1.0.13
+git+https://github.com/tonfactory/tonsdk.git@3ebbf0b702f48c2519e4c6c425f9514f673b9d48#egg=tonsdk
 httpx==0.25.0
 docker==7.0.0
 pycryptodome==3.20.0
@@ -15,4 +16,5 @@ aiofiles==23.2.1
 pydub==0.25.1
 pillow==10.2.0
 ffmpeg-python==0.2.0
-
+python-magic==0.4.27
+cryptography==42.0.5