improve work

app/api/routes/_blockchain.py

@@ -1,4 +1,4 @@
-from base64 import b64encode, b32decode
+from base64 import b64encode
 from datetime import datetime
 import traceback
 
@@ -7,7 +7,6 @@ from sqlalchemy import and_, select, func
 from tonsdk.boc import begin_cell, begin_dict
 from tonsdk.utils import Address
 
-from base58 import b58encode, b58decode
 from app.core._blockchain.ton.connect import TonConnect, wallet_obj_by_name
 from app.core._blockchain.ton.platform import platform
 from app.core._config import PROJECT_HOST
@@ -58,92 +57,21 @@ async def s_api_v1_blockchain_send_new_content_message(request):
             assert field_value(request.json[field_key]), f"Invalid {field_key} provided"
 
         # Support legacy: 'content' as decrypted ContentId; and new: 'content' as encrypted IPFS CID
+        source_content_cid, cid_err = resolve_content(request.json['content'])
+        assert not cid_err, f"Invalid content CID provided: {cid_err}"
+
         encrypted_content_cid = None
-        try:
-            # Legacy path
-            decrypted_content_cid, err = resolve_content(request.json['content'])
-            assert not err
-            decrypted_content = (await request.ctx.db_session.execute(
-                select(StoredContent).where(StoredContent.hash == decrypted_content_cid.content_hash_b58)
-            )).scalars().first()
-            assert decrypted_content and decrypted_content.type == "local/content_bin"
+        decrypted_content = (await request.ctx.db_session.execute(
+            select(StoredContent).where(StoredContent.hash == source_content_cid.content_hash_b58)
+        )).scalars().first()
+
+        if decrypted_content and decrypted_content.type == "local/content_bin":
             encrypted_content = await create_encrypted_content(request.ctx.db_session, decrypted_content)
             encrypted_content_cid = encrypted_content.cid
-        except BaseException:
-            # New path: treat provided string as encrypted IPFS CID (ENCF v1)
-            encrypted_ipfs_cid = request.json['content']
-
-            class _EC:
-                """Adapter to provide ContentId-like interface for IPFS CID strings."""
-
-                def __init__(self, cid_str: str):
-                    self._cid = cid_str
-                    self.content_hash = self._extract_content_hash(cid_str)
-                    self._content_hash_b58 = None
-
-                @staticmethod
-                def _decode_multibase(cid_str: str) -> bytes:
-                    if not cid_str:
-                        raise ValueError("empty CID")
-                    prefix = cid_str[0]
-                    if prefix in ('b', 'B'):
-                        payload = cid_str[1:]
-                        padding = (-len(payload)) % 8
-                        return b32decode(payload.upper() + ('=' * padding), casefold=True)
-                    if prefix in ('z', 'Z'):
-                        return b58decode(cid_str[1:])
-                    # CIDv0 (base58btc without explicit multibase prefix)
-                    return b58decode(cid_str)
-
-                @staticmethod
-                def _read_varint(data: bytes, offset: int):
-                    result = 0
-                    shift = 0
-                    while True:
-                        if offset >= len(data):
-                            raise ValueError("truncated varint")
-                        byte = data[offset]
-                        offset += 1
-                        result |= (byte & 0x7F) << shift
-                        if not (byte & 0x80):
-                            break
-                        shift += 7
-                        if shift > 63:
-                            raise ValueError("varint overflow")
-                    return result, offset
-
-                @classmethod
-                def _extract_content_hash(cls, cid_str: str) -> bytes:
-                    data = cls._decode_multibase(cid_str)
-                    offset = 0
-                    if data and data[0] == 0x01:
-                        version, offset = cls._read_varint(data, offset)
-                        if version != 1:
-                            raise ValueError("unsupported CID version")
-                        _, offset = cls._read_varint(data, offset)  # skip codec
-                    code, offset = cls._read_varint(data, offset)
-                    length, offset = cls._read_varint(data, offset)
-                    digest = data[offset:offset + length]
-                    if len(digest) != length:
-                        raise ValueError("truncated multihash digest")
-                    if code != 0x12 or length != 32:
-                        raise ValueError("unsupported multihash (expect sha2-256)")
-                    return digest
-
-                def serialize_v2(self, include_accept_type: bool = False):
-                    return self._cid
-
-                @property
-                def content_hash_b58(self) -> str:
-                    if self._content_hash_b58 is None:
-                        self._content_hash_b58 = b58encode(self.content_hash).decode()
-                    return self._content_hash_b58
-
-            try:
-                encrypted_content_cid = _EC(encrypted_ipfs_cid)
-            except Exception as exc:
-                make_log("Blockchain", f"Provided encrypted IPFS CID is invalid: {exc}", level='error')
-                raise AssertionError("Invalid encrypted content CID provided") from exc
+        elif source_content_cid.cid_format == 'ipfs':
+            encrypted_content_cid = source_content_cid
+        else:
+            raise AssertionError("Provided content is neither locally available nor a valid encrypted CID")
 
         if request.json['image']:
             image_content_cid, err = resolve_content(request.json['image'])
@@ -218,7 +146,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                 user_id = str(request.ctx.user.id),
                 user_internal_id=request.ctx.user.id,
                 action_type='freeUpload',
-                action_ref=str(encrypted_content_cid.content_hash),
+                action_ref=encrypted_content_cid.serialize_v2(),
                 created=datetime.now()
             )
             request.ctx.db_session.add(promo_action)
@@ -275,7 +203,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                     title=content_title,
                     free_count=(promo_free_upload_available - 1)
                 ), message_type='hint', message_meta={
-                    'encrypted_content_hash': b58encode(encrypted_content_cid.content_hash).decode(),
+                    'encrypted_content_hash': encrypted_content_cid.content_hash_b58,
                     'hint_type': 'uploadContentTxRequested'
                 }
             )
@@ -285,54 +213,59 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                 'payload': ""
             })
 
+        user_wallet_address = await request.ctx.user.wallet_address_async(request.ctx.db_session)
+        assert user_wallet_address, "Wallet address is not linked"
+
         await request.ctx.user_uploader_wrapper.send_message(
             request.ctx.user.translated('p_uploadContentTxRequested').format(
                 title=content_title,
             ), message_type='hint', message_meta={
-                'encrypted_content_hash': b58encode(encrypted_content_cid.content_hash).decode(),
+                'encrypted_content_hash': encrypted_content_cid.content_hash_b58,
                 'hint_type': 'uploadContentTxRequested'
             }
         )
 
+        payload_cell = (
+            begin_cell()
+                .store_uint(0x5491d08c, 32)
+                .store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
+                .store_address(Address(user_wallet_address))
+                .store_ref(
+                    begin_cell()
+                        .store_ref(
+                            begin_cell()
+                                .store_coins(int(0))
+                                .store_coins(int(0))
+                                .store_coins(int(request.json['price']))
+                                .end_cell()
+                        )
+                        .store_maybe_ref(royalties_dict.end_dict())
+                        .store_uint(0, 1)
+                        .end_cell()
+                )
+                .store_ref(
+                    begin_cell()
+                        .store_ref(
+                            begin_cell()
+                                .store_bytes(f"{PROJECT_HOST}/api/v1.5/storage/{metadata_content.cid.serialize_v2(include_accept_type=True)}".encode())
+                                .end_cell()
+                        )
+                        .store_ref(
+                            begin_cell()
+                                .store_ref(begin_cell().store_bytes(f"{encrypted_content_cid.serialize_v2()}".encode()).end_cell())
+                                .store_ref(begin_cell().store_bytes(f"{image_content_cid.serialize_v2() if image_content_cid else ''}".encode()).end_cell())
+                                .store_ref(begin_cell().store_bytes(f"{metadata_content.cid.serialize_v2()}".encode()).end_cell())
+                                .end_cell()
+                        )
+                        .end_cell()
+                )
+                .end_cell()
+        )
+
         return response.json({
             'address': platform.address.to_string(1, 1, 1),
             'amount': str(int(0.03 * 10 ** 9)),
-            'payload': b64encode(
-                begin_cell()
-                    .store_uint(0x5491d08c, 32)
-                    .store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
-                    .store_uint(0, 2)
-                    .store_ref(
-                        begin_cell()
-                            .store_ref(
-                                begin_cell()
-                                    .store_coins(int(0))
-                                    .store_coins(int(0))
-                                    .store_coins(int(request.json['price']))
-                                    .end_cell()
-                            )
-                            .store_maybe_ref(royalties_dict.end_dict())
-                            .store_uint(0, 1)
-                            .end_cell()
-                    )
-                    .store_ref(
-                        begin_cell()
-                            .store_ref(
-                                begin_cell()
-                                    .store_bytes(f"{PROJECT_HOST}/api/v1.5/storage/{metadata_content.cid.serialize_v2(include_accept_type=True)}".encode())
-                                    .end_cell()
-                            )
-                            .store_ref(
-                                begin_cell()
-                                    .store_ref(begin_cell().store_bytes(f"{encrypted_content_cid.serialize_v2()}".encode()).end_cell())
-                                    .store_ref(begin_cell().store_bytes(f"{image_content_cid.serialize_v2() if image_content_cid else ''}".encode()).end_cell())
-                                    .store_ref(begin_cell().store_bytes(f"{metadata_content.cid.serialize_v2()}".encode()).end_cell())
-                                    .end_cell()
-                            )
-                            .end_cell()
-                    )
-                    .end_cell().to_boc(False)
-            ).decode()
+            'payload': b64encode(payload_cell.to_boc(False)).decode()
         })
     except BaseException as e:
         make_log("Blockchain", f"Error while sending new content message: {e}" + '\n' + traceback.format_exc(), level='error')
@@ -356,14 +289,15 @@ async def s_api_v1_blockchain_send_purchase_content_message(request):
     license_exist = (await request.ctx.db_session.execute(select(UserContent).where(
         UserContent.onchain_address == request.json['content_address']
     ))).scalars().first()
-    if license_exist:
-        from app.core.content.content_id import ContentId
-        _cid = ContentId.deserialize(license_exist.content.cid.serialize_v2())
-        r_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
+    from app.core.content.content_id import ContentId
+
+    if license_exist and license_exist.content_id:
+        r_content = (await request.ctx.db_session.execute(select(StoredContent).where(
+            StoredContent.id == license_exist.content_id
+        ))).scalars().first()
     else:
-        from app.core.content.content_id import ContentId
-        _cid = ContentId.deserialize(request.json['content_address'])
-        r_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
+        requested_cid = ContentId.deserialize(request.json['content_address'])
+        r_content = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == requested_cid.content_hash_b58))).scalars().first()
     
     async def open_content_async(session, sc: StoredContent):
         if not sc.encrypted:

app/api/routes/admin.py

@@ -19,6 +19,7 @@ from app.core._config import (
     BACKEND_DATA_DIR_HOST,
     BACKEND_LOGS_DIR_HOST,
     LOG_DIR,
+    CLIENT_TELEGRAM_BOT_USERNAME,
     PROJECT_HOST,
     UPLOADS_DIR,
 )
@@ -35,6 +36,15 @@ from app.core.models.content_v3 import (
 )
 from app.core.models.my_network import KnownNode
 from app.core.models.tasks import BlockchainTask
+from app.core.models.node_storage import StoredContent
+from app.core.models.user import User
+from app.core.models.content.user_content import UserContent
+from app.core._utils.share_links import build_content_links
+from app.core.content.content_id import ContentId
+
+MIN_ONCHAIN_INDEX = int(os.getenv("MIN_ONCHAIN_INDEX", "8"))
+
+ALLOWED_UPLOAD_FILTERS = {"all", "issues", "processing", "ready", "unindexed"}
 
 ADMIN_COOKIE_NAME = os.getenv('ADMIN_COOKIE_NAME', 'admin_session')
 ADMIN_COOKIE_MAX_AGE = int(os.getenv('ADMIN_COOKIE_MAX_AGE', '172800'))  # 48h default
@@ -148,6 +158,42 @@ def _service_states(request) -> List[Dict[str, Any]]:
     return items
 
 
+def _format_dt(value: Optional[datetime]) -> Optional[str]:
+    return value.isoformat() + 'Z' if isinstance(value, datetime) else None
+
+
+def _extract_file_hash(local_path: Optional[str]) -> Optional[str]:
+    if not local_path:
+        return None
+    name = Path(local_path).name
+    return name or None
+
+
+def _storage_download_url(file_hash: Optional[str]) -> Optional[str]:
+    if not file_hash:
+        return None
+    return f"{PROJECT_HOST}/api/v1.5/storage/{file_hash}"
+
+
+def _pick_primary_download(candidates: List[tuple[str, Optional[str], Optional[int]]]) -> Optional[str]:
+    priority = (
+        'decrypted_high',
+        'decrypted_low',
+        'decrypted_preview',
+        'high',
+        'low',
+        'preview',
+    )
+    for target in priority:
+        for kind, url, _ in candidates:
+            if kind == target and url:
+                return url
+    for _, url, _ in candidates:
+        if url:
+            return url
+    return None
+
+
 async def s_api_v1_admin_login(request):
     token = os.getenv('ADMIN_API_TOKEN')
     if not token:
@@ -309,6 +355,31 @@ async def s_api_v1_admin_uploads(request):
 
     session = request.ctx.db_session
 
+    raw_filter = (request.args.get('filter') or '').lower()
+    raw_filters: List[str] = []
+    if raw_filter:
+        raw_filters = [item.strip() for item in raw_filter.split(',') if item.strip()]
+    effective_filters = [item for item in raw_filters if item in ALLOWED_UPLOAD_FILTERS and item != 'all']
+
+    search_query = (request.args.get('search') or '').strip()
+    search_lower = search_query.lower()
+
+    try:
+        limit = int(request.args.get('limit') or 50)
+    except Exception:
+        limit = 50
+    limit = max(1, min(limit, 200))
+
+    try:
+        scan_limit = int(request.args.get('scan') or 0)
+    except Exception:
+        scan_limit = 0
+    if scan_limit <= 0:
+        scan_default = max(limit, 100 if (effective_filters or search_lower) else limit)
+        scan_limit = min(max(scan_default, limit), 500)
+    else:
+        scan_limit = max(limit, min(scan_limit, 500))
+
     counts_rows = (await session.execute(
         select(UploadSession.state, func.count()).group_by(UploadSession.state)
     )).all()
@@ -332,10 +403,337 @@ async def s_api_v1_admin_uploads(request):
         for row in recent_rows
     ]
 
+    content_rows = (await session.execute(
+        select(EncryptedContent).order_by(EncryptedContent.created_at.desc()).limit(scan_limit)
+    )).scalars().all()
+
+    content_ids = [row.id for row in content_rows]
+    encrypted_cids = [row.encrypted_cid for row in content_rows]
+
+    derivatives_map: Dict[int, list[ContentDerivative]] = {cid: [] for cid in content_ids}
+    if content_ids:
+        derivative_rows = (await session.execute(
+            select(ContentDerivative).where(ContentDerivative.content_id.in_(content_ids))
+        )).scalars().all()
+        for derivative in derivative_rows:
+            derivatives_map.setdefault(derivative.content_id, []).append(derivative)
+
+    ipfs_map: Dict[int, Optional[IpfsSync]] = {}
+    if content_ids:
+        ipfs_rows = (await session.execute(
+            select(IpfsSync).where(IpfsSync.content_id.in_(content_ids))
+        )).scalars().all()
+        for sync in ipfs_rows:
+            ipfs_map[sync.content_id] = sync
+
+    uploads_map: Dict[str, List[UploadSession]] = {cid: [] for cid in encrypted_cids}
+    if encrypted_cids:
+        uploads_for_content = (await session.execute(
+            select(UploadSession).where(UploadSession.encrypted_cid.in_(encrypted_cids))
+        )).scalars().all()
+        for upload in uploads_for_content:
+            uploads_map.setdefault(upload.encrypted_cid, []).append(upload)
+        for chain in uploads_map.values():
+            chain.sort(key=lambda u: (u.updated_at or u.created_at or datetime.min))
+
+    stored_map: Dict[str, StoredContent] = {}
+    stored_by_id: Dict[int, StoredContent] = {}
+    if encrypted_cids:
+        stored_rows = (await session.execute(
+            select(StoredContent).where(StoredContent.content_id.in_(encrypted_cids))
+        )).scalars().all()
+        for stored in stored_rows:
+            stored_map[stored.content_id] = stored
+            stored_by_id[stored.id] = stored
+
+    user_map: Dict[int, User] = {}
+    user_ids = {stored.user_id for stored in stored_map.values() if stored.user_id}
+    if user_ids:
+        user_rows = (await session.execute(select(User).where(User.id.in_(user_ids)))).scalars().all()
+        for user in user_rows:
+            user_map[user.id] = user
+
+    license_counts: Dict[int, int] = {}
+    stored_ids = list(stored_by_id.keys())
+    if stored_ids:
+        license_rows = (await session.execute(
+            select(UserContent.content_id, func.count())
+            .where(UserContent.content_id.in_(stored_ids))
+            .group_by(UserContent.content_id)
+        )).all()
+        for content_id, count in license_rows:
+            license_counts[int(content_id)] = int(count)
+
+    contents_payload: List[Dict[str, Any]] = []
+    category_totals: Dict[str, int] = {key: 0 for key in ALLOWED_UPLOAD_FILTERS if key != 'all'}
+    matched_total = 0
+    for content in content_rows:
+        derivatives = derivatives_map.get(content.id, [])
+        attempts: Dict[str, int] = defaultdict(int)
+        derivative_entries: List[Dict[str, Any]] = []
+        summary: Dict[str, int] = defaultdict(int)
+        download_candidates: List[tuple[str, Optional[str], Optional[int]]] = []
+
+        for derivative in sorted(derivatives, key=lambda item: item.created_at or datetime.min):
+            summary[derivative.status] += 1
+            attempts[derivative.kind] += 1
+            file_hash = _extract_file_hash(derivative.local_path)
+            download_url = _storage_download_url(file_hash)
+            derivative_entries.append({
+                'kind': derivative.kind,
+                'status': derivative.status,
+                'size_bytes': derivative.size_bytes,
+                'error': derivative.error,
+                'created_at': _format_dt(derivative.created_at),
+                'updated_at': _format_dt(derivative.last_access_at or derivative.created_at),
+                'attempts': attempts[derivative.kind],
+                'download_url': download_url,
+            })
+            download_candidates.append((derivative.kind, download_url, derivative.size_bytes))
+
+        conversion_state = None
+        if summary.get('ready'):
+            conversion_state = 'ready'
+        elif summary.get('processing'):
+            conversion_state = 'processing'
+        elif summary.get('pending'):
+            conversion_state = 'pending'
+        elif summary.get('failed'):
+            conversion_state = 'failed'
+
+        upload_chain = uploads_map.get(content.encrypted_cid, [])
+        latest_upload = upload_chain[-1] if upload_chain else None
+        upload_history = [
+            {
+                'state': entry.state,
+                'at': _format_dt(entry.updated_at or entry.created_at),
+                'error': entry.error,
+                'filename': entry.filename,
+            }
+            for entry in upload_chain
+        ]
+
+        ipfs_sync = ipfs_map.get(content.id)
+
+        stored = stored_map.get(content.encrypted_cid)
+        metadata_cid = None
+        content_hash = None
+        stored_payload: Optional[Dict[str, Any]] = None
+        blockchain_payload: Optional[Dict[str, Any]] = None
+        if stored:
+            metadata_cid = (stored.meta or {}).get('metadata_cid')
+            content_hash = stored.hash
+            download_candidates.append(('stored', stored.web_url, None))
+            stored_payload = {
+                'stored_id': stored.id,
+                'type': stored.type,
+                'owner_address': stored.owner_address,
+                'user_id': stored.user_id,
+                'status': stored.status,
+                'content_url': stored.web_url,
+                'download_url': stored.web_url,
+                'created': _format_dt(stored.created),
+                'updated': _format_dt(stored.updated),
+            }
+            if stored.user_id and stored.user_id in user_map:
+                user = user_map[stored.user_id]
+                stored_payload['user'] = {
+                    'id': user.id,
+                    'telegram_id': user.telegram_id,
+                    'username': user.username,
+                    'first_name': (user.meta or {}).get('first_name') if user.meta else None,
+                    'last_name': (user.meta or {}).get('last_name') if user.meta else None,
+                }
+            blockchain_payload = {
+                'onchain_index': stored.onchain_index,
+                'item_address': (stored.meta or {}).get('item_address'),
+                'indexed': stored.onchain_index is not None and stored.onchain_index >= MIN_ONCHAIN_INDEX,
+                'license_count': license_counts.get(stored.id, 0),
+            }
+
+        else:
+            try:
+                cid_obj = ContentId.deserialize(content.encrypted_cid)
+                content_hash = cid_obj.content_hash_b58
+            except Exception:
+                content_hash = None
+
+        share_target = None
+        if stored:
+            try:
+                share_target = stored.cid.serialize_v2()
+            except Exception:
+                share_target = content.encrypted_cid
+        else:
+            share_target = content.encrypted_cid
+
+        _, startapp_url, web_view_url = build_content_links(
+            share_target,
+            None,
+            project_host=PROJECT_HOST,
+            bot_username=CLIENT_TELEGRAM_BOT_USERNAME,
+        )
+
+        primary_download = _pick_primary_download(download_candidates)
+        derivative_downloads = [
+            {
+                'kind': kind,
+                'url': url,
+                'size_bytes': size_bytes,
+            }
+            for kind, url, size_bytes in download_candidates
+            if url
+        ]
+
+        upload_state_norm = (latest_upload.state or '').lower() if latest_upload else ''
+        conversion_state_norm = (conversion_state or '').lower() if conversion_state else ''
+        ipfs_state_norm = (ipfs_sync.pin_state or '').lower() if (ipfs_sync and ipfs_sync.pin_state) else ''
+        derivative_states_norm = [(derivative.status or '').lower() for derivative in derivatives]
+        status_values = [upload_state_norm, conversion_state_norm, ipfs_state_norm] + derivative_states_norm
+
+        has_issue = any(
+            value and ("fail" in value or "error" in value or "timeout" in value)
+            for value in status_values
+        )
+        if not has_issue and any(event.get('error') for event in upload_history):
+            has_issue = True
+        if not has_issue and any(derivative.error for derivative in derivatives):
+            has_issue = True
+        if not has_issue and ipfs_sync and ipfs_sync.pin_error:
+            has_issue = True
+
+        is_onchain_indexed = bool(blockchain_payload and blockchain_payload.get('indexed'))
+        is_unindexed = not is_onchain_indexed
+
+        conversion_done = (
+            summary.get('ready', 0) > 0
+            or conversion_state_norm in ('ready', 'converted')
+            or any(state in ('ready', 'converted', 'complete') for state in derivative_states_norm)
+        )
+        ipfs_done = ipfs_state_norm in ('pinned', 'ready')
+
+        is_ready = not has_issue and conversion_done and (ipfs_done or not ipfs_sync) and is_onchain_indexed
+
+        processing_tokens = ('process', 'pending', 'queue', 'upload', 'pin', 'sync')
+        has_processing_keywords = any(
+            value and any(token in value for token in processing_tokens)
+            for value in status_values
+        )
+
+        categories = set()
+        if has_issue:
+            categories.add('issues')
+        if is_ready:
+            categories.add('ready')
+        if is_unindexed:
+            categories.add('unindexed')
+
+        is_processing = not is_ready and not has_issue and has_processing_keywords
+        if is_processing:
+            categories.add('processing')
+        if not is_ready and not has_issue and 'processing' not in categories:
+            categories.add('processing')
+
+        flags = {
+            'issues': 'issues' in categories,
+            'processing': 'processing' in categories,
+            'ready': 'ready' in categories,
+            'unindexed': 'unindexed' in categories,
+        }
+
+        search_parts: List[Any] = [
+            content.title,
+            content.description,
+            content.encrypted_cid,
+            metadata_cid,
+            content_hash,
+        ]
+        if blockchain_payload:
+            search_parts.append(blockchain_payload.get('item_address') or '')
+        if stored_payload:
+            search_parts.append(stored_payload.get('owner_address') or '')
+            user_info = stored_payload.get('user') or {}
+            search_parts.extend(
+                [
+                    str(user_info.get('id') or ''),
+                    str(user_info.get('telegram_id') or ''),
+                    user_info.get('username') or '',
+                    user_info.get('first_name') or '',
+                    user_info.get('last_name') or '',
+                ]
+            )
+        search_blob = ' '.join(str(part) for part in search_parts if part).lower()
+
+        matches_filter = (not effective_filters) or any(cat in categories for cat in effective_filters)
+        matches_search = (not search_lower) or (search_lower in search_blob)
+
+        if not matches_filter or not matches_search:
+            continue
+
+        matched_total += 1
+        for cat in categories:
+            if cat in category_totals:
+                category_totals[cat] += 1
+
+        if len(contents_payload) >= limit:
+            continue
+
+        contents_payload.append({
+            'encrypted_cid': content.encrypted_cid,
+            'metadata_cid': metadata_cid,
+            'content_hash': content_hash,
+            'title': content.title,
+            'description': content.description,
+            'content_type': content.content_type,
+            'size': {
+                'encrypted': content.enc_size_bytes,
+                'plain': content.plain_size_bytes,
+            },
+            'created_at': _format_dt(content.created_at),
+            'updated_at': _format_dt(content.updated_at),
+            'status': {
+                'upload_state': latest_upload.state if latest_upload else None,
+                'conversion_state': conversion_state,
+                'ipfs_state': ipfs_sync.pin_state if ipfs_sync else None,
+                'onchain': blockchain_payload,
+            },
+            'upload_history': upload_history,
+            'derivative_summary': dict(summary),
+            'derivatives': derivative_entries,
+            'ipfs': (
+                {
+                    'pin_state': ipfs_sync.pin_state,
+                    'pin_error': ipfs_sync.pin_error,
+                    'bytes_total': ipfs_sync.bytes_total,
+                    'bytes_fetched': ipfs_sync.bytes_fetched,
+                    'pinned_at': _format_dt(ipfs_sync.pinned_at),
+                    'updated_at': _format_dt(ipfs_sync.updated_at),
+                }
+                if ipfs_sync else None
+            ),
+            'stored': stored_payload,
+            'links': {
+                'web_view': web_view_url,
+                'start_app': startapp_url,
+                'api_view': f"{PROJECT_HOST}/api/v1/content.view/{share_target}",
+                'download_primary': primary_download,
+                'download_derivatives': derivative_downloads,
+            },
+            'flags': flags,
+        })
+
     payload = {
         'total': total,
         'states': counts,
         'recent': recent,
+        'contents': contents_payload,
+        'matching_total': matched_total,
+        'filter': effective_filters or ['all'],
+        'search': search_query or None,
+        'limit': limit,
+        'scan': scan_limit,
+        'scanned': len(content_rows),
+        'category_totals': category_totals,
     }
     return response.json(payload)
 
@@ -347,10 +745,15 @@ async def s_api_v1_admin_system(request):
     session = request.ctx.db_session
 
     config_rows = (await session.execute(select(ServiceConfigValue).order_by(ServiceConfigValue.key))).scalars().all()
-    config_payload = [
-        {'key': row.key, 'value': row.value, 'raw': row.packed_value}
-        for row in config_rows
-    ]
+    config_payload = []
+    for row in config_rows:
+        key_lower = (row.key or '').lower()
+        masked = ('private' in key_lower and 'key' in key_lower) or ('seed' in key_lower)
+        config_payload.append({
+            'key': row.key,
+            'value': '*** hidden ***' if masked else row.value,
+            'raw': None if masked else row.packed_value,
+        })
 
     env_summary = {
         'PROJECT_NAME': os.getenv('PROJECT_NAME'),

app/api/routes/auth.py

@@ -61,6 +61,21 @@ async def s_api_v1_auth_twa(request):
         )).scalars().first()
         assert known_user, "User not created"
 
+    meta_updated = False
+    if not (known_user.meta or {}).get('ref_id'):
+        known_user.ensure_ref_id()
+        meta_updated = True
+
+    incoming_ref_id = auth_data.get('ref_id')
+    stored_ref_id = (known_user.meta or {}).get('ref_id')
+    if incoming_ref_id and incoming_ref_id != stored_ref_id:
+        if (known_user.meta or {}).get('referrer_id') != incoming_ref_id:
+            known_user.meta = {
+                **(known_user.meta or {}),
+                'referrer_id': incoming_ref_id
+            }
+            meta_updated = True
+
     new_user_key = await known_user.create_api_token_v1(request.ctx.db_session, "USER_API_V1")
     if auth_data['ton_proof']:
         try:
@@ -116,6 +131,8 @@ async def s_api_v1_auth_twa(request):
         )
     ).order_by(WalletConnection.created.desc()))).scalars().first()
     known_user.last_use = datetime.now()
+    if meta_updated:
+        known_user.updated = datetime.now()
     await request.ctx.db_session.commit()
 
     return response.json({

app/api/routes/content.py

@@ -10,7 +10,8 @@ from app.core.models.keys import KnownKey
 from app.core.models import StarsInvoice
 from app.core.models.content.user_content import UserContent
 from app.core._config import CLIENT_TELEGRAM_API_KEY, PROJECT_HOST
-from app.core.models.content_v3 import EncryptedContent as ECv3, ContentDerivative as CDv3
+from app.core.models.content_v3 import EncryptedContent as ECv3, ContentDerivative as CDv3, UploadSession
+from app.core.content.content_id import ContentId
 import json
 import uuid
 
@@ -50,8 +51,15 @@ async def s_api_v1_content_view(request, content_address: str):
     license_exist = (await request.ctx.db_session.execute(
         select(UserContent).where(UserContent.onchain_address == content_address)
     )).scalars().first()
+    license_address = None
     if license_exist:
-        content_address = license_exist.content.cid.serialize_v2()
+        license_address = license_exist.onchain_address
+        if license_exist.content_id:
+            linked_content = (await request.ctx.db_session.execute(
+                select(StoredContent).where(StoredContent.id == license_exist.content_id)
+            )).scalars().first()
+            if linked_content:
+                content_address = linked_content.cid.serialize_v2()
 
     from app.core.content.content_id import ContentId
     cid = ContentId.deserialize(content_address)
@@ -74,9 +82,12 @@ async def s_api_v1_content_view(request, content_address: str):
         return {'encrypted_content': encrypted, 'decrypted_content': decrypted, 'content_type': content_type}
     content = await open_content_async(request.ctx.db_session, r_content)
 
+    master_address = content['encrypted_content'].meta.get('item_address', '')
     opts = {
         'content_type': content['content_type'],  # возможно с ошибками, нужно переделать на ffprobe
-        'content_address': content['encrypted_content'].meta.get('item_address', '')
+        'content_address': license_address or master_address,
+        'license_address': license_address,
+        'master_address': master_address,
     }
     if content['encrypted_content'].key_id:
         known_key = (await request.ctx.db_session.execute(
@@ -157,59 +168,168 @@ async def s_api_v1_content_view(request, content_address: str):
                     'amount': stars_cost,
                 }
 
-    display_options = {
-        'content_url': None,
-    }
+    display_options = {'content_url': None}
 
     if have_access:
         opts['have_licenses'].append('listen')
 
-    converted_content = content['encrypted_content'].meta.get('converted_content')
-    if converted_content:
-        user_content_option = 'low_preview'
-        if have_access:
-            user_content_option = 'low'
+    enc_cid = content['encrypted_content'].meta.get('content_cid') or content['encrypted_content'].meta.get('encrypted_cid')
+    ec_v3 = None
+    derivative_rows = []
+    if enc_cid:
+        ec_v3 = (await request.ctx.db_session.execute(select(ECv3).where(ECv3.encrypted_cid == enc_cid))).scalars().first()
+        if ec_v3:
+            derivative_rows = (await request.ctx.db_session.execute(select(CDv3).where(CDv3.content_id == ec_v3.id))).scalars().all()
 
-        converted_content = (await request.ctx.db_session.execute(select(StoredContent).where(
-            StoredContent.hash == converted_content[user_content_option]
-        ))).scalars().first()
-        if converted_content:
-            display_options['content_url'] = converted_content.web_url
-            opts['content_ext'] = converted_content.filename.split('.')[-1]
+    upload_row = None
+    if enc_cid:
+        upload_row = (await request.ctx.db_session.execute(select(UploadSession).where(UploadSession.encrypted_cid == enc_cid))).scalars().first()
+
+    converted_meta_map = dict(content['encrypted_content'].meta.get('converted_content') or {})
+
+    derivative_latest = {}
+    if derivative_rows:
+        derivative_sorted = sorted(derivative_rows, key=lambda row: row.created_at or datetime.min)
+        for row in derivative_sorted:
+            derivative_latest[row.kind] = row
+
+    def _row_to_hash_and_url(row):
+        if not row or not row.local_path:
+            return None, None
+        file_hash = row.local_path.split('/')[-1]
+        return file_hash, f"{PROJECT_HOST}/api/v1.5/storage/{file_hash}"
+
+    chosen_row = None
+    if have_access:
+        for key in ('decrypted_low', 'decrypted_high'):
+            if key in derivative_latest:
+                chosen_row = derivative_latest[key]
+                break
     else:
-        # v3 fallback: use derivatives table linked via encrypted_cid from onchain meta
-        enc_cid = content['encrypted_content'].meta.get('content_cid') or content['encrypted_content'].meta.get('encrypted_cid')
-        if enc_cid:
-            ec = (await request.ctx.db_session.execute(select(ECv3).where(ECv3.encrypted_cid == enc_cid))).scalars().first()
-            if ec:
-                # choose preview for non-access; low for access
-                desired = ['decrypted_preview'] if not have_access else ['decrypted_low', 'decrypted_high']
-                rows = (await request.ctx.db_session.execute(select(CDv3).where(CDv3.content_id == ec.id, CDv3.status == 'ready'))).scalars().all()
-                chosen = None
-                for kind in desired:
-                    chosen = next((r for r in rows if r.kind == kind), None)
-                    if chosen:
-                        break
-                if chosen and chosen.local_path:
-                    h = chosen.local_path.split('/')[-1]
-                    display_options['content_url'] = f"{PROJECT_HOST}/api/v1.5/storage/{h}"
-                    opts['content_ext'] = (chosen.content_type or '').split('/')[-1] if chosen.content_type else None
+        for key in ('decrypted_preview', 'decrypted_low'):
+            if key in derivative_latest:
+                chosen_row = derivative_latest[key]
+                break
 
+    if chosen_row:
+        file_hash, url = _row_to_hash_and_url(chosen_row)
+        if url:
+            display_options['content_url'] = url
+            opts['content_ext'] = (chosen_row.content_type or '').split('/')[-1] if chosen_row.content_type else None
+            converted_meta_map.setdefault('low' if have_access else 'low_preview', file_hash)
+
+    if not display_options['content_url'] and converted_meta_map:
+        preference = ['low', 'high', 'low_preview'] if have_access else ['low_preview', 'low', 'high']
+        for key in preference:
+            hash_value = converted_meta_map.get(key)
+            if not hash_value:
+                continue
+            stored = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == hash_value))).scalars().first()
+            if stored:
+                display_options['content_url'] = stored.web_url
+                opts['content_ext'] = stored.filename.split('.')[-1]
+                break
+
+    # Metadata fallback
     content_meta = content['encrypted_content'].json_format()
-    from app.core.content.content_id import ContentId
+    content_metadata_json = None
     _mcid = content_meta.get('metadata_cid') or None
-    content_metadata = None
     if _mcid:
         _cid = ContentId.deserialize(_mcid)
         content_metadata = (await request.ctx.db_session.execute(select(StoredContent).where(StoredContent.hash == _cid.content_hash_b58))).scalars().first()
-    with open(content_metadata.filepath, 'r') as f:
-        content_metadata_json = json.loads(f.read())
+        if content_metadata:
+            try:
+                with open(content_metadata.filepath, 'r') as f:
+                    content_metadata_json = json.loads(f.read())
+            except Exception as exc:
+                make_log("Content", f"Can't read metadata file: {exc}", level='warning')
+
+    if not content_metadata_json:
+        fallback_name = (ec_v3.title if ec_v3 else None) or content_meta.get('title') or content_meta.get('cid')
+        fallback_description = (ec_v3.description if ec_v3 else '') or ''
+        content_metadata_json = {
+            'name': fallback_name or 'Без названия',
+            'description': fallback_description,
+            'downloadable': False,
+        }
+        cover_cid = content_meta.get('cover_cid')
+        if cover_cid:
+            content_metadata_json.setdefault('image', f"{PROJECT_HOST}/api/v1.5/storage/{cover_cid}")
 
     display_options['metadata'] = content_metadata_json
+
     opts['downloadable'] = content_metadata_json.get('downloadable', False)
-    if opts['downloadable']:
-        if not ('listen' in opts['have_licenses']):
-            opts['downloadable'] = False
+    if opts['downloadable'] and 'listen' not in opts['have_licenses']:
+        opts['downloadable'] = False
+
+    # Conversion status summary
+    conversion_summary = {}
+    conversion_details = []
+    derivative_summary_map = {}
+    for row in derivative_latest.values():
+        conversion_summary[row.status] = conversion_summary.get(row.status, 0) + 1
+        derivative_summary_map[row.kind] = row
+        conversion_details.append({
+            'kind': row.kind,
+            'status': row.status,
+            'size_bytes': row.size_bytes,
+            'content_type': row.content_type,
+            'error': row.error,
+            'updated_at': (row.last_access_at or row.created_at).isoformat() + 'Z' if (row.last_access_at or row.created_at) else None,
+        })
+
+    required_kinds = {'decrypted_low', 'decrypted_high'}
+    if ec_v3 and ec_v3.content_type.startswith('video/'):
+        required_kinds.add('decrypted_preview')
+
+    statuses_by_kind = {kind: row.status for kind, row in derivative_summary_map.items() if kind in required_kinds}
+    conversion_state = 'pending'
+    if required_kinds and all(statuses_by_kind.get(kind) == 'ready' for kind in required_kinds):
+        conversion_state = 'ready'
+    elif any(statuses_by_kind.get(kind) == 'failed' for kind in required_kinds):
+        conversion_state = 'failed'
+    elif any(statuses_by_kind.get(kind) in ('processing', 'pending') for kind in required_kinds):
+        conversion_state = 'processing'
+    elif statuses_by_kind:
+        conversion_state = 'partial'
+
+    if display_options['content_url']:
+        conversion_state = 'ready'
+
+    upload_info = None
+    if upload_row:
+        upload_info = {
+            'id': upload_row.id,
+            'state': upload_row.state,
+            'error': upload_row.error,
+            'created_at': upload_row.created_at.isoformat() + 'Z' if upload_row.created_at else None,
+            'updated_at': upload_row.updated_at.isoformat() + 'Z' if upload_row.updated_at else None,
+        }
+
+    final_state = 'ready' if display_options['content_url'] else None
+    if final_state != 'ready':
+        upload_state = upload_row.state if upload_row else None
+        if conversion_state == 'failed' or upload_state in ('failed', 'conversion_failed'):
+            final_state = 'failed'
+        elif conversion_state in ('processing', 'partial') or upload_state in ('processing', 'pinned'):
+            final_state = 'processing'
+        else:
+            final_state = 'uploaded'
+
+    conversion_info = {
+        'state': conversion_state,
+        'summary': conversion_summary,
+        'details': conversion_details,
+        'required_kinds': list(required_kinds),
+    }
+
+    opts['conversion'] = conversion_info
+    opts['upload'] = upload_info
+    opts['status'] = {
+        'state': final_state,
+        'conversion_state': conversion_state,
+        'upload_state': upload_info['state'] if upload_info else None,
+    }
 
     return response.json({
         **opts,

app/api/routes/upload_status.py

@@ -1,5 +1,8 @@
 from sanic import response
-from app.core.models.content_v3 import UploadSession
+from sqlalchemy import select
+
+from app.core.models.content_v3 import UploadSession, EncryptedContent, ContentDerivative
+from app.core._utils.resolve_content import resolve_content
 
 
 async def s_api_v1_upload_status(request, upload_id: str):
@@ -7,11 +10,45 @@ async def s_api_v1_upload_status(request, upload_id: str):
     row = await session.get(UploadSession, upload_id)
     if not row:
         return response.json({"error": "NOT_FOUND"}, status=404)
+
+    encrypted_hash = None
+    conversion = {"state": "not_started", "details": []}
+
+    if row.encrypted_cid:
+        cid_obj, err = resolve_content(row.encrypted_cid)
+        if not err:
+            encrypted_hash = cid_obj.content_hash_b58
+        ec = (await session.execute(select(EncryptedContent).where(EncryptedContent.encrypted_cid == row.encrypted_cid))).scalars().first()
+        if ec:
+            derivative_rows = (await session.execute(
+                select(ContentDerivative.kind, ContentDerivative.status).where(ContentDerivative.content_id == ec.id)
+            )).all()
+            details = [
+                {"kind": kind, "status": status}
+                for kind, status in derivative_rows
+            ]
+            required = {"decrypted_high", "decrypted_low"}
+            if ec.preview_enabled and ec.content_type.startswith("video/"):
+                required.add("decrypted_preview")
+            statuses = {kind: status for kind, status in derivative_rows}
+            if required and all(statuses.get(k) == "ready" for k in required):
+                conv_state = "ready"
+            elif any(statuses.get(k) == "failed" for k in required):
+                conv_state = "failed"
+            elif any(statuses.get(k) in ("processing", "pending") for k in required):
+                conv_state = "processing"
+            elif required:
+                conv_state = "pending"
+            else:
+                conv_state = "not_started"
+            conversion = {"state": conv_state, "details": details}
+
     return response.json({
         "id": row.id,
         "state": row.state,
         "encrypted_cid": row.encrypted_cid,
+        "encrypted_hash": encrypted_hash,
         "size_bytes": row.size_bytes,
         "error": row.error,
+        "conversion": conversion,
     })
-

app/api/routes/upload_tus.py

@@ -6,16 +6,21 @@ import os
 from datetime import datetime
 from typing import Dict, Any
 
+import aiofiles
 from base58 import b58encode
 from sanic import response
 
+from app.core._config import UPLOADS_DIR
 from app.core._secrets import hot_pubkey
 from app.core.crypto.aes_gcm_stream import encrypt_file_to_encf, CHUNK_BYTES
 from app.core.crypto.keywrap import wrap_dek, KeyWrapError
 from app.core.ipfs_client import add_streamed_file
 from app.core.logger import make_log
 from app.core.models.content_v3 import EncryptedContent, ContentKey, IpfsSync, ContentIndexItem, UploadSession
+from app.core.models.node_storage import StoredContent
 from app.core.storage import db_session
+from app.core._utils.resolve_content import resolve_content
+from sqlalchemy import select
 
 
 def _b64(s: bytes) -> str:
@@ -139,6 +144,12 @@ async def s_api_v1_upload_tus_hook(request):
     except Exception:
         enc_size = None
 
+    encrypted_cid_obj, cid_err = resolve_content(encrypted_cid)
+    if cid_err:
+        make_log("tus-hook", f"Encrypted CID resolve failed: {cid_err}", level="error")
+        return response.json({"ok": False, "error": "INVALID_ENCRYPTED_CID"}, status=500)
+    encrypted_hash_b58 = encrypted_cid_obj.content_hash_b58
+
     # Persist records
     async with db_session() as session:
         ec = EncryptedContent(
@@ -165,6 +176,7 @@ async def s_api_v1_upload_tus_hook(request):
             allow_auto_grant=True,
         )
         session.add(ck)
+        await session.flush()
 
         sync = IpfsSync(
             content_id=ec.id,
@@ -175,6 +187,32 @@ async def s_api_v1_upload_tus_hook(request):
         )
         session.add(sync)
 
+        existing_encrypted_content = (await session.execute(
+            select(StoredContent).where(StoredContent.hash == encrypted_hash_b58)
+        )).scalars().first()
+        if not existing_encrypted_content:
+            placeholder_meta = {
+                'content_type': content_type,
+                'storage': 'ipfs',
+                'encrypted_cid': encrypted_cid,
+                'upload_id': upload_id,
+                'source': 'tusd'
+            }
+            encrypted_stored_content = StoredContent(
+                type="local/encrypted_ipfs",
+                hash=encrypted_hash_b58,
+                content_id=encrypted_cid,
+                filename=os.path.basename(file_path),
+                meta=placeholder_meta,
+                user_id=request.ctx.user.id if request.ctx.user else None,
+                owner_address=None,
+                encrypted=True,
+                decrypted_content_id=None,
+                key_id=None,
+                created=datetime.utcnow(),
+            )
+            session.add(encrypted_stored_content)
+
         # Publish signed index item
         item = {
             "encrypted_cid": encrypted_cid,
@@ -206,6 +244,9 @@ async def s_api_v1_upload_tus_hook(request):
             if us:
                 us.state = 'pinned'
                 us.encrypted_cid = encrypted_cid
+                us.error = None
+                if size:
+                    us.size_bytes = size
                 # prefer using IPFS for downstream conversion; remove staging
                 try:
                     if file_path and os.path.exists(file_path):
@@ -216,4 +257,15 @@ async def s_api_v1_upload_tus_hook(request):
                 await session.commit()
 
     make_log("tus-hook", f"Uploaded+encrypted {file_path} -> {encrypted_cid}")
+    placeholder_path = os.path.join(UPLOADS_DIR, encrypted_hash_b58)
+    if not os.path.exists(placeholder_path):
+        try:
+            async with aiofiles.open(placeholder_path, "wb") as ph:
+                await ph.write(json.dumps({
+                    "ipfs_cid": encrypted_cid,
+                    "note": "Encrypted payload stored in IPFS"
+                }).encode())
+        except Exception as e:
+            make_log("tus-hook", f"Failed to create placeholder for {encrypted_hash_b58}: {e}", level="warning")
+
     return response.json({"ok": True, "encrypted_cid": encrypted_cid, "upload_id": upload_id})