free upload & player

app/api/__init__.py

@@ -12,7 +12,7 @@ app.register_middleware(attach_user_to_request, "request")
 app.register_middleware(close_db_session, "response")
 
 from app.api.routes._index import s_index, s_favicon
-from app.api.routes._system import s_api_v1_node, s_api_system_version, s_api_system_send_status
+from app.api.routes._system import s_api_v1_node, s_api_system_version, s_api_system_send_status, s_api_v1_node_friendly
 from app.api.routes.auth import s_api_v1_auth_twa
 from app.api.routes.statics import s_api_tonconnect_manifest, s_api_platform_metadata
 from app.api.routes.node_storage import s_api_v1_storage_post, s_api_v1_storage_get, \
@@ -29,6 +29,7 @@ app.add_route(s_index, "/", methods=["GET", "OPTIONS"])
 app.add_route(s_favicon, "/favicon.ico", methods=["GET", "OPTIONS"])
 
 app.add_route(s_api_v1_node, "/api/v1/node", methods=["GET", "OPTIONS"])
+app.add_route(s_api_v1_node_friendly, "/api/v1/nodeFriendly", methods=["GET", "OPTIONS"])
 app.add_route(s_api_system_version, "/api/system.version", methods=["GET", "OPTIONS"])
 app.add_route(s_api_system_send_status, "/api/system.sendStatus", methods=["POST", "OPTIONS"])
 

app/api/routes/_blockchain.py

@@ -3,6 +3,7 @@ from datetime import datetime
 import traceback
 
 from sanic import response
+from sqlalchemy import and_
 from tonsdk.boc import begin_cell, begin_dict
 from tonsdk.utils import Address
 
@@ -18,6 +19,8 @@ from app.core.models.content.user_content import UserContent
 from app.core.models.node_storage import StoredContent
 from app.core.models._telegram import Wrapped_CBotChat
 from app.core._keyboards import get_inline_keyboard
+from app.core.models.promo import PromoAction
+from app.core.models.tasks import BlockchainTask
 
 
 def valid_royalty_params(royalty_params):
@@ -101,6 +104,92 @@ async def s_api_v1_blockchain_send_new_content_message(request):
             )
             i += 1
 
+        promo_free_upload_available = (
+            3 - (request.ctx.db_session.query(PromoAction).filter(
+                PromoAction.user_internal_id == request.ctx.user.id,
+                PromoAction.action_type == 'freeUpload',
+            ).count())
+        )
+        if request.ctx.db_session.query(BlockchainTask).filter(
+            and_(
+                BlockchainTask.user_id == request.ctx.user.id,
+                BlockchainTask.status != 'done',
+            )
+        ).first():
+            make_log("Blockchain", f"User {request.ctx.user.id} already has a pending task", level='warning')
+            promo_free_upload_available = 0
+
+        make_log("Blockchain", f"User {request.ctx.user.id} has {promo_free_upload_available} free uploads available", level='info')
+
+        if promo_free_upload_available > 0:
+            promo_action = PromoAction(
+                user_id = str(request.ctx.user.id),
+                user_internal_id=request.ctx.user.id,
+                action_type='freeUpload',
+                action_ref=str(encrypted_content_cid.content_hash),
+                created=datetime.now()
+            )
+            request.ctx.db_session.add(promo_action)
+
+            blockchain_task = BlockchainTask(
+                destination=platform.address.to_string(1, 1, 1),
+                payload=b64encode(
+                    begin_cell()
+                        .store_uint(0x5491d08c, 32)
+                        .store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
+                        .store_address(Address(request.ctx.user.wallet_address(request.ctx.db_session)))
+                        .store_ref(
+                            begin_cell()
+                                .store_ref(
+                                    begin_cell()
+                                        .store_coins(int(0))
+                                        .store_coins(int(0))
+                                        .store_coins(int(request.json['price']))
+                                        .end_cell()
+                                )
+                                .store_maybe_ref(royalties_dict.end_dict())
+                                .store_uint(0, 1)
+                                .end_cell()
+                        )
+                        .store_ref(
+                            begin_cell()
+                                .store_ref(
+                                    begin_cell()
+                                        .store_bytes(f"{PROJECT_HOST}/api/v1.5/storage/{metadata_content.cid.serialize_v2(include_accept_type=True)}".encode())
+                                        .end_cell()
+                                )
+                                .store_ref(
+                                    begin_cell()
+                                        .store_ref(begin_cell().store_bytes(f"{encrypted_content_cid.serialize_v2()}".encode()).end_cell())
+                                        .store_ref(begin_cell().store_bytes(f"{image_content_cid.serialize_v2() if image_content_cid else ''}".encode()).end_cell())
+                                        .store_ref(begin_cell().store_bytes(f"{metadata_content.cid.serialize_v2()}".encode()).end_cell())
+                                        .end_cell()
+                                )
+                                .end_cell()
+                        )
+                        .end_cell().to_boc(False)
+                ).decode(),
+                epoch=None, seqno=None,
+                created = datetime.now(),
+                status='wait',
+                user_id = request.ctx.user.id
+            )
+            request.ctx.db_session.add(blockchain_task)
+            request.ctx.db_session.commit()
+
+            await request.ctx.user_uploader_wrapper.send_message(
+                request.ctx.user.translated('p_uploadContentTxPromo').format(
+                    title=content_title,
+                    free_count=(promo_free_upload_available - 1)
+                ), message_type='hint', message_meta={
+                    'encrypted_content_hash': b58encode(encrypted_content_cid.content_hash).decode(),
+                    'hint_type': 'uploadContentTxRequested'
+                }
+            )
+            return response.json({
+                'promoUpload': True,
+            })
+
         await request.ctx.user_uploader_wrapper.send_message(
             request.ctx.user.translated('p_uploadContentTxRequested').format(
                 title=content_title,
@@ -117,6 +206,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
                 begin_cell()
                     .store_uint(0x5491d08c, 32)
                     .store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
+                    .store_uint(0, 2)
                     .store_ref(
                         begin_cell()
                             .store_ref(

app/api/routes/_system.py

@@ -38,6 +38,27 @@ async def s_api_v1_node(request):  # /api/v1/node
         }
     })
 
+async def s_api_v1_node_friendly(request):
+    last_known_index = request.ctx.db_session.query(StoredContent).filter(
+        StoredContent.onchain_index != None
+    ).order_by(StoredContent.onchain_index.desc()).first()
+    last_known_index = last_known_index.onchain_index if last_known_index else 0
+    last_known_index = max(last_known_index, 0)
+    response_plain_text = f"""
+    Node address: {service_wallet.address.to_string(1, 1, 1)}
+    Node ID: {b58encode(hot_pubkey).decode()}
+    Master address: {platform.address.to_string(1, 1, 1)}
+    Indexer height: {last_known_index}
+    Services:
+    """
+    for service_key, service in request.app.ctx.memory.known_states.items():
+        response_plain_text += f"""
+            {service_key}:
+                status: {service['status'] if (service['timestamp'] and (datetime.now() - service['timestamp']).total_seconds() < 30) else 'not working: timeout'}
+                delay: {round((datetime.now() - service['timestamp']).total_seconds(), 3) if service['timestamp'] else -1}
+        """
+    return response.text(response_plain_text, content_type='text/plain')
+
 
 async def s_api_system_send_status(request):
     if not request.json:

app/core/background/ton_service.py

@@ -1,13 +1,22 @@
 import asyncio
+from base64 import b64decode
 import os
+import traceback
+import httpx
 
-from tonsdk.boc import begin_cell
+from sqlalchemy import and_, func
+from tonsdk.boc import begin_cell, Cell
+from tonsdk.contract.wallet import Wallets
+from tonsdk.utils import HighloadQueryId
+from datetime import datetime, timedelta
 
 from app.core._blockchain.ton.platform import platform
 from app.core._blockchain.ton.toncenter import toncenter
+from app.core.models.tasks import BlockchainTask
 from app.core._config import MY_FUND_ADDRESS
 from app.core._secrets import service_wallet
 from app.core._utils.send_status import send_status
+from app.core.storage import db_session
 from app.core.logger import make_log
 
 
@@ -75,16 +84,194 @@ async def main_fn(memory):
         await asyncio.sleep(15)
         return await main_fn(memory)
     
+    highload_wallet = Wallets.ALL['hv3'](
+        private_key=service_wallet.options['private_key'],
+        public_key=service_wallet.options['public_key'],
+        wc=0
+    )
+    make_log("TON", f"Highload wallet address: {highload_wallet.address.to_string(1, 1, 1)}", level="info")
+    highload_state = await toncenter.get_account(highload_wallet.address.to_string(1, 1, 1))
+    if int(highload_state.get('balance', '0')) / 1e9 < 0.05:
+        make_log("TON", "Highload wallet balance is less than 0.05, send topup transaction..", level="info")
+        await toncenter.send_boc(
+            service_wallet.create_transfer_message(
+                [{
+                    'address': highload_wallet.address.to_string(1, 1, 0),
+                    'amount': int(0.08 * 10 ** 9),
+                    'send_mode': 1,
+                    'payload': begin_cell().store_uint(0, 32).end_cell()
+                }], sw_seqno_value
+            )['message'].to_boc(False)
+        )
+        await send_status("ton_daemon", "working: topup highload wallet")
+        await asyncio.sleep(15)
+        return await main_fn(memory)
+    
+    if not highload_state.get('code'):
+        make_log("TON", "Highload wallet contract is not deployed, send deploy transaction..", level="info")
+        created_at_ts = int(datetime.utcnow().timestamp()) - 60
+        await toncenter.send_boc(
+            highload_wallet.create_transfer_message(
+                service_wallet.address.to_string(1, 1, 1),
+                1, HighloadQueryId.from_seqno(0), created_at_ts, send_mode=1, payload="hello world", need_deploy=True
+            )['message'].to_boc(False)
+        )
+        await send_status("ton_daemon", "working: deploying highload wallet")
+        await asyncio.sleep(15)
+        return await main_fn(memory)
+
     while True:
         try:
             sw_seqno_value = await get_sw_seqno()
             make_log("TON", f"Service running ({sw_seqno_value})", level="debug")
 
-            # with db_session() as session:
-            #     for stored_content in session.query(StoredContent).filter(StoredContent.uploaded == False).all():
-            #         pass
+            with db_session() as session:
+                # Проверка отправленных сообщений
+                async def process_incoming_transaction(transaction: dict):
+                    transaction_hash = transaction['transaction_id']['hash']
+                    transaction_lt = str(transaction['transaction_id']['lt'])
+                    # transaction_success = bool(transaction['success'])
 
-            await asyncio.sleep(5)
+                    async def process_incoming_message(blockchain_message: dict):
+                        in_msg_cell = Cell.one_from_boc(b64decode(blockchain_message['msg_data']['body']))
+                        in_msg_slice = in_msg_cell.refs[0].begin_parse()
+                        in_msg_slice.read_uint(32)
+                        in_msg_slice.read_uint(8)
+                        in_msg_query_id = in_msg_slice.read_uint(23)
+                        in_msg_created_at = in_msg_slice.read_uint(64)
+                        in_msg_epoch = int(in_msg_created_at // (60 * 60))
+                        in_msg_seqno = HighloadQueryId.from_query_id(in_msg_query_id).to_seqno()
+                        in_msg_blockchain_task = (
+                            session.query(BlockchainTask).filter(
+                                and_(
+                                    BlockchainTask.seqno == in_msg_seqno,
+                                    BlockchainTask.epoch == in_msg_epoch,
+                                )
+                            )
+                        ).first()
+                        if not in_msg_blockchain_task:
+                            return
+
+                        if not (in_msg_blockchain_task.status in ['done']) or in_msg_blockchain_task.transaction_hash != transaction_hash:
+                            in_msg_blockchain_task.status = 'done'
+                            in_msg_blockchain_task.transaction_hash = transaction_hash
+                            in_msg_blockchain_task.transaction_lt = transaction_lt
+                            await session.commit()
+
+                    for blockchain_message in [transaction['in_msg']]:
+                        try:
+                            await process_incoming_message(blockchain_message)
+                        except BaseException as e:
+                            pass # make_log("TON_Daemon", f"Error while processing incoming message: {e}" + '\n' + traceback.format_exc(), level='debug')
+
+                try:
+                    sw_transactions = await toncenter.get_transactions(highload_wallet.address.to_string(1, 1, 1), limit=100)
+                    for sw_transaction in sw_transactions:
+                        try:
+                            await process_incoming_transaction(sw_transaction)
+                        except BaseException as e:
+                            make_log("TON_Daemon", f"Error while processing incoming transaction: {e}", level="debug")
+                except BaseException as e:
+                    make_log("TON_Daemon", f"Error while getting service wallet transactions: {e}", level="ERROR")
+
+                # Отправка подписанных сообщений
+                for blockchain_task in (
+                    session.query(BlockchainTask).filter(
+                        BlockchainTask.status == 'processing',
+                    ).order_by(BlockchainTask.updated.asc()).all()
+                ):
+                    make_log("TON_Daemon", f"Processing task (processing) {blockchain_task.id}")
+                    query_boc = bytes.fromhex(blockchain_task.meta['signed_message'])
+                    errors_list = []
+
+                    try:
+                        await toncenter.send_boc(query_boc)
+                    except BaseException as e:
+                        errors_list.append(f"{e}")
+
+                    try:
+                        make_log("TON_Daemon", str(
+                            httpx.post(
+                                'https://tonapi.io/v2/blockchain/message',
+                                json={
+                                    'boc': query_boc.hex()
+                                }
+                            ).text
+                        ))
+                    except BaseException as e:
+                        make_log("TON_Daemon", f"Error while pushing task to tonkeeper ({blockchain_task.id}): {e}", level="ERROR")
+                        errors_list.append(f"{e}")
+
+                    blockchain_task.updated = datetime.utcnow()
+
+                    if blockchain_task.meta['sign_created'] + 10 * 60 < datetime.utcnow().timestamp():
+                            # or sum([int("terminating vm with exit code 36" in e) for e in errors_list]) > 0:
+                        make_log("TON_Daemon", f"Task {blockchain_task.id} done", level="DEBUG")
+                        blockchain_task.status = 'done'
+                        await session.commit()
+                        continue
+
+                    await asyncio.sleep(0.5)
+
+                # Создание новых подписей
+                for blockchain_task in (
+                    session.query(BlockchainTask).filter(BlockchainTask.status == 'wait').all()
+                ):
+                    try:
+                        # Check processing tasks in current epoch < 3_000_000
+                        if (
+                            session.query(BlockchainTask).filter(
+                                BlockchainTask.epoch == blockchain_task.epoch,
+                            ).count() > 3_000_000
+                        ):
+                            make_log("TON", f"Too many processing tasks in epoch {blockchain_task.epoch}", level="error")
+                            await send_status("ton_daemon", f"working: too many tasks in epoch {blockchain_task.epoch}")
+                            await asyncio.sleep(5)
+                            continue
+
+                        sign_created = int(datetime.utcnow().timestamp()) - 60
+                        try:
+                            current_epoch = int(datetime.utcnow().timestamp() // (60 * 60))
+                            max_epoch_seqno = (
+                                session.query(func.max(BlockchainTask.seqno)).filter(
+                                    BlockchainTask.epoch == current_epoch
+                                ).scalar() or 0
+                            )
+                            current_epoch_shift = 3_000_000 if current_epoch % 2 == 0 else 0
+                            current_seqno = max_epoch_seqno + 1 + (current_epoch_shift if max_epoch_seqno == 0 else 0)
+                        except BaseException as e:
+                            make_log("CRITICAL", f"Error calculating epoch,seqno: {e}", level="error")
+                            current_epoch = 0
+                            current_seqno = 0
+
+                        blockchain_task.seqno = current_seqno
+                        blockchain_task.epoch = current_epoch
+                        blockchain_task.status = 'processing'
+                        try:
+                            query = highload_wallet.create_transfer_message(
+                                blockchain_task.destination, int(blockchain_task.amount), HighloadQueryId.from_seqno(current_seqno),
+                                sign_created, send_mode=1,
+                                payload=Cell.one_from_boc(b64decode(blockchain_task.payload))
+                            )
+                            query_boc = query['message'].to_boc(False)
+                        except BaseException as e:
+                            make_log("TON", f"Error creating transfer message: {e}", level="error")
+                            query_boc = begin_cell().end_cell().to_boc(False)
+                        
+                        blockchain_task.meta = {
+                            **blockchain_task.meta,
+                            'sign_created': sign_created,
+                            'signed_message': query_boc,
+                        }
+                        await session.commit()
+                        make_log("TON", f"Created signed message for task {blockchain_task.id}" + '\n' + traceback.format_exc(), level="info")
+                    except BaseException as e:
+                        make_log("TON", f"Error processing task {blockchain_task.id}: {e}" + '\n' + traceback.format_exc(), level="error")
+                        continue
+
+                    await asyncio.sleep(1)
+
+            await asyncio.sleep(1)
             await send_status("ton_daemon", f"working (seqno={sw_seqno_value})")
         except BaseException as e:
             make_log("TON", f"Error: {e}", level="error")

app/core/models/__init__.py

@@ -11,3 +11,5 @@ from app.core.models.content.user_content import UserContent, UserAction
 from app.core.models._config import ServiceConfigValue, ServiceConfig
 from app.core.models.asset import Asset
 from app.core.models.my_network import KnownNode, KnownNodeIncident, RemoteContentIndex
+from app.core.models.promo import PromoAction
+from app.core.models.tasks import BlockchainTask

app/core/models/_telegram/templates/player.py

@@ -182,7 +182,7 @@ class PlayerTemplates:
                         )
                 ).first())
             )
-            if have_access:
+            if False and have_access:
                 full_content = self.db_session.query(StoredContent).filter_by(
                     hash=content.meta.get('converted_content', {}).get('low')  # TODO: support high quality
                 ).first()

app/core/models/promo.py

@@ -0,0 +1,16 @@
+from .base import AlchemyBase
+from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
+from datetime import datetime
+
+
+class PromoAction:
+    __tablename__ = 'promo_actions'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    user_id = Column(String(512), nullable=False)
+    user_internal_id = Column(Integer, ForeignKey('users.id'), nullable=True)
+
+    action_type = Column(String(64), nullable=False)  # Type of action, e.g., 'referral', 'discount'
+    action_ref = Column(String(512), nullable=False)  # Reference to the action, e.g., promo code
+
+    created = Column(DateTime, nullable=False, default=datetime.now)

app/core/models/tasks.py

@@ -0,0 +1,26 @@
+from .base import AlchemyBase
+from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
+from datetime import datetime
+
+
+class BlockchainTask:
+    __tablename__ = 'blockchain_tasks'
+
+    id = Column(Integer, autoincrement=True, primary_key=True)
+    
+    destination = Column(String(1024), nullable=False)
+    amount = Column(String(256), nullable=False)
+    payload = Column(String(4096), nullable=False)
+
+    epoch = Column(Integer, nullable=True)
+    seqno = Column(Integer, nullable=True)
+
+    created = Column(DateTime, nullable=False, default=datetime.now)
+    updated = Column(DateTime, nullable=False, default=datetime.now)
+
+    user_id = Column(Integer, ForeignKey('users.id'), nullable=True)
+    meta = Column(JSON, nullable=False, default={})
+    status = Column(String(256), nullable=False)
+
+    transaction_hash = Column(String(1024), nullable=True)
+    transaction_lt = Column(String(1024), nullable=True)

app/core/projscale_logger.py

@@ -68,7 +68,7 @@ file_handler.setLevel(logging.DEBUG)
 file_handler.setFormatter(logging.Formatter(FORMAT_STRING))
 logger.addHandler(file_handler)
 
-if os.getenv('APP_ENABLE_STDOUT_LOGS', '0') == '1':
+if int(os.getenv('APP_ENABLE_STDOUT_LOGS', '0')) == 1:
     stdout_handler = logging.StreamHandler()
     stdout_handler.setLevel(logging.DEBUG)
     stdout_handler.setFormatter(logging.Formatter(FORMAT_STRING))

locale/en/LC_MESSAGES/sanic_telegram_bot.po

@@ -194,3 +194,12 @@ msgstr "🔗 Поделиться"
 
 msgid "p_shareLinkContext"
 msgstr "🎉 Наслаждайтесь {title} на MY!"
+
+msgid "p_uploadContentTxPromo"
+msgstr ""
+"🎉 Вам доступно ещё <b>{free_count}</b> бесплатных приветственных загрузок контента! "
+"Контент <b>{title}</b> уже находится в процессе загрузки. Как только блокчейн обработает транзакцию, "
+"вы получите NFT-лицензию."
+
+msgid "p_playerContext_contentNotReady"
+msgstr "⚠️ Контент, который вы хотите просмотреть, ещё не готов. Пожалуйста, попробуйте позже."

requirements.txt

@@ -6,7 +6,7 @@ pymysql==1.1.0
 aiogram==3.13.0
 pytonconnect==0.3.0
 base58==2.1.1
-tonsdk==1.0.13
+git+https://github.com/tonfactory/tonsdk.git@3ebbf0b702f48c2519e4c6c425f9514f673b9d48#egg=tonsdk
 httpx==0.25.0
 docker==7.0.0
 pycryptodome==3.20.0