27 Commits
Author SHA1 Message Date
user 14fa6fe096 q 2025-05-30 18:56:53 +03:00
user 4605a46765 nice text 2025-05-29 21:01:24 +03:00
user 6de350c2a3 add qq command 2025-05-04 19:14:13 +03:00
user 11e2645aa9 edit player ui 2025-05-03 16:00:30 +03:00
user 3d9e7c7966 fix 2025-04-26 13:29:31 +03:00
user 4483c194da fix magic 2025-04-26 13:24:35 +03:00
user d2ff332490 fix convert service 2025-04-26 13:13:47 +03:00
user 2476d3b3b6 fix someth 2025-04-26 13:01:10 +03:00
user 0e4268fb4d python magic for images 2025-04-26 12:52:32 +03:00
user 0586ed9d94 disable response header credentials 2025-03-19 14:29:32 +03:00
user a6750fe35c fix misprint 2025-03-18 14:40:54 +03:00
user 3ab014d358 fix selectWallet 2025-03-17 13:54:36 +03:00
user 4a739e4b1b fix misprint 2025-03-17 13:47:22 +03:00
user 1a1f4301cf downloadble fix 2025-03-17 13:43:31 +03:00
user b02ded982c dowload & selectWallet & fixes & auth me 2025-03-17 13:40:37 +03:00
user ac6d102a3f downloadable, selectWallet 2025-03-14 13:57:30 +03:00
user 66da3541e7 try fix licenses 2025-03-14 10:54:57 +03:00
user 7d962d463b add free upload hint 2025-03-14 00:41:31 +03:00
user 4643d7f202 fix misprint 2025-03-13 21:00:04 +03:00
user 54bc545090 add status for ton_daemon 2025-03-13 20:14:40 +03:00
user b63f663bd2 set newContent amount task 2025-03-13 20:01:48 +03:00
user a35481fa71 fix misprint 2025-03-13 19:47:04 +03:00
user 276a09fbf6 restart policy 2025-03-13 18:41:55 +03:00
user d852913692 free upload & player 2025-03-13 18:34:08 +03:00
user 862683b36e add topup highload before deploy 2025-03-13 16:33:37 +03:00
user 446bd74464 fix ton daemon 2025-03-13 16:23:41 +03:00
user b8a5cf4965 add highload wallet 2025-03-13 15:46:48 +03:00
24 changed files with 578 additions and 112 deletions

No files matched your search

+1 -1
View File
@@ -21,7 +21,7 @@ RUN apt-get update && apt-get install -y \
apt-get update && \
apt-get install -y docker-ce-cli
RUN apt-get install -y ffmpeg
RUN apt-get install libmagic1 -y
CMD ["python", "app"]
+5 -2
View File
@@ -12,8 +12,8 @@ app.register_middleware(attach_user_to_request, "request")
app.register_middleware(close_db_session, "response")
from app.api.routes._index import s_index, s_favicon
from app.api.routes._system import s_api_v1_node, s_api_system_version, s_api_system_send_status
from app.api.routes.auth import s_api_v1_auth_twa
from app.api.routes._system import s_api_v1_node, s_api_system_version, s_api_system_send_status, s_api_v1_node_friendly
from app.api.routes.auth import s_api_v1_auth_twa, s_api_v1_auth_select_wallet, s_api_v1_auth_me
from app.api.routes.statics import s_api_tonconnect_manifest, s_api_platform_metadata
from app.api.routes.node_storage import s_api_v1_storage_post, s_api_v1_storage_get, \
s_api_v1_storage_decode_cid
@@ -29,6 +29,7 @@ app.add_route(s_index, "/", methods=["GET", "OPTIONS"])
app.add_route(s_favicon, "/favicon.ico", methods=["GET", "OPTIONS"])
app.add_route(s_api_v1_node, "/api/v1/node", methods=["GET", "OPTIONS"])
app.add_route(s_api_v1_node_friendly, "/api/v1/nodeFriendly", methods=["GET", "OPTIONS"])
app.add_route(s_api_system_version, "/api/system.version", methods=["GET", "OPTIONS"])
app.add_route(s_api_system_send_status, "/api/system.sendStatus", methods=["POST", "OPTIONS"])
@@ -36,6 +37,8 @@ app.add_route(s_api_tonconnect_manifest, "/api/tonconnect-manifest.json", method
app.add_route(s_api_platform_metadata, "/api/platform-metadata.json", methods=["GET", "OPTIONS"])
app.add_route(s_api_v1_auth_twa, "/api/v1/auth.twa", methods=["POST", "OPTIONS"])
app.add_route(s_api_v1_auth_me, "/api/v1/auth.me", methods=["GET", "OPTIONS"])
app.add_route(s_api_v1_auth_select_wallet, "/api/v1/auth.selectWallet", methods=["POST", "OPTIONS"])
app.add_route(s_api_v1_tonconnect_new, "/api/v1/tonconnect.new", methods=["GET", "OPTIONS"])
app.add_route(s_api_v1_tonconnect_logout, "/api/v1/tonconnect.logout", methods=["POST", "OPTIONS"])
+1 -1
View File
@@ -16,7 +16,7 @@ def attach_headers(response):
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id"
response.headers["Access-Control-Allow-Credentials"] = "true"
# response.headers["Access-Control-Allow-Credentials"] = "true"
return response
+98 -1
View File
@@ -3,6 +3,7 @@ from datetime import datetime
import traceback
from sanic import response
from sqlalchemy import and_
from tonsdk.boc import begin_cell, begin_dict
from tonsdk.utils import Address
@@ -18,6 +19,8 @@ from app.core.models.content.user_content import UserContent
from app.core.models.node_storage import StoredContent
from app.core.models._telegram import Wrapped_CBotChat
from app.core._keyboards import get_inline_keyboard
from app.core.models.promo import PromoAction
from app.core.models.tasks import BlockchainTask
def valid_royalty_params(royalty_params):
@@ -87,7 +90,8 @@ async def s_api_v1_blockchain_send_new_content_message(request):
title=content_title,
cover_url=f"{PROJECT_HOST}/api/v1.5/storage/{image_content_cid.serialize_v2()}" if image_content_cid else None,
authors=request.json['authors'],
hashtags=request.json['hashtags']
hashtags=request.json['hashtags'],
downloadable=request.json['downloadable'] if 'downloadable' in request.json else False,
)
royalties_dict = begin_dict(8)
@@ -101,6 +105,95 @@ async def s_api_v1_blockchain_send_new_content_message(request):
)
i += 1
promo_free_upload_available = (
3 - (request.ctx.db_session.query(PromoAction).filter(
PromoAction.user_internal_id == request.ctx.user.id,
PromoAction.action_type == 'freeUpload',
).count())
)
if request.ctx.db_session.query(BlockchainTask).filter(
and_(
BlockchainTask.user_id == request.ctx.user.id,
BlockchainTask.status != 'done',
)
).first():
make_log("Blockchain", f"User {request.ctx.user.id} already has a pending task", level='warning')
promo_free_upload_available = 0
make_log("Blockchain", f"User {request.ctx.user.id} has {promo_free_upload_available} free uploads available", level='info')
if promo_free_upload_available > 0:
promo_action = PromoAction(
user_id = str(request.ctx.user.id),
user_internal_id=request.ctx.user.id,
action_type='freeUpload',
action_ref=str(encrypted_content_cid.content_hash),
created=datetime.now()
)
request.ctx.db_session.add(promo_action)
blockchain_task = BlockchainTask(
destination=platform.address.to_string(1, 1, 1),
amount=str(int(0.03 * 10 ** 9)),
payload=b64encode(
begin_cell()
.store_uint(0x5491d08c, 32)
.store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
.store_address(Address(request.ctx.user.wallet_address(request.ctx.db_session)))
.store_ref(
begin_cell()
.store_ref(
begin_cell()
.store_coins(int(0))
.store_coins(int(0))
.store_coins(int(request.json['price']))
.end_cell()
)
.store_maybe_ref(royalties_dict.end_dict())
.store_uint(0, 1)
.end_cell()
)
.store_ref(
begin_cell()
.store_ref(
begin_cell()
.store_bytes(f"{PROJECT_HOST}/api/v1.5/storage/{metadata_content.cid.serialize_v2(include_accept_type=True)}".encode())
.end_cell()
)
.store_ref(
begin_cell()
.store_ref(begin_cell().store_bytes(f"{encrypted_content_cid.serialize_v2()}".encode()).end_cell())
.store_ref(begin_cell().store_bytes(f"{image_content_cid.serialize_v2() if image_content_cid else ''}".encode()).end_cell())
.store_ref(begin_cell().store_bytes(f"{metadata_content.cid.serialize_v2()}".encode()).end_cell())
.end_cell()
)
.end_cell()
)
.end_cell().to_boc(False)
).decode(),
epoch=None, seqno=None,
created = datetime.now(),
status='wait',
user_id = request.ctx.user.id
)
request.ctx.db_session.add(blockchain_task)
request.ctx.db_session.commit()
await request.ctx.user_uploader_wrapper.send_message(
request.ctx.user.translated('p_uploadContentTxPromo').format(
title=content_title,
free_count=(promo_free_upload_available - 1)
), message_type='hint', message_meta={
'encrypted_content_hash': b58encode(encrypted_content_cid.content_hash).decode(),
'hint_type': 'uploadContentTxRequested'
}
)
return response.json({
'address': "free",
'amount': str(int(0.03 * 10 ** 9)),
'payload': ""
})
await request.ctx.user_uploader_wrapper.send_message(
request.ctx.user.translated('p_uploadContentTxRequested').format(
title=content_title,
@@ -117,6 +210,7 @@ async def s_api_v1_blockchain_send_new_content_message(request):
begin_cell()
.store_uint(0x5491d08c, 32)
.store_uint(int.from_bytes(encrypted_content_cid.content_hash, "big", signed=False), 256)
.store_uint(0, 2)
.store_ref(
begin_cell()
.store_ref(
@@ -164,6 +258,9 @@ async def s_api_v1_blockchain_send_purchase_content_message(request):
assert field_key in request.json, f"No {field_key} provided"
assert field_value(request.json[field_key]), f"Invalid {field_key} provided"
if not request.ctx.user.wallet_address(request.ctx.db_session):
return response.json({"error": "No wallet address provided"}, status=400)
license_exist = request.ctx.db_session.query(UserContent).filter_by(
onchain_address=request.json['content_address'],
).first()
+21
View File
@@ -38,6 +38,27 @@ async def s_api_v1_node(request): # /api/v1/node
}
})
async def s_api_v1_node_friendly(request):
last_known_index = request.ctx.db_session.query(StoredContent).filter(
StoredContent.onchain_index != None
).order_by(StoredContent.onchain_index.desc()).first()
last_known_index = last_known_index.onchain_index if last_known_index else 0
last_known_index = max(last_known_index, 0)
response_plain_text = f"""
Node address: {service_wallet.address.to_string(1, 1, 1)}
Node ID: {b58encode(hot_pubkey).decode()}
Master address: {platform.address.to_string(1, 1, 1)}
Indexer height: {last_known_index}
Services:
"""
for service_key, service in request.app.ctx.memory.known_states.items():
response_plain_text += f"""
{service_key}:
status: {service['status'] if (service['timestamp'] and (datetime.now() - service['timestamp']).total_seconds() < 120) else 'not working: timeout'}
delay: {round((datetime.now() - service['timestamp']).total_seconds(), 3) if service['timestamp'] else -1}
"""
return response.text(response_plain_text, content_type='text/plain')
async def s_api_system_send_status(request):
if not request.json:
+71 -2
View File
@@ -1,4 +1,5 @@
from datetime import datetime
from uuid import uuid4
from aiogram.utils.web_app import safe_parse_webapp_init_data
from sanic import response
@@ -109,8 +110,7 @@ async def s_api_v1_auth_twa(request):
WalletConnection.network == 'ton',
WalletConnection.invalidated == False
)
))).scalars().first()
).order_by(WalletConnection.created.desc()))).scalars().first()
known_user.last_use = datetime.now()
request.ctx.db_session.commit()
@@ -119,3 +119,72 @@ async def s_api_v1_auth_twa(request):
'connected_wallet': ton_connection.json_format() if ton_connection else None,
'auth_v1_token': new_user_key['auth_v1_token']
})
async def s_api_v1_auth_me(request):
if not request.ctx.user:
return response.json({"error": "Unauthorized"}, status=401)
ton_connection = (request.ctx.db_session.execute(
select(WalletConnection).where(
and_(
WalletConnection.user_id == request.ctx.user.id,
WalletConnection.network == 'ton',
WalletConnection.invalidated == False
)
).order_by(WalletConnection.created.desc())
)).scalars().first()
return response.json({
'user': request.ctx.user.json_format(),
'connected_wallet': ton_connection.json_format() if ton_connection else None
})
async def s_api_v1_auth_select_wallet(request):
if not request.ctx.user:
return response.json({"error": "Unauthorized"}, status=401)
try:
data = request.json
except Exception as e:
return response.json({"error": "Invalid JSON"}, status=400)
if "wallet_address" not in data:
return response.json({"error": "wallet_address is required"}, status=400)
# Convert raw wallet address to canonical format using Address from tonsdk.utils
raw_addr = data["wallet_address"]
canonical_address = Address(raw_addr).to_string(1, 1, 1)
db_session = request.ctx.db_session
user = request.ctx.user
# Check if a WalletConnection already exists for this user with the given canonical wallet address
existing_connection = db_session.query(WalletConnection).filter(
WalletConnection.user_id == user.id,
WalletConnection.wallet_address == canonical_address
).first()
if not existing_connection:
return response.json({"error": "Wallet connection not found"}, status=404)
saved_values = {
'keys': existing_connection.keys,
'meta': existing_connection.meta,
'wallet_key': existing_connection.wallet_key,
'connection_id': existing_connection.connection_id + uuid4().hex,
'network': existing_connection.network,
}
new_connection = WalletConnection(
**saved_values,
user_id=user.id,
wallet_address=canonical_address,
created=datetime.now(),
updated=datetime.now(),
invalidated=False,
without_pk=False
)
db_session.add(new_connection)
db_session.commit()
return response.empty(status=200)
+6 -1
View File
@@ -147,6 +147,7 @@ async def s_api_v1_content_view(request, content_address: str):
).first()
if converted_content:
display_options['content_url'] = converted_content.web_url
opts['content_ext'] = converted_content.filename.split('.')[-1]
content_meta = content['encrypted_content'].json_format()
content_metadata = StoredContent.from_cid(request.ctx.db_session, content_meta.get('metadata_cid') or None)
@@ -154,11 +155,15 @@ async def s_api_v1_content_view(request, content_address: str):
content_metadata_json = json.loads(f.read())
display_options['metadata'] = content_metadata_json
opts['downloadable'] = content_metadata_json.get('downloadable', False)
if opts['downloadable']:
if not ('listen' in opts['have_licenses']):
opts['downloadable'] = False
return response.json({
**opts,
'encrypted': content['encrypted_content'].json_format(),
'display_options': display_options
'display_options': display_options,
})
+3 -3
View File
@@ -162,7 +162,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
'mov': 'video'
}.get(preview_content.filename.split('.')[-1], content_type_declared)
hashtags_str = (' '.join(f"#{_h}" for _h in metadata_content_json.get('hashtags', []))).strip()
hashtags_str = metadata_content_json.get('description', '').strip()
if hashtags_str:
hashtags_str = hashtags_str + '\n'
@@ -212,7 +212,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
types.InlineQueryResultCachedAudio(
id=f"NC_{content.id}_{int(datetime.now().timestamp() // 60)}",
audio_file_id=decrypted_content.meta['telegram_file_cache_preview'],
caption=hashtags_str + user.translated('p_playerContext_preview'),
caption=hashtags_str + user.translated('p_playerAudioContext_preview'),
parse_mode='html',
reply_markup=get_inline_keyboard([
[
@@ -242,7 +242,7 @@ async def t_inline_query_node_content(query: types.InlineQuery, memory=None, use
id=f"NC_{content.id}_{int(datetime.now().timestamp() // 60)}",
video_file_id=decrypted_content.meta['telegram_file_cache_preview'],
title=title,
caption=hashtags_str + user.translated('p_playerContext_preview'),
caption=hashtags_str + user.translated('p_playerVideoContext_preview'),
parse_mode='html',
reply_markup=get_inline_keyboard([
[
+2 -2
View File
@@ -1,10 +1,10 @@
from app.core.models import Asset
from app.core.models import BlockchainTask
from app.core.models.base import AlchemyBase
def create_maria_tables(engine):
"""Create all tables in the database."""
Asset()
BlockchainTask()
AlchemyBase.metadata.create_all(engine)
+91 -54
View File
@@ -4,6 +4,7 @@ import os
import uuid
import json
import shutil
import magic # python-magic for MIME detection
from base58 import b58decode, b58encode
from sqlalchemy import and_, or_
from app.core.models.node_storage import StoredContent
@@ -33,12 +34,7 @@ async def convert_loop(memory):
make_log("ConvertProcess", "No content to convert", level="debug")
return
# Static preview interval in seconds
preview_interval = [0, 30]
if unprocessed_encrypted_content.onchain_index in [2]:
preview_interval = [0, 60]
make_log("ConvertProcess", f"Processing content {unprocessed_encrypted_content.id} with preview interval {preview_interval}", level="info")
# Достаем расшифрованный файл
decrypted_content = session.query(StoredContent).filter(
StoredContent.id == unprocessed_encrypted_content.decrypted_content_id
).first()
@@ -46,35 +42,80 @@ async def convert_loop(memory):
make_log("ConvertProcess", "Decrypted content not found", level="error")
return
# List of conversion options to process
REQUIRED_CONVERT_OPTIONS = ['high', 'low', 'low_preview']
converted_content = {} # Mapping: option -> sha256 hash of output file
# Define input file path and extract its extension from filename
# Определяем путь и расширение входного файла
input_file_path = f"/Storage/storedContent/{decrypted_content.hash}"
input_ext = (unprocessed_encrypted_content.filename.split('.')[-1]
if '.' in unprocessed_encrypted_content.filename else "mp4")
# ==== Новая логика: определение MIME-тип через python-magic ====
try:
mime_type = magic.from_file(input_file_path.replace("/Storage/storedContent", "/app/data"), mime=True)
except Exception as e:
make_log("ConvertProcess", f"magic probe failed: {e}", level="warning")
mime_type = ""
input_ext = unprocessed_encrypted_content.filename.split('.')[-1] if '.' in unprocessed_encrypted_content.filename else "mp4"
if mime_type.startswith("video/"):
content_kind = "video"
elif mime_type.startswith("audio/"):
content_kind = "audio"
else:
content_kind = "other"
# Logs directory mapping
make_log("ConvertProcess", f"Detected content_kind={content_kind}, mime={mime_type}", level="info")
# Для прочих типов сохраняем raw копию и выходим
if content_kind == "other":
make_log("ConvertProcess", f"Content {unprocessed_encrypted_content.id} processed. Not audio/video, copy just", level="info")
unprocessed_encrypted_content.btfs_cid = ContentId(
version=2, content_hash=b58decode(decrypted_content.hash)
).serialize_v2()
unprocessed_encrypted_content.ipfs_cid = ContentId(
version=2, content_hash=b58decode(decrypted_content.hash)
).serialize_v2()
unprocessed_encrypted_content.meta = {
**unprocessed_encrypted_content.meta,
'converted_content': {
option_name: decrypted_content.hash for option_name in ['high', 'low', 'low_preview']
}
}
session.commit()
return
# ==== Конвертация для видео или аудио: оригинальная логика ====
# Static preview interval in seconds
preview_interval = [0, 30]
if unprocessed_encrypted_content.onchain_index in [2]:
preview_interval = [0, 60]
make_log(
"ConvertProcess",
f"Processing content {unprocessed_encrypted_content.id} as {content_kind} with preview interval {preview_interval}",
level="info"
)
# Выбираем опции конвертации для видео и аудио
if content_kind == "video":
REQUIRED_CONVERT_OPTIONS = ['high', 'low', 'low_preview']
else:
REQUIRED_CONVERT_OPTIONS = ['high', 'low'] # no preview for audio
converted_content = {}
logs_dir = "/Storage/logs/converter"
# Process each conversion option in sequence
for option in REQUIRED_CONVERT_OPTIONS:
# Set quality parameter and trim option (only for preview)
if option == "low_preview":
quality = "low"
trim_value = f"{preview_interval[0]}-{preview_interval[1]}"
else:
quality = option # 'high' or 'low'
quality = option
trim_value = None
# Generate a unique output directory for docker container
output_uuid = str(uuid.uuid4())
output_dir = f"/Storage/storedContent/converter-output/{output_uuid}"
# Build the docker command with appropriate volume mounts and parameters
# Build the docker command
cmd = [
"docker", "run", "--rm",
"-v", f"{input_file_path}:/app/input",
@@ -86,8 +127,9 @@ async def convert_loop(memory):
]
if trim_value:
cmd.extend(["--trim", trim_value])
if content_kind == "audio":
cmd.append("--audio-only") # audio-only flag
# Run the docker container asynchronously
process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
@@ -98,22 +140,24 @@ async def convert_loop(memory):
make_log("ConvertProcess", f"Docker conversion failed for option {option}: {stderr.decode()}", level="error")
return
# List files in the output directory
# List files in output dir
try:
files = os.listdir(output_dir.replace("/Storage/storedContent", "/app/data"))
except Exception as e:
make_log("ConvertProcess", f"Error reading output directory {output_dir}: {e}", level="error")
return
# Exclude 'output.json' and expect exactly one media output file
media_files = [f for f in files if f != "output.json"]
if len(media_files) != 1:
make_log("ConvertProcess", f"Expected one media file, found {len(media_files)} for option {option}", level="error")
return
output_file = os.path.join(output_dir.replace("/Storage/storedContent", "/app/data"), media_files[0])
output_file = os.path.join(
output_dir.replace("/Storage/storedContent", "/app/data"),
media_files[0]
)
# Compute SHA256 hash of the output file using async subprocess
# Compute SHA256 hash of the output file
hash_process = await asyncio.create_subprocess_exec(
"sha256sum", output_file,
stdout=asyncio.subprocess.PIPE,
@@ -126,6 +170,7 @@ async def convert_loop(memory):
file_hash = hash_stdout.decode().split()[0]
file_hash = b58encode(bytes.fromhex(file_hash)).decode()
# Save new StoredContent if not exists
if not session.query(StoredContent).filter(
StoredContent.hash == file_hash
).first():
@@ -134,9 +179,7 @@ async def convert_loop(memory):
hash=file_hash,
user_id=unprocessed_encrypted_content.user_id,
filename=media_files[0],
meta={
'encrypted_file_hash': unprocessed_encrypted_content.hash,
},
meta={'encrypted_file_hash': unprocessed_encrypted_content.hash},
created=datetime.now(),
)
session.add(new_content)
@@ -156,41 +199,32 @@ async def convert_loop(memory):
converted_content[option] = file_hash
# Process output.json: read its contents and update meta['ffprobe_meta']
output_json_path = os.path.join(output_dir.replace("/Storage/storedContent", "/app/data"), "output.json")
if os.path.exists(output_json_path):
if unprocessed_encrypted_content.meta.get('ffprobe_meta') is None:
# Process output.json for ffprobe_meta
output_json_path = os.path.join(
output_dir.replace("/Storage/storedContent", "/app/data"),
"output.json"
)
if os.path.exists(output_json_path) and unprocessed_encrypted_content.meta.get('ffprobe_meta') is None:
try:
with open(output_json_path, "r") as f:
output_json_content = f.read()
except Exception as e:
make_log("ConvertProcess", f"Error reading output.json for option {option}: {e}", level="error")
return
try:
ffprobe_meta = json.loads(output_json_content)
except Exception as e:
make_log("ConvertProcess", f"Error parsing output.json for option {option}: {e}", level="error")
return
ffprobe_meta = json.load(f)
unprocessed_encrypted_content.meta = {
**unprocessed_encrypted_content.meta,
'ffprobe_meta': ffprobe_meta
}
else:
make_log("ConvertProcess", f"output.json not found for option {option}", level="error")
except Exception as e:
make_log("ConvertProcess", f"Error handling output.json for option {option}: {e}", level="error")
# Remove the output directory after processing
# Cleanup output directory
try:
shutil.rmtree(output_dir.replace("/Storage/storedContent", "/app/data"))
except Exception as e:
make_log("ConvertProcess", f"Error removing output directory {output_dir}: {e}", level="error")
# Continue even if deletion fails
make_log("ConvertProcess", f"Error removing output dir {output_dir}: {e}", level="warning")
# Finalize original record
make_log("ConvertProcess", f"Content {unprocessed_encrypted_content.id} processed. Converted content: {converted_content}", level="info")
unprocessed_encrypted_content.btfs_cid = ContentId(
version=2, content_hash=b58decode(converted_content['high'])
version=2, content_hash=b58decode(converted_content['high' if content_kind=='video' else 'low'])
).serialize_v2()
unprocessed_encrypted_content.ipfs_cid = ContentId(
version=2, content_hash=b58decode(converted_content['low'])
@@ -199,22 +233,25 @@ async def convert_loop(memory):
**unprocessed_encrypted_content.meta,
'converted_content': converted_content
}
session.commit()
# Notify user if needed
if not unprocessed_encrypted_content.meta.get('upload_notify_msg_id'):
wallet_owner_connection = session.query(WalletConnection).filter(
WalletConnection.wallet_address == unprocessed_encrypted_content.owner_address
).order_by(WalletConnection.id.desc()).first()
if wallet_owner_connection:
wallet_owner_user = wallet_owner_connection.user
wallet_owner_bot = Wrapped_CBotChat(memory._client_telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
unprocessed_encrypted_content.meta = {
**unprocessed_encrypted_content.meta,
'upload_notify_msg_id': await wallet_owner_bot.send_content(session, unprocessed_encrypted_content)
}
bot = Wrapped_CBotChat(
memory._client_telegram_bot,
chat_id=wallet_owner_user.telegram_id,
user=wallet_owner_user,
db_session=session
)
unprocessed_encrypted_content.meta['upload_notify_msg_id'] = await bot.send_content(session, unprocessed_encrypted_content)
session.commit()
async def main_fn(memory):
make_log("ConvertProcess", "Service started", level="info")
seqno = 0
+1 -1
View File
@@ -76,7 +76,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
).order_by(desc(WalletConnection.id)).first()
wallet_owner_user = wallet_owner_connection.user
if wallet_owner_user.telegram_id:
wallet_owner_bot = Wrapped_CBotChat(memory._client_telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
wallet_owner_bot = Wrapped_CBotChat(memory._telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
await wallet_owner_bot.send_message(
user.translated('p_licenseWasBought').format(
username=user.front_format(),
+7 -5
View File
@@ -3,7 +3,7 @@ from base64 import b64decode
from datetime import datetime, timedelta
from base58 import b58encode
from sqlalchemy import and_
from sqlalchemy import and_, or_
from tonsdk.boc import Cell
from tonsdk.utils import Address
@@ -74,12 +74,14 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
# Проверка кошельков пользователей на появление новых NFT, добавление их в базу как неопознанные
for user in session.query(User).filter(
User.last_use > datetime.now() - timedelta(minutes=10)
).all():
if not user.wallet_address(session):
User.last_use > datetime.now() - timedelta(hours=4)
).order_by(User.updated.asc()).all():
user_wallet_address = user.wallet_address(session)
if not user_wallet_address:
make_log("LicenseIndex", f"User {user.id} has no wallet address", level="info")
continue
make_log("LicenseIndex", f"User {user.id} has wallet address {user_wallet_address}", level="info")
last_updated_licenses = user.meta.get('last_updated_licenses')
must_skip = last_updated_licenses and (datetime.now() - datetime.fromisoformat(last_updated_licenses)) < timedelta(minutes=1)
make_log("LicenseIndex", f"User: {user.id}, last_updated_licenses: {last_updated_licenses}, must_skip: {must_skip}", level="info")
@@ -99,7 +101,7 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
UserContent.type.startswith('nft/'),
UserContent.updated < (datetime.now() - timedelta(minutes=60)),
)
).first()
).order_by(UserContent.updated.asc()).first()
if process_content:
make_log("LicenseIndex", f"Syncing content with blockchain: {process_content.id}", level="info")
try:
+194 -4
View File
@@ -1,13 +1,22 @@
import asyncio
from base64 import b64decode
import os
import traceback
import httpx
from tonsdk.boc import begin_cell
from sqlalchemy import and_, func
from tonsdk.boc import begin_cell, Cell
from tonsdk.contract.wallet import Wallets
from tonsdk.utils import HighloadQueryId
from datetime import datetime, timedelta
from app.core._blockchain.ton.platform import platform
from app.core._blockchain.ton.toncenter import toncenter
from app.core.models.tasks import BlockchainTask
from app.core._config import MY_FUND_ADDRESS
from app.core._secrets import service_wallet
from app.core._utils.send_status import send_status
from app.core.storage import db_session
from app.core.logger import make_log
@@ -75,16 +84,197 @@ async def main_fn(memory):
await asyncio.sleep(15)
return await main_fn(memory)
highload_wallet = Wallets.ALL['hv3'](
private_key=service_wallet.options['private_key'],
public_key=service_wallet.options['public_key'],
wc=0
)
make_log("TON", f"Highload wallet address: {highload_wallet.address.to_string(1, 1, 1)}", level="info")
highload_state = await toncenter.get_account(highload_wallet.address.to_string(1, 1, 1))
if int(highload_state.get('balance', '0')) / 1e9 < 0.05:
make_log("TON", "Highload wallet balance is less than 0.05, send topup transaction..", level="info")
await toncenter.send_boc(
service_wallet.create_transfer_message(
[{
'address': highload_wallet.address.to_string(1, 1, 0),
'amount': int(0.08 * 10 ** 9),
'send_mode': 1,
'payload': begin_cell().store_uint(0, 32).end_cell()
}], sw_seqno_value
)['message'].to_boc(False)
)
await send_status("ton_daemon", "working: topup highload wallet")
await asyncio.sleep(15)
return await main_fn(memory)
if not highload_state.get('code'):
make_log("TON", "Highload wallet contract is not deployed, send deploy transaction..", level="info")
created_at_ts = int(datetime.utcnow().timestamp()) - 60
await toncenter.send_boc(
highload_wallet.create_transfer_message(
service_wallet.address.to_string(1, 1, 1),
1, HighloadQueryId.from_seqno(0), created_at_ts, send_mode=1, payload="hello world", need_deploy=True
)['message'].to_boc(False)
)
await send_status("ton_daemon", "working: deploying highload wallet")
await asyncio.sleep(15)
return await main_fn(memory)
while True:
try:
sw_seqno_value = await get_sw_seqno()
make_log("TON", f"Service running ({sw_seqno_value})", level="debug")
# with db_session() as session:
# for stored_content in session.query(StoredContent).filter(StoredContent.uploaded == False).all():
# pass
with db_session() as session:
# Проверка отправленных сообщений
await send_status("ton_daemon", f"working: processing in-txs (seqno={sw_seqno_value})")
async def process_incoming_transaction(transaction: dict):
transaction_hash = transaction['transaction_id']['hash']
transaction_lt = str(transaction['transaction_id']['lt'])
# transaction_success = bool(transaction['success'])
async def process_incoming_message(blockchain_message: dict):
in_msg_cell = Cell.one_from_boc(b64decode(blockchain_message['msg_data']['body']))
in_msg_slice = in_msg_cell.refs[0].begin_parse()
in_msg_slice.read_uint(32)
in_msg_slice.read_uint(8)
in_msg_query_id = in_msg_slice.read_uint(23)
in_msg_created_at = in_msg_slice.read_uint(64)
in_msg_epoch = int(in_msg_created_at // (60 * 60))
in_msg_seqno = HighloadQueryId.from_query_id(in_msg_query_id).to_seqno()
in_msg_blockchain_task = (
session.query(BlockchainTask).filter(
and_(
BlockchainTask.seqno == in_msg_seqno,
BlockchainTask.epoch == in_msg_epoch,
)
)
).first()
if not in_msg_blockchain_task:
return
if not (in_msg_blockchain_task.status in ['done']) or in_msg_blockchain_task.transaction_hash != transaction_hash:
in_msg_blockchain_task.status = 'done'
in_msg_blockchain_task.transaction_hash = transaction_hash
in_msg_blockchain_task.transaction_lt = transaction_lt
session.commit()
for blockchain_message in [transaction['in_msg']]:
try:
await process_incoming_message(blockchain_message)
except BaseException as e:
pass # make_log("TON_Daemon", f"Error while processing incoming message: {e}" + '\n' + traceback.format_exc(), level='debug')
try:
sw_transactions = await toncenter.get_transactions(highload_wallet.address.to_string(1, 1, 1), limit=100)
for sw_transaction in sw_transactions:
try:
await process_incoming_transaction(sw_transaction)
except BaseException as e:
make_log("TON_Daemon", f"Error while processing incoming transaction: {e}", level="debug")
except BaseException as e:
make_log("TON_Daemon", f"Error while getting service wallet transactions: {e}", level="ERROR")
await send_status("ton_daemon", f"working: processing out-txs (seqno={sw_seqno_value})")
# Отправка подписанных сообщений
for blockchain_task in (
session.query(BlockchainTask).filter(
BlockchainTask.status == 'processing',
).order_by(BlockchainTask.updated.asc()).all()
):
make_log("TON_Daemon", f"Processing task (processing) {blockchain_task.id}")
query_boc = bytes.fromhex(blockchain_task.meta['signed_message'])
errors_list = []
try:
await toncenter.send_boc(query_boc)
except BaseException as e:
errors_list.append(f"{e}")
try:
make_log("TON_Daemon", str(
httpx.post(
'https://tonapi.io/v2/blockchain/message',
json={
'boc': query_boc.hex()
}
).text
))
except BaseException as e:
make_log("TON_Daemon", f"Error while pushing task to tonkeeper ({blockchain_task.id}): {e}", level="ERROR")
errors_list.append(f"{e}")
blockchain_task.updated = datetime.utcnow()
if blockchain_task.meta['sign_created'] + 10 * 60 < datetime.utcnow().timestamp():
# or sum([int("terminating vm with exit code 36" in e) for e in errors_list]) > 0:
make_log("TON_Daemon", f"Task {blockchain_task.id} done", level="DEBUG")
blockchain_task.status = 'done'
session.commit()
continue
await asyncio.sleep(0.5)
await send_status("ton_daemon", f"working: creating new messages (seqno={sw_seqno_value})")
# Создание новых подписей
for blockchain_task in (
session.query(BlockchainTask).filter(BlockchainTask.status == 'wait').all()
):
try:
# Check processing tasks in current epoch < 3_000_000
if (
session.query(BlockchainTask).filter(
BlockchainTask.epoch == blockchain_task.epoch,
).count() > 3_000_000
):
make_log("TON", f"Too many processing tasks in epoch {blockchain_task.epoch}", level="error")
await send_status("ton_daemon", f"working: too many tasks in epoch {blockchain_task.epoch}")
await asyncio.sleep(5)
continue
sign_created = int(datetime.utcnow().timestamp()) - 60
try:
current_epoch = int(datetime.utcnow().timestamp() // (60 * 60))
max_epoch_seqno = (
session.query(func.max(BlockchainTask.seqno)).filter(
BlockchainTask.epoch == current_epoch
).scalar() or 0
)
current_epoch_shift = 3_000_000 if current_epoch % 2 == 0 else 0
current_seqno = max_epoch_seqno + 1 + (current_epoch_shift if max_epoch_seqno == 0 else 0)
except BaseException as e:
make_log("CRITICAL", f"Error calculating epoch,seqno: {e}", level="error")
current_epoch = 0
current_seqno = 0
blockchain_task.seqno = current_seqno
blockchain_task.epoch = current_epoch
blockchain_task.status = 'processing'
try:
query = highload_wallet.create_transfer_message(
blockchain_task.destination, int(blockchain_task.amount), HighloadQueryId.from_seqno(current_seqno),
sign_created, send_mode=1,
payload=Cell.one_from_boc(b64decode(blockchain_task.payload))
)
query_boc = query['message'].to_boc(False)
except BaseException as e:
make_log("TON", f"Error creating transfer message: {e}", level="error")
query_boc = begin_cell().end_cell().to_boc(False)
blockchain_task.meta = {
**blockchain_task.meta,
'sign_created': sign_created,
'signed_message': query_boc.hex(),
}
session.commit()
make_log("TON", f"Created signed message for task {blockchain_task.id}" + '\n' + traceback.format_exc(), level="info")
except BaseException as e:
make_log("TON", f"Error processing task {blockchain_task.id}: {e}" + '\n' + traceback.format_exc(), level="error")
continue
await asyncio.sleep(1)
await asyncio.sleep(1)
await send_status("ton_daemon", f"working (seqno={sw_seqno_value})")
except BaseException as e:
make_log("TON", f"Error: {e}", level="error")
+2
View File
@@ -51,6 +51,7 @@ async def create_metadata_for_item(
cover_url: str = None,
authors: list = None,
hashtags: list = [],
downloadable: bool = False,
) -> StoredContent:
assert title, "No title provided"
# assert cover_url, "No cover_url provided"
@@ -66,6 +67,7 @@ async def create_metadata_for_item(
# 'value': 'Unknown'
# },
],
'downloadable': downloadable,
}
if cover_url:
item_metadata['image'] = cover_url
+2
View File
@@ -11,3 +11,5 @@ from app.core.models.content.user_content import UserContent, UserAction
from app.core.models._config import ServiceConfigValue, ServiceConfig
from app.core.models.asset import Asset
from app.core.models.my_network import KnownNode, KnownNodeIncident, RemoteContentIndex
from app.core.models.promo import PromoAction
from app.core.models.tasks import BlockchainTask
+3 -29
View File
@@ -131,33 +131,6 @@ class PlayerTemplates:
if cover_content:
# Add thumbnail if cover content is available
template_kwargs['thumbnail'] = URLInputFile(cover_content.web_url)
if self.bot_id == 1:
# Buttons for sharing and opening in app
inline_keyboard_array.append([
{
'text': self.user.translated('shareVideo_button'),
'switch_inline_query': f"Q{user_existing_license.onchain_address}" if user_existing_license else f"C{content.cid.serialize_v2()}",
},
{
'text': self.user.translated('shareLink_button'),
'url': f"https://t.me/share/url?text={urllib.parse.quote(content_share_link['text'])}&url={urllib.parse.quote(content_share_link['url'])}"
}
])
inline_keyboard_array.append([{
'text': self.user.translated('openTrackInApp_button'),
'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}/content?startapp={content.cid.serialize_v2()}"
}])
else:
# Buttons for viewing as a client and opening contract
inline_keyboard_array.append([{
'text': self.user.translated('viewTrackAsClient_button'),
'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}?start=C{content.cid.serialize_v2()}"
}])
inline_keyboard_array.append([{
'text': self.user.translated('openContractPage_button'),
'url': f"https://tonviewer.com/address/{content_meta['item_address']}"
}])
else:
local_content = None
@@ -168,6 +141,7 @@ class PlayerTemplates:
)
inline_keyboard_array = []
extra_buttons = []
else:
text = content_metadata_json.get('description').strip()
@@ -182,7 +156,7 @@ class PlayerTemplates:
)
).first())
)
if have_access:
if False and have_access:
full_content = self.db_session.query(StoredContent).filter_by(
hash=content.meta.get('converted_content', {}).get('low') # TODO: support high quality
).first()
@@ -203,7 +177,7 @@ class PlayerTemplates:
await self.delete_message(kmsg.message_id)
r = await tg_process_template(
self, text, message_id=message_id, **template_kwargs,
self, text + '\n\n' + f"""<a href="https://t.me/MY_Web3Bot/content?startapp={content.cid.serialize_v2()}"><code>🌐 Открыть на MY</code></a>""", message_id=message_id, **template_kwargs,
keyboard=get_inline_keyboard([*inline_keyboard_array, *extra_buttons]) if inline_keyboard_array else None,
message_type=f'content/{content_type}',
message_meta={'content_sha256': content_meta['hash']} if local_content else {},
+16
View File
@@ -0,0 +1,16 @@
from .base import AlchemyBase
from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
from datetime import datetime
class PromoAction(AlchemyBase):
__tablename__ = 'promo_actions'
id = Column(Integer, autoincrement=True, primary_key=True)
user_id = Column(String(512), nullable=False)
user_internal_id = Column(Integer, ForeignKey('users.id'), nullable=True)
action_type = Column(String(64), nullable=False) # Type of action, e.g., 'referral', 'discount'
action_ref = Column(String(512), nullable=False) # Reference to the action, e.g., promo code
created = Column(DateTime, nullable=False, default=datetime.now)
+26
View File
@@ -0,0 +1,26 @@
from .base import AlchemyBase
from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
from datetime import datetime
class BlockchainTask(AlchemyBase):
__tablename__ = 'blockchain_tasks'
id = Column(Integer, autoincrement=True, primary_key=True)
destination = Column(String(1024), nullable=False)
amount = Column(String(256), nullable=False)
payload = Column(String(4096), nullable=False)
epoch = Column(Integer, nullable=True)
seqno = Column(Integer, nullable=True)
created = Column(DateTime, nullable=False, default=datetime.now)
updated = Column(DateTime, nullable=False, default=datetime.now)
user_id = Column(Integer, ForeignKey('users.id'), nullable=True)
meta = Column(JSON, nullable=False, default={})
status = Column(String(256), nullable=False)
transaction_hash = Column(String(1024), nullable=True)
transaction_lt = Column(String(1024), nullable=True)
+4 -2
View File
@@ -1,3 +1,4 @@
from datetime import datetime
from sqlalchemy import Column, Integer, String, BigInteger, DateTime, JSON
from sqlalchemy.orm import relationship
@@ -19,8 +20,9 @@ class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1, W
lang_code = Column(String(8), nullable=False, default="en")
meta = Column(JSON, nullable=False, default={})
last_use = Column(DateTime, nullable=False, default=0)
created = Column(DateTime, nullable=False, default=0)
last_use = Column(DateTime, nullable=False, default=datetime.utcnow)
updated = Column(DateTime, nullable=False, default=datetime.utcnow)
created = Column(DateTime, nullable=False, default=datetime.utcnow)
balances = relationship('UserBalance', back_populates='user')
internal_transactions = relationship('InternalTransaction', back_populates='user')
+1 -1
View File
@@ -68,7 +68,7 @@ file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(logging.Formatter(FORMAT_STRING))
logger.addHandler(file_handler)
if os.getenv('APP_ENABLE_STDOUT_LOGS', '0') == '1':
if int(os.getenv('APP_ENABLE_STDOUT_LOGS', '0')) == 1:
stdout_handler = logging.StreamHandler()
stdout_handler.setLevel(logging.DEBUG)
stdout_handler.setFormatter(logging.Formatter(FORMAT_STRING))
+6
View File
@@ -8,6 +8,7 @@ services:
- .env
volumes:
- /Storage/sqlStorage:/var/lib/mysql
restart: always
healthcheck:
test: [ "CMD", "healthcheck.sh", "--connect", "--innodb_initialized" ]
interval: 10s
@@ -21,6 +22,7 @@ services:
command: python -m app
env_file:
- .env
restart: always
links:
- maria_db
ports:
@@ -36,6 +38,7 @@ services:
build:
context: .
dockerfile: Dockerfile
restart: always
command: python -m app indexer
env_file:
- .env
@@ -53,6 +56,7 @@ services:
context: .
dockerfile: Dockerfile
command: python -m app ton_daemon
restart: always
env_file:
- .env
links:
@@ -69,6 +73,7 @@ services:
context: .
dockerfile: Dockerfile
command: python -m app license_index
restart: always
env_file:
- .env
links:
@@ -85,6 +90,7 @@ services:
context: .
dockerfile: Dockerfile
command: python -m app convert_process
restart: always
env_file:
- .env
links:
Binary file not shown.
+14 -2
View File
@@ -148,8 +148,11 @@ msgid "buyTrackListenLicense_button"
msgstr "💶 Купить лицензию ({price} TON)"
#: app/core/models/_telegram/templates/player.py:117
msgid "p_playerContext_preview"
msgstr "<i>Это демонстрационная версия трека. Чтобы прослушать полную версию, необходимо приобрести лицензию 🎧.</i>"
msgid "p_playerAudioContext_preview"
msgstr "<i>Превью аудиозаписи. Полная версия доступна только по лицензии 🎧</i>"
msgid "p_playerVideoContext_preview"
msgstr "<i>Превью видео. Полная версия доступна только по лицензии 🎬</i>"
#: app/client_bot/routers/content.py:32
msgid "error_contentNotFound"
@@ -194,3 +197,12 @@ msgstr "🔗 Поделиться"
msgid "p_shareLinkContext"
msgstr "🎉 Наслаждайтесь {title} на MY!"
msgid "p_uploadContentTxPromo"
msgstr ""
"🎉 Вам доступно ещё <b>{free_count}</b> бесплатных приветственных загрузок контента! "
"Контент <b>{title}</b> уже находится в процессе загрузки. Как только блокчейн обработает транзакцию, "
"вы получите NFT-лицензию."
msgid "p_playerContext_contentNotReady"
msgstr "⚠️ Контент, который вы хотите просмотреть, ещё не готов. Пожалуйста, попробуйте позже."
+3 -1
View File
@@ -6,7 +6,7 @@ pymysql==1.1.0
aiogram==3.13.0
pytonconnect==0.3.0
base58==2.1.1
tonsdk==1.0.13
git+https://github.com/tonfactory/tonsdk.git@3ebbf0b702f48c2519e4c6c425f9514f673b9d48#egg=tonsdk
httpx==0.25.0
docker==7.0.0
pycryptodome==3.20.0
@@ -15,4 +15,6 @@ aiofiles==23.2.1
pydub==0.25.1
pillow==10.2.0
ffmpeg-python==0.2.0
python-magic==0.4.27