mariadb -> postgres

This commit is contained in:
user committed 2025-08-22 14:04:21 +03:00
1 parent 21964fa986
commit e51bb86dc0
36 files changed
+650 -464

No files matched your search

+2 -3
View File
@@ -19,9 +19,8 @@ import httpx
TELEGRAM_BOT_USERNAME = httpx.get(f"https://api.telegram.org/bot{TELEGRAM_API_KEY}/getMe").json()['result']['username']
CLIENT_TELEGRAM_BOT_USERNAME = httpx.get(f"https://api.telegram.org/bot{CLIENT_TELEGRAM_API_KEY}/getMe").json()['result']['username']
MYSQL_URI = os.environ['MYSQL_URI']
MYSQL_DATABASE = os.environ['MYSQL_DATABASE']
# Unified database URL (PostgreSQL)
DATABASE_URL = os.environ['DATABASE_URL']
LOG_LEVEL = os.getenv('LOG_LEVEL', 'DEBUG')
LOG_DIR = os.getenv('LOG_DIR', 'logs')
+11 -16
View File
@@ -36,9 +36,10 @@ async def create_new_encryption_key(db_session, user_id: int = None) -> KnownKey
meta={"I_user_id": user_id} if user_id else None,
created=datetime.now()
)
from sqlalchemy import select
db_session.add(new_key)
db_session.commit()
new_key = db_session.query(KnownKey).filter(KnownKey.seed_hash == new_seed_hash).first()
await db_session.commit()
new_key = (await db_session.execute(select(KnownKey).where(KnownKey.seed_hash == new_seed_hash))).scalars().first()
assert new_key, "Key not created"
return new_key
@@ -46,9 +47,10 @@ async def create_new_encryption_key(db_session, user_id: int = None) -> KnownKey
async def create_encrypted_content(
db_session, decrypted_content: StoredContent,
) -> StoredContent:
encrypted_content = db_session.query(StoredContent).filter(
from sqlalchemy import select
encrypted_content = (await db_session.execute(select(StoredContent).where(
StoredContent.id == decrypted_content.decrypted_content_id
).first()
))).scalars().first()
if encrypted_content:
make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): already exist (found by decrypted content)", level="debug")
return encrypted_content
@@ -57,10 +59,8 @@ async def create_encrypted_content(
if decrypted_content.key is None:
key = await create_new_encryption_key(db_session, user_id=decrypted_content.user_id)
decrypted_content.key_id = key.id
db_session.commit()
decrypted_content = db_session.query(StoredContent).filter(
StoredContent.id == decrypted_content.id
).first()
await db_session.commit()
decrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == decrypted_content.id))).scalars().first()
assert decrypted_content.key_id, "Key not assigned"
decrypted_path = os.path.join(UPLOADS_DIR, decrypted_content.hash)
@@ -72,9 +72,7 @@ async def create_encrypted_content(
encrypted_bin = cipher.encrypt(decrypted_bin)
encrypted_hash_bin = sha256(encrypted_bin).digest()
encrypted_hash = b58encode(encrypted_hash_bin).decode()
encrypted_content = db_session.query(StoredContent).filter(
StoredContent.hash == encrypted_hash
).first()
encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash))).scalars().first()
if encrypted_content:
make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): already exist (found by encrypted_hash)", level="debug")
return encrypted_content
@@ -99,19 +97,16 @@ async def create_encrypted_content(
created=datetime.now(),
)
db_session.add(encrypted_content)
db_session.commit()
await db_session.commit()
encrypted_path = os.path.join(UPLOADS_DIR, encrypted_hash)
async with aiofiles.open(encrypted_path, mode='wb') as file:
await file.write(encrypted_bin)
encrypted_content = db_session.query(StoredContent).filter(
StoredContent.hash == encrypted_hash
).first()
encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash))).scalars().first()
assert encrypted_content, "Content not created"
make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): created new content/bin", level="debug")
return encrypted_content
+27 -19
View File
@@ -1,4 +1,6 @@
import asyncio
from os import getenv, urandom
import os
from nacl.bindings import crypto_sign_seed_keypair
from tonsdk.utils import Address
@@ -7,38 +9,44 @@ from app.core._blockchain.ton.wallet_v3cr3 import WalletV3CR3
from app.core.models._config import ServiceConfig
from app.core.storage import db_session
from app.core.logger import make_log
import os
def load_hot_pair():
with db_session() as session:
async def load_hot_pair_async():
async with db_session() as session:
service_config = ServiceConfig(session)
hot_seed = service_config.get('private_key')
hot_seed = await service_config.get('private_key')
if hot_seed is None:
make_log("HotWallet", "No seed found, generating new one", level='info')
hot_seed = os.getenv("TON_INIT_HOT_SEED")
if not hot_seed:
hot_seed = urandom(32)
hot_seed_env = os.getenv("TON_INIT_HOT_SEED")
if not hot_seed_env:
hot_seed_bytes = urandom(32)
make_log("HotWallet", f"Generated random seed")
else:
hot_seed = bytes.fromhex(hot_seed)
hot_seed_bytes = bytes.fromhex(hot_seed_env)
make_log("HotWallet", f"Loaded seed from env")
service_config.set('private_key', hot_seed.hex())
return load_hot_pair()
await service_config.set('private_key', hot_seed_bytes.hex())
hot_seed = hot_seed_bytes.hex()
hot_seed = bytes.fromhex(hot_seed)
public_key, private_key = crypto_sign_seed_keypair(hot_seed)
return hot_seed, public_key, private_key
hot_seed_bytes = bytes.fromhex(hot_seed)
public_key, private_key = crypto_sign_seed_keypair(hot_seed_bytes)
return hot_seed_bytes, public_key, private_key
_extra_ton_wallet_options = {}
if getenv('TON_CUSTOM_WALLET_ADDRESS'):
_extra_ton_wallet_options['address'] = Address(getenv('TON_CUSTOM_WALLET_ADDRESS'))
hot_seed, hot_pubkey, hot_privkey = load_hot_pair()
service_wallet = WalletV3CR3(
private_key=hot_privkey,
public_key=hot_pubkey,
**_extra_ton_wallet_options
)
def _init_wallet():
# Safe to call at import time; Sanic event loop not running yet
hot_seed, hot_pubkey, hot_privkey = asyncio.run(load_hot_pair_async())
wallet = WalletV3CR3(
private_key=hot_privkey,
public_key=hot_pubkey,
**_extra_ton_wallet_options
)
return hot_seed, hot_pubkey, hot_privkey, wallet
hot_seed, hot_pubkey, hot_privkey, service_wallet = _init_wallet()
+6 -4
View File
@@ -1,10 +1,12 @@
from sqlalchemy.ext.asyncio import AsyncEngine
from app.core.models import BlockchainTask
from app.core.models.base import AlchemyBase
def create_maria_tables(engine):
"""Create all tables in the database."""
async def create_db_tables(engine: AsyncEngine):
"""Create all tables in the database (PostgreSQL, async)."""
# ensure model import side-effects initialize mappers
BlockchainTask()
AlchemyBase.metadata.create_all(engine)
async with engine.begin() as conn:
await conn.run_sync(AlchemyBase.metadata.create_all)
+3 -2
View File
@@ -56,9 +56,10 @@ class AuthenticationMixin:
},
created=datetime.fromtimestamp(init_ts)
)
from sqlalchemy import select
db_session.add(new_key)
db_session.commit()
new_key = db_session.query(KnownKey).filter(KnownKey.seed_hash == new_key.seed_hash).first()
await db_session.commit()
new_key = (await db_session.execute(select(KnownKey).where(KnownKey.seed_hash == new_key.seed_hash))).scalars().first()
assert new_key, "Key not created"
make_log("auth", f"[new-K] User {user_id} created new {token_type} key {new_key.id}")
return {
+13 -15
View File
@@ -6,7 +6,7 @@ import json
import shutil
import magic # python-magic for MIME detection
from base58 import b58decode, b58encode
from sqlalchemy import and_, or_
from sqlalchemy import and_, or_, select
from app.core.models.node_storage import StoredContent
from app.core.models._telegram import Wrapped_CBotChat
from app.core._utils.send_status import send_status
@@ -19,9 +19,9 @@ from app.core.content.content_id import ContentId
async def convert_loop(memory):
with db_session() as session:
async with db_session() as session:
# Query for unprocessed encrypted content
unprocessed_encrypted_content = session.query(StoredContent).filter(
unprocessed_encrypted_content = (await session.execute(select(StoredContent).where(
and_(
StoredContent.type == "onchain/content",
or_(
@@ -29,15 +29,15 @@ async def convert_loop(memory):
StoredContent.ipfs_cid == None,
)
)
).first()
))).scalars().first()
if not unprocessed_encrypted_content:
make_log("ConvertProcess", "No content to convert", level="debug")
return
# Достаем расшифрованный файл
decrypted_content = session.query(StoredContent).filter(
decrypted_content = (await session.execute(select(StoredContent).where(
StoredContent.id == unprocessed_encrypted_content.decrypted_content_id
).first()
))).scalars().first()
if not decrypted_content:
make_log("ConvertProcess", "Decrypted content not found", level="error")
return
@@ -78,7 +78,7 @@ async def convert_loop(memory):
option_name: decrypted_content.hash for option_name in ['high', 'low', 'low_preview']
}
}
session.commit()
await session.commit()
return
# ==== Конвертация для видео или аудио: оригинальная логика ====
@@ -171,9 +171,7 @@ async def convert_loop(memory):
file_hash = b58encode(bytes.fromhex(file_hash)).decode()
# Save new StoredContent if not exists
if not session.query(StoredContent).filter(
StoredContent.hash == file_hash
).first():
if not (await session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first():
new_content = StoredContent(
type="local/content_bin",
hash=file_hash,
@@ -183,7 +181,7 @@ async def convert_loop(memory):
created=datetime.now(),
)
session.add(new_content)
session.commit()
await session.commit()
save_path = os.path.join(UPLOADS_DIR, file_hash)
try:
@@ -233,13 +231,13 @@ async def convert_loop(memory):
**unprocessed_encrypted_content.meta,
'converted_content': converted_content
}
session.commit()
await session.commit()
# Notify user if needed
if not unprocessed_encrypted_content.meta.get('upload_notify_msg_id'):
wallet_owner_connection = session.query(WalletConnection).filter(
wallet_owner_connection = (await session.execute(select(WalletConnection).where(
WalletConnection.wallet_address == unprocessed_encrypted_content.owner_address
).order_by(WalletConnection.id.desc()).first()
).order_by(WalletConnection.id.desc()))).scalars().first()
if wallet_owner_connection:
wallet_owner_user = wallet_owner_connection.user
bot = Wrapped_CBotChat(
@@ -249,7 +247,7 @@ async def convert_loop(memory):
db_session=session
)
unprocessed_encrypted_content.meta['upload_notify_msg_id'] = await bot.send_content(session, unprocessed_encrypted_content)
session.commit()
await session.commit()
async def main_fn(memory):
+29 -28
View File
@@ -17,6 +17,7 @@ from app.core._utils.resolve_content import resolve_content
from app.core.models.wallet_connection import WalletConnection
from app.core._keyboards import get_inline_keyboard
from app.core.models._telegram import Wrapped_CBotChat
from sqlalchemy import select, and_, desc
from app.core.storage import db_session
import os
import traceback
@@ -33,7 +34,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
platform_found = True
make_log("Indexer", "Service running", level="debug")
with db_session() as session:
async with db_session() as session:
try:
result = await toncenter.run_get_method('EQD8TJ8xEWB1SpnRE4d89YO3jl0W0EiBnNS4IBaHaUmdfizE', 'get_pool_data')
assert result['exit_code'] == 0, f"Error in get-method: {result}"
@@ -41,21 +42,21 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
assert result['stack'][1][0] == 'num', f"get second element is not num"
usdt_per_ton = (int(result['stack'][0][1], 16) * 1e3) / int(result['stack'][1][1], 16)
ton_per_star = 0.014 / usdt_per_ton
ServiceConfig(session).set('live_tonPerStar', [ton_per_star, datetime.utcnow().timestamp()])
await ServiceConfig(session).set('live_tonPerStar', [ton_per_star, datetime.utcnow().timestamp()])
make_log("TON_Daemon", f"TON per STAR price: {ton_per_star}", level="DEBUG")
except BaseException as e:
make_log("TON_Daemon", f"Error while saving TON per STAR price: {e}" + '\n' + traceback.format_exc(), level="ERROR")
new_licenses = session.query(UserContent).filter(
new_licenses = (await session.execute(select(UserContent).where(
and_(
~UserContent.meta.contains({'notification_sent': True}),
UserContent.type == 'nft/listen'
)
).all()
))).scalars().all()
for new_license in new_licenses:
licensed_content = session.query(StoredContent).filter(
licensed_content = (await session.execute(select(StoredContent).where(
StoredContent.id == new_license.content_id
).first()
))).scalars().first()
if not licensed_content:
make_log("Indexer", f"Licensed content not found: {new_license.content_id}", level="error")
@@ -70,10 +71,12 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
session, licensed_content
)
wallet_owner_connection = session.query(WalletConnection).filter_by(
wallet_address=licensed_content.owner_address,
invalidated=False
).order_by(desc(WalletConnection.id)).first()
wallet_owner_connection = (await session.execute(
select(WalletConnection).where(
WalletConnection.wallet_address == licensed_content.owner_address,
WalletConnection.invalidated == False
).order_by(desc(WalletConnection.id))
)).scalars().first()
wallet_owner_user = wallet_owner_connection.user
if wallet_owner_user.telegram_id:
wallet_owner_bot = Wrapped_CBotChat(memory._telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
@@ -89,21 +92,19 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
make_log("IndexerSendNewLicense", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
new_license.meta = {**new_license.meta, 'notification_sent': True}
session.commit()
await session.commit()
content_without_cid = session.query(StoredContent).filter(
StoredContent.content_id == None
)
content_without_cid = (await session.execute(select(StoredContent).where(StoredContent.content_id == None))).scalars().all()
for target_content in content_without_cid:
target_cid = target_content.cid.serialize_v2()
make_log("Indexer", f"Content without CID: {target_content.hash}, setting CID: {target_cid}", level="debug")
target_content.content_id = target_cid
session.commit()
await session.commit()
last_known_index_ = session.query(StoredContent).filter(
StoredContent.onchain_index != None
).order_by(StoredContent.onchain_index.desc()).first()
last_known_index_ = (await session.execute(
select(StoredContent).where(StoredContent.onchain_index != None).order_by(StoredContent.onchain_index.desc())
)).scalars().first()
last_known_index = last_known_index_.onchain_index if last_known_index_ else 0
last_known_index = max(last_known_index, 0)
make_log("Indexer", f"Last known index: {last_known_index}", level="debug")
@@ -196,14 +197,13 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
user_wallet_connection = None
if item_owner_address:
user_wallet_connection = session.query(WalletConnection).filter(
user_wallet_connection = (await session.execute(select(WalletConnection).where(
WalletConnection.wallet_address == item_owner_address.to_string(1, 1, 1)
).first()
))).scalars().first()
encrypted_stored_content = session.query(StoredContent).filter(
StoredContent.hash == item_content_hash_str,
# StoredContent.type.like("local%")
).first()
encrypted_stored_content = (await session.execute(select(StoredContent).where(
StoredContent.hash == item_content_hash_str
))).scalars().first()
if encrypted_stored_content:
is_duplicate = encrypted_stored_content.type.startswith("onchain") \
and encrypted_stored_content.onchain_index != item_index
@@ -234,14 +234,15 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
)
try:
for hint_message in session.query(KnownTelegramMessage).filter(
result = await session.execute(select(KnownTelegramMessage).where(
and_(
KnownTelegramMessage.chat_id == user.telegram_id,
KnownTelegramMessage.type == 'hint',
cast(KnownTelegramMessage.meta['encrypted_content_hash'], String) == encrypted_stored_content.hash,
KnownTelegramMessage.deleted == False
)
).all():
))
for hint_message in result.scalars().all():
await user_uploader_wrapper.delete_message(hint_message.message_id)
except BaseException as e:
make_log("Indexer", f"Error while deleting hint messages: {e}" + '\n' + traceback.format_exc(), level="error")
@@ -260,7 +261,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
**item_metadata_packed
}
session.commit()
await session.commit()
return platform_found, seqno
else:
item_metadata_packed['copied_from'] = encrypted_stored_content.id
@@ -282,7 +283,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
updated=datetime.now()
)
session.add(onchain_stored_content)
session.commit()
await session.commit()
make_log("Indexer", f"Item indexed: {item_content_hash_str}", level="info")
last_known_index += 1
+14 -13
View File
@@ -3,7 +3,7 @@ from base64 import b64decode
from datetime import datetime, timedelta
from base58 import b58encode
from sqlalchemy import and_, or_
from sqlalchemy import and_, or_, select, desc
from tonsdk.boc import Cell
from tonsdk.utils import Address
@@ -27,7 +27,7 @@ import traceback
async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
make_log("LicenseIndex", "Service running", level="debug")
with db_session() as session:
async with db_session() as session:
async def check_telegram_stars_transactions():
# Проверка звездных telegram транзакций, обновление paid
offset = {'desc': 'Статичное число заранее известного количества транзакций, которое даже не знает наш бот', 'value': 1}['value'] + \
@@ -45,19 +45,19 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
continue
try:
existing_invoice = session.query(StarsInvoice).filter(
existing_invoice = (await session.execute(select(StarsInvoice).where(
StarsInvoice.external_id == star_payment.source.invoice_payload
).first()
))).scalars().first()
if not existing_invoice:
continue
if star_payment.amount == existing_invoice.amount:
if not existing_invoice.paid:
existing_invoice.paid = True
session.commit()
await session.commit()
licensed_content = session.query(StoredContent).filter(StoredContent.hash == existing_invoice.content_hash).first()
user = session.query(User).filter(User.id == existing_invoice.user_id).first()
licensed_content = (await session.execute(select(StoredContent).where(StoredContent.hash == existing_invoice.content_hash))).scalars().first()
user = (await session.execute(select(User).where(User.id == existing_invoice.user_id))).scalars().first()
await (Wrapped_CBotChat(memory._client_telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)).send_content(
session, licensed_content
@@ -73,9 +73,10 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
make_log("StarsProcessing", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
# Проверка кошельков пользователей на появление новых NFT, добавление их в базу как неопознанные
for user in session.query(User).filter(
users = (await session.execute(select(User).where(
User.last_use > datetime.now() - timedelta(hours=4)
).order_by(User.updated.asc()).all():
).order_by(User.updated.asc()))).scalars().all()
for user in users:
user_wallet_address = user.wallet_address(session)
if not user_wallet_address:
make_log("LicenseIndex", f"User {user.id} has no wallet address", level="info")
@@ -91,17 +92,17 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
try:
await user.scan_owned_user_content(session)
user.meta = {**user.meta, 'last_updated_licenses': datetime.now().isoformat()}
session.commit()
await session.commit()
except BaseException as e:
make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
# Проверка NFT на актуальность данных, в том числе уже проверенные
process_content = session.query(UserContent).filter(
process_content = (await session.execute(select(UserContent).where(
and_(
UserContent.type.startswith('nft/'),
UserContent.updated < (datetime.now() - timedelta(minutes=60)),
)
).order_by(UserContent.updated.asc()).first()
).order_by(UserContent.updated.asc()))).scalars().first()
if process_content:
make_log("LicenseIndex", f"Syncing content with blockchain: {process_content.id}", level="info")
try:
@@ -110,7 +111,7 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
finally:
process_content.updated = datetime.now()
session.commit()
await session.commit()
return platform_found, seqno
+27 -25
View File
@@ -125,7 +125,7 @@ async def main_fn(memory):
sw_seqno_value = await get_sw_seqno()
make_log("TON", f"Service running ({sw_seqno_value})", level="debug")
with db_session() as session:
async with db_session() as session:
# Проверка отправленных сообщений
await send_status("ton_daemon", f"working: processing in-txs (seqno={sw_seqno_value})")
async def process_incoming_transaction(transaction: dict):
@@ -142,14 +142,17 @@ async def main_fn(memory):
in_msg_created_at = in_msg_slice.read_uint(64)
in_msg_epoch = int(in_msg_created_at // (60 * 60))
in_msg_seqno = HighloadQueryId.from_query_id(in_msg_query_id).to_seqno()
from sqlalchemy import select
in_msg_blockchain_task = (
session.query(BlockchainTask).filter(
and_(
BlockchainTask.seqno == in_msg_seqno,
BlockchainTask.epoch == in_msg_epoch,
await session.execute(
select(BlockchainTask).where(
and_(
BlockchainTask.seqno == in_msg_seqno,
BlockchainTask.epoch == in_msg_epoch,
)
)
)
).first()
).scalars().first()
if not in_msg_blockchain_task:
return
@@ -157,7 +160,7 @@ async def main_fn(memory):
in_msg_blockchain_task.status = 'done'
in_msg_blockchain_task.transaction_hash = transaction_hash
in_msg_blockchain_task.transaction_lt = transaction_lt
session.commit()
await session.commit()
for blockchain_message in [transaction['in_msg']]:
try:
@@ -177,11 +180,11 @@ async def main_fn(memory):
await send_status("ton_daemon", f"working: processing out-txs (seqno={sw_seqno_value})")
# Отправка подписанных сообщений
for blockchain_task in (
session.query(BlockchainTask).filter(
BlockchainTask.status == 'processing',
).order_by(BlockchainTask.updated.asc()).all()
):
from sqlalchemy import select
_processing = (await session.execute(select(BlockchainTask).where(
BlockchainTask.status == 'processing'
).order_by(BlockchainTask.updated.asc()))).scalars().all()
for blockchain_task in _processing:
make_log("TON_Daemon", f"Processing task (processing) {blockchain_task.id}")
query_boc = bytes.fromhex(blockchain_task.meta['signed_message'])
errors_list = []
@@ -210,23 +213,22 @@ async def main_fn(memory):
# or sum([int("terminating vm with exit code 36" in e) for e in errors_list]) > 0:
make_log("TON_Daemon", f"Task {blockchain_task.id} done", level="DEBUG")
blockchain_task.status = 'done'
session.commit()
await session.commit()
continue
await asyncio.sleep(0.5)
await send_status("ton_daemon", f"working: creating new messages (seqno={sw_seqno_value})")
# Создание новых подписей
for blockchain_task in (
session.query(BlockchainTask).filter(BlockchainTask.status == 'wait').all()
):
_waiting = (await session.execute(select(BlockchainTask).where(BlockchainTask.status == 'wait'))).scalars().all()
for blockchain_task in _waiting:
try:
# Check processing tasks in current epoch < 3_000_000
if (
session.query(BlockchainTask).filter(
BlockchainTask.epoch == blockchain_task.epoch,
).count() > 3_000_000
):
from sqlalchemy import func
_cnt = (await session.execute(select(func.count()).select_from(BlockchainTask).where(
BlockchainTask.epoch == blockchain_task.epoch
))).scalar() or 0
if _cnt > 3_000_000:
make_log("TON", f"Too many processing tasks in epoch {blockchain_task.epoch}", level="error")
await send_status("ton_daemon", f"working: too many tasks in epoch {blockchain_task.epoch}")
await asyncio.sleep(5)
@@ -235,10 +237,11 @@ async def main_fn(memory):
sign_created = int(datetime.utcnow().timestamp()) - 60
try:
current_epoch = int(datetime.utcnow().timestamp() // (60 * 60))
from sqlalchemy import func
max_epoch_seqno = (
session.query(func.max(BlockchainTask.seqno)).filter(
(await session.execute(select(func.max(BlockchainTask.seqno)).where(
BlockchainTask.epoch == current_epoch
).scalar() or 0
))).scalar() or 0
)
current_epoch_shift = 3_000_000 if current_epoch % 2 == 0 else 0
current_seqno = max_epoch_seqno + 1 + (current_epoch_shift if max_epoch_seqno == 0 else 0)
@@ -266,7 +269,7 @@ async def main_fn(memory):
'sign_created': sign_created,
'signed_message': query_boc.hex(),
}
session.commit()
await session.commit()
make_log("TON", f"Created signed message for task {blockchain_task.id}" + '\n' + traceback.format_exc(), level="info")
except BaseException as e:
make_log("TON", f"Error processing task {blockchain_task.id}: {e}" + '\n' + traceback.format_exc(), level="error")
@@ -287,4 +290,3 @@ async def main_fn(memory):
+6 -3
View File
@@ -26,7 +26,9 @@ async def create_new_content(
content_hash_bin = sha256(content_bin).digest()
content_hash_b58 = b58encode(content_hash_bin).decode()
new_content = db_session.query(StoredContent).filter(StoredContent.hash == content_hash_b58).first()
from sqlalchemy import select
result = await db_session.execute(select(StoredContent).where(StoredContent.hash == content_hash_b58))
new_content = result.scalars().first()
if new_content:
return new_content, False
@@ -38,8 +40,9 @@ async def create_new_content(
)
db_session.add(new_content)
db_session.commit()
new_content = db_session.query(StoredContent).filter(StoredContent.hash == content_hash_b58).first()
await db_session.commit()
result = await db_session.execute(select(StoredContent).where(StoredContent.hash == content_hash_b58))
new_content = result.scalars().first()
assert new_content, "Content not created (through utils)"
content_filepath = os.path.join(UPLOADS_DIR, content_hash_b58)
async with aiofiles.open(content_filepath, 'wb') as file:
+12 -14
View File
@@ -1,6 +1,6 @@
from app.core.models.base import AlchemyBase
from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
from sqlalchemy import Column, Integer, String, JSON, select
class ServiceConfigValue(AlchemyBase):
@@ -19,20 +19,18 @@ class ServiceConfig:
def __init__(self, session):
self.session = session
def get(self, key, default=None):
result = self.session.query(ServiceConfigValue).filter(ServiceConfigValue.key == key).first()
async def get(self, key, default=None):
result = (await self.session.execute(select(ServiceConfigValue).where(ServiceConfigValue.key == key))).scalars().first()
return (result.value if result else None) or default
def set(self, key, value):
config_value = self.session.query(ServiceConfigValue).filter(
ServiceConfigValue.key == key
).first()
if not config_value:
config_value = ServiceConfigValue(key=key)
self.session.add(config_value)
self.session.commit()
return self.set(key, value)
async def set(self, key, value):
result = (await self.session.execute(select(ServiceConfigValue).where(ServiceConfigValue.key == key))).scalars().first()
if not result:
result = ServiceConfigValue(key=key)
self.session.add(result)
await self.session.commit()
return await self.set(key, value)
config_value.packed_value = {'value': value}
self.session.commit()
result.packed_value = {'value': value}
await self.session.commit()
return
+17 -14
View File
@@ -1,4 +1,4 @@
from sqlalchemy import and_
from sqlalchemy import and_, select
from app.core.models.node_storage import StoredContent
from app.core.models.content.user_content import UserContent, UserAction
from app.core.logger import make_log
@@ -27,17 +27,17 @@ class PlayerTemplates:
if not content.encrypted:
local_content = content
else:
local_content = db_session.query(StoredContent).filter_by(
id=content.decrypted_content_id
).first()
local_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == content.decrypted_content_id))).scalars().first()
# TODO: add check decrypted_content by .format_json()['content_cid']
if local_content:
cd_log += f"Decrypted: {local_content.hash}. "
else:
cd_log += "Can't decrypt content. "
user_wallet_address = self.user.wallet_address(self.db_session)
user_existing_license = self.db_session.query(UserContent).filter_by(user_id=self.user.id, content_id=content.id).first()
user_wallet_address = await self.user.wallet_address_async(self.db_session)
user_existing_license = (await self.db_session.execute(select(UserContent).where(
and_(UserContent.user_id == self.user.id, UserContent.content_id == content.id)
))).scalars().first()
if local_content:
content_meta = content.json_format()
@@ -48,12 +48,12 @@ class PlayerTemplates:
except:
content_type, content_encoding = 'application', 'x-binary'
content_metadata = StoredContent.from_cid(db_session, content_meta.get('metadata_cid') or None)
content_metadata = await StoredContent.from_cid_async(db_session, content_meta.get('metadata_cid') or None)
with open(content_metadata.filepath, 'r') as f:
content_metadata_json = json.loads(f.read())
try:
cover_content = StoredContent.from_cid(self.db_session, content_meta.get('cover_cid') or None)
cover_content = await StoredContent.from_cid_async(self.db_session, content_meta.get('cover_cid') or None)
cd_log += f"Cover content: {cover_content.cid.serialize_v2()}. "
except BaseException as e:
cd_log += f"Can't get cover content: {e}. "
@@ -88,12 +88,15 @@ class PlayerTemplates:
<blockquote><a href="{content_share_link['url']}">🔴 «открыть в MY»</a></blockquote>"""
make_log("TG-Player", f"Send content {content_type} ({content_encoding}) to chat {self._chat_id}. {cd_log}")
for kmsg in self.db_session.query(KnownTelegramMessage).filter_by(
content_id=content.id,
chat_id=self._chat_id,
type=f'content/{content_type}',
deleted=False
).all():
kmsgs = (await self.db_session.execute(select(KnownTelegramMessage).where(
and_(
KnownTelegramMessage.content_id == content.id,
KnownTelegramMessage.chat_id == self._chat_id,
KnownTelegramMessage.type == f'content/{content_type}',
KnownTelegramMessage.deleted == False
)
))).scalars().all()
for kmsg in kmsgs:
await self.delete_message(kmsg.message_id)
r = await tg_process_template(
+13 -10
View File
@@ -1,7 +1,7 @@
from aiogram import Bot, types
from datetime import datetime, timedelta
from sqlalchemy import and_
from sqlalchemy import and_, select
from app.core.logger import make_log
from app.core.models.messages import KnownTelegramMessage
@@ -46,14 +46,15 @@ class Wrapped_CBotChat(T, PlayerTemplates):
if self.db_session:
if message_type == 'common':
ci = 0
for oc_msg in self.db_session.query(KnownTelegramMessage).filter(
result = await self.db_session.execute(select(KnownTelegramMessage).where(
and_(
KnownTelegramMessage.type == 'common',
KnownTelegramMessage.bot_id == self.bot_id,
KnownTelegramMessage.chat_id == self._chat_id,
KnownTelegramMessage.deleted == False
)
).all():
))
for oc_msg in result.scalars().all():
make_log(self, f"Delete old message {oc_msg.message_id} {oc_msg.type} {oc_msg.bot_id} {oc_msg.chat_id}")
await self.delete_message(oc_msg.message_id)
ci += 1
@@ -75,7 +76,7 @@ class Wrapped_CBotChat(T, PlayerTemplates):
content_id=content_id
)
)
self.db_session.commit()
await self.db_session.commit()
else:
make_log(self, f"Unknown result type: {type(result)}", level='warning')
@@ -127,14 +128,16 @@ class Wrapped_CBotChat(T, PlayerTemplates):
message_id
)):
if self.db_session:
known_message = self.db_session.query(KnownTelegramMessage).filter(
KnownTelegramMessage.bot_id == self.bot_id,
KnownTelegramMessage.chat_id == self._chat_id,
KnownTelegramMessage.message_id == message_id
).first()
known_message = (await self.db_session.execute(select(KnownTelegramMessage).where(
and_(
KnownTelegramMessage.bot_id == self.bot_id,
KnownTelegramMessage.chat_id == self._chat_id,
KnownTelegramMessage.message_id == message_id
)
))).scalars().first()
if known_message:
known_message.deleted = True
self.db_session.commit()
await self.db_session.commit()
except Exception as e:
make_log(self, f"Error deleting message {self._chat_id}/{message_id}. Error: {e}", level='warning')
return None
+14 -15
View File
@@ -29,22 +29,21 @@ class Asset(AlchemyBase):
AlchemyBase.metadata.create_all(engine)
@classmethod
def find(cls, session, **kwargs):
async def find_async(cls, session, **kwargs):
from sqlalchemy import select, func
if 'symbol' in kwargs:
kwargs['symbol'] = kwargs['symbol'].upper()
result = session.query(cls).filter_by(**kwargs)
results_count = result.count()
if results_count == 0:
any_count = session.query(cls).count()
if any_count == 0:
init_asset = cls(**DEFAULT_ASSET_INITOBJ)
session.add(init_asset)
session.commit()
return cls.find(session, **kwargs)
result = await session.execute(select(cls).filter_by(**kwargs))
row = result.scalars().first()
if row:
return row
raise Exception(f"Asset not found: {kwargs}")
elif results_count == 1:
return result.first()
else:
raise Exception(f"Multiple assets found: {results_count}")
any_count = (await session.execute(select(func.count()).select_from(cls))).scalar() or 0
if any_count == 0:
init_asset = cls(**DEFAULT_ASSET_INITOBJ)
session.add(init_asset)
await session.commit()
return await cls.find_async(session, **kwargs)
raise Exception(f"Asset not found: {kwargs}")
+6 -11
View File
@@ -1,7 +1,7 @@
import traceback
import base58
from sqlalchemy import and_
from sqlalchemy import and_, select
from app.core.logger import make_log
from app.core.models import StoredContent
@@ -57,13 +57,9 @@ class UserContentIndexationMixin:
values_slice = cc_indexator_data['values'].begin_parse()
content_hash_b58 = base58.b58encode(bytes.fromhex(hex(values_slice.read_uint(256))[2:])).decode()
make_log("UserContent", f"License ({self.onchain_address}) content hash: {content_hash_b58}", level="info")
stored_content = db_session.query(StoredContent).filter(
and_(
StoredContent.type == 'onchain/content',
StoredContent.hash == content_hash_b58,
)
).first()
stored_content = (await db_session.execute(select(StoredContent).where(
and_(StoredContent.type == 'onchain/content', StoredContent.hash == content_hash_b58)
))).scalars().first()
trusted_cop_address_result = await toncenter.run_get_method(stored_content.meta['item_address'], 'get_nft_address_by_index', [['num', cc_indexator_data['index']]])
assert trusted_cop_address_result.get('exit_code', -1) == 0, "Trusted cop address error"
trusted_cop_address = Cell.one_from_boc(b64decode(trusted_cop_address_result['stack'][0][1]['bytes'])).begin_parse().read_msg_addr().to_string(1, 1, 1)
@@ -72,7 +68,7 @@ class UserContentIndexationMixin:
self.owner_address = cc_indexator_data['owner_address']
self.type = 'nft/listen'
self.content_id = stored_content.id
db_session.commit()
await db_session.commit()
except BaseException as e:
errored = True
make_log("UserContent", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
@@ -80,7 +76,6 @@ class UserContentIndexationMixin:
if errored is True:
self.type = 'nft/unknown'
self.content_id = None
db_session.commit()
await db_session.commit()
+36
View File
@@ -96,6 +96,30 @@ class StoredContent(AlchemyBase, AudioContentMixin):
make_log("NodeStorage.open_content", f"Can't open content: {self.id} {e}", level='warning')
raise e
async def open_content_async(self, db_session, content_type=None):
from sqlalchemy import select
try:
decrypted_content = self if not self.encrypted else None
encrypted_content = self if self.encrypted else None
if not decrypted_content:
decrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == self.decrypted_content_id))).scalars().first()
else:
encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.decrypted_content_id == self.id))).scalars().first()
assert decrypted_content, "Can't get decrypted content"
assert encrypted_content, "Can't get encrypted content"
_ct = content_type or decrypted_content.json_format()['content_type']
content_type = _ct.split('/')[0] if _ct else 'application'
return {
'encrypted_content': encrypted_content,
'decrypted_content': decrypted_content,
'content_type': content_type or 'application/x-binary'
}
except BaseException as e:
make_log("NodeStorage.open_content_async", f"Can't open content: {self.id} {e}", level='warning')
raise e
def json_format(self):
extra_fields = {}
if self.type.startswith('local'):
@@ -155,3 +179,15 @@ class StoredContent(AlchemyBase, AudioContentMixin):
assert content, "Content not found"
return content
@classmethod
async def from_cid_async(cls, db_session, content_id):
from sqlalchemy import select
if isinstance(content_id, str):
cid = ContentId.deserialize(content_id)
else:
cid = content_id
result = await db_session.execute(select(StoredContent).where(StoredContent.hash == cid.content_hash_b58))
content = result.scalars().first()
assert content, "Content not found"
return content
+24 -23
View File
@@ -10,18 +10,21 @@ from httpx import AsyncClient
class WalletMixin:
def wallet_connection(self, db_session):
return db_session.query(WalletConnection).filter(
WalletConnection.user_id == self.id,
WalletConnection.invalidated == False
).order_by(WalletConnection.created.desc()).first()
async def wallet_connection_async(self, db_session):
from sqlalchemy import select, and_, desc
result = await db_session.execute(
select(WalletConnection)
.where(and_(WalletConnection.user_id == self.id, WalletConnection.invalidated == False))
.order_by(WalletConnection.created.desc())
)
return result.scalars().first()
def wallet_address(self, db_session):
wallet_connection = self.wallet_connection(db_session)
return wallet_connection.wallet_address if wallet_connection else None
async def wallet_address_async(self, db_session):
wc = await self.wallet_connection_async(db_session)
return wc.wallet_address if wc else None
async def scan_owned_user_content(self, db_session):
user_wallet_address = self.wallet_address(db_session)
user_wallet_address = await self.wallet_address_async(db_session)
async def get_nft_items_list():
try:
@@ -40,9 +43,8 @@ class WalletMixin:
item_address = Address(nft_item['address']).to_string(1, 1, 1)
owner_address = Address(nft_item['owner']['address']).to_string(1, 1, 1)
user_content = db_session.query(UserContent).filter(
UserContent.onchain_address == item_address
).first()
from sqlalchemy import select
user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
if user_content:
continue
@@ -57,18 +59,18 @@ class WalletMixin:
created=datetime.now(),
meta={},
user_id=self.id,
wallet_connection_id=self.wallet_connection(db_session).id,
wallet_connection_id=(await self.wallet_connection_async(db_session)).id,
status="active"
)
db_session.add(user_content)
db_session.commit()
await db_session.commit()
make_log(self, f"New onchain NFT found: {item_address}", level='info')
async def ____scan_owned_user_content(self, db_session):
page_id = -1
page_size = 100
have_next_page = True
user_wallet_address = self.wallet_address(db_session)
user_wallet_address = await self.wallet_address_async(db_session)
while have_next_page:
page_id += 1
nfts_list = await toncenter.get_nft_items(limit=100, offset=page_id * page_size, owner_address=user_wallet_address)
@@ -81,9 +83,8 @@ class WalletMixin:
item_address = Address(nft_item['address']).to_string(1, 1, 1)
owner_address = Address(nft_item['owner_address']).to_string(1, 1, 1)
user_content = db_session.query(UserContent).filter(
UserContent.onchain_address == item_address
).first()
from sqlalchemy import select
user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
if user_content:
continue
@@ -105,11 +106,11 @@ class WalletMixin:
'metadata_uri': nft_content,
},
user_id=self.id,
wallet_connection_id=self.wallet_connection(db_session).id,
wallet_connection_id=(await self.wallet_connection_async(db_session)).id,
status="active"
)
db_session.add(user_content)
db_session.commit()
await db_session.commit()
make_log(self, f"New onchain NFT found: {item_address}", level='info')
except BaseException as e:
@@ -122,6 +123,6 @@ class WalletMixin:
except BaseException as e:
make_log(self, f"Error while scanning user content: {e}", level='error')
return self.db_session.query(UserContent).filter(
UserContent.user_id == self.id
).offset(offset).limit(limit).all()
from sqlalchemy import select
result = await db_session.execute(select(UserContent).where(UserContent.user_id == self.id).offset(offset).limit(limit))
return result.scalars().all()
+46 -34
View File
@@ -1,45 +1,57 @@
import time
from contextlib import contextmanager
from contextlib import asynccontextmanager
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql import text
from sqlalchemy import text
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
from app.core._config import MYSQL_URI, MYSQL_DATABASE
from app.core._config import DATABASE_URL
from app.core.logger import make_log
from sqlalchemy.pool import NullPool
engine = create_engine(MYSQL_URI, poolclass=NullPool) #, echo=True)
Session = sessionmaker(bind=engine)
database_initialized = False
while not database_initialized:
def _to_async_dsn(url: str) -> str:
# Convert psycopg2 DSN to asyncpg DSN
# postgresql+psycopg2://user:pass@host:5432/db -> postgresql+asyncpg://user:pass@host:5432/db
return url.replace("+psycopg2", "+asyncpg")
# Async engine for PostgreSQL
engine = create_async_engine(
_to_async_dsn(DATABASE_URL),
pool_size=10,
max_overflow=20,
pool_timeout=30,
pool_recycle=1800,
pool_pre_ping=True,
)
AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
async def wait_db_ready():
ready = False
while not ready:
try:
async with engine.connect() as conn:
await conn.execute(text("SELECT 1"))
ready = True
except Exception as e:
make_log("SQL", 'PostgreSQL is not ready yet: ' + str(e), level='debug')
time.sleep(1)
@asynccontextmanager
async def db_session(auto_commit: bool = False):
session: AsyncSession = AsyncSessionLocal()
try:
with Session() as session:
databases_list = session.execute(text("SHOW DATABASES;"))
databases_list = [row[0] for row in databases_list]
make_log("SQL", 'Database list: ' + str(databases_list), level='debug')
assert MYSQL_DATABASE in databases_list, 'Database not found'
database_initialized = True
except Exception as e:
make_log("SQL", 'MariaDB is not ready yet: ' + str(e), level='debug')
time.sleep(1)
engine = create_engine(f"{MYSQL_URI}/{MYSQL_DATABASE}", poolclass=NullPool)
Session = sessionmaker(bind=engine)
@contextmanager
def db_session(auto_commit=False):
_session = Session()
try:
yield _session
if auto_commit is True:
_session.commit()
yield session
if auto_commit:
await session.commit()
except BaseException as e:
_session.rollback()
await session.rollback()
raise e
finally:
_session.close()
await session.close()
def new_session() -> AsyncSession:
return AsyncSessionLocal()
+15 -18
View File
@@ -1,19 +1,19 @@
from datetime import datetime
from sqlalchemy import select, and_, func
from app.core.logger import make_log
from app.core.models import Memory, User, UserBalance, Asset, InternalTransaction
from app.core.storage import db_session
def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
async def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
assert user, "No user"
assert asset, "No asset"
result = session.query(UserBalance).filter(
UserBalance.user_id == user.id,
UserBalance.asset_id == asset.id
)
results_count = result.count()
if results_count == 0:
result = await session.execute(select(UserBalance).where(
and_(UserBalance.user_id == user.id, UserBalance.asset_id == asset.id)
))
row = result.scalars().first()
if not row:
user_balance = UserBalance(
user_id=user.id,
asset_id=asset.id,
@@ -21,12 +21,9 @@ def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
created=datetime.now(),
)
session.add(user_balance)
session.commit()
return get_user_balance(session, user, asset)
elif results_count == 1:
return result.first()
else:
raise Exception(f"Multiple user balances found: {results_count}")
await session.commit()
return await get_user_balance(session, user, asset)
return row
async def make_internal_transaction(
@@ -46,13 +43,13 @@ async def make_internal_transaction(
raise Exception(f"Invalid amount: {amount}")
abs_amount = abs(amount)
with db_session(auto_commit=False) as session:
async with db_session(auto_commit=False) as session:
async with memory.transaction():
user = session.query(User).filter_by(id=user_id).first()
user = (await session.execute(select(User).where(User.id == user_id))).scalars().first()
assert user, "No user"
asset = session.query(Asset).filter_by(id=asset_id).first()
asset = (await session.execute(select(Asset).where(Asset.id == asset_id))).scalars().first()
assert asset, "No asset"
user_balance = get_user_balance(session, user, asset)
user_balance = await get_user_balance(session, user, asset)
assert user_balance, "No user balance"
if is_spent is True:
if abs_amount > user_balance.balance:
@@ -71,6 +68,6 @@ async def make_internal_transaction(
created=datetime.now(),
)
session.add(internal_transaction)
session.commit()
await session.commit()
make_log(user, f"Made internal transaction: {'-' if is_spent else ''}{abs_amount} {asset.symbol}, type: {type}")