mariadb -> postgres
This commit is contained in:
1 parent
21964fa986
commit
e51bb86dc0
36 files changed
+650
-464
No files matched your search
+2
-3
@@ -19,9 +19,8 @@ import httpx
|
||||
TELEGRAM_BOT_USERNAME = httpx.get(f"https://api.telegram.org/bot{TELEGRAM_API_KEY}/getMe").json()['result']['username']
|
||||
CLIENT_TELEGRAM_BOT_USERNAME = httpx.get(f"https://api.telegram.org/bot{CLIENT_TELEGRAM_API_KEY}/getMe").json()['result']['username']
|
||||
|
||||
|
||||
MYSQL_URI = os.environ['MYSQL_URI']
|
||||
MYSQL_DATABASE = os.environ['MYSQL_DATABASE']
|
||||
# Unified database URL (PostgreSQL)
|
||||
DATABASE_URL = os.environ['DATABASE_URL']
|
||||
|
||||
LOG_LEVEL = os.getenv('LOG_LEVEL', 'DEBUG')
|
||||
LOG_DIR = os.getenv('LOG_DIR', 'logs')
|
||||
|
||||
+11
-16
@@ -36,9 +36,10 @@ async def create_new_encryption_key(db_session, user_id: int = None) -> KnownKey
|
||||
meta={"I_user_id": user_id} if user_id else None,
|
||||
created=datetime.now()
|
||||
)
|
||||
from sqlalchemy import select
|
||||
db_session.add(new_key)
|
||||
db_session.commit()
|
||||
new_key = db_session.query(KnownKey).filter(KnownKey.seed_hash == new_seed_hash).first()
|
||||
await db_session.commit()
|
||||
new_key = (await db_session.execute(select(KnownKey).where(KnownKey.seed_hash == new_seed_hash))).scalars().first()
|
||||
assert new_key, "Key not created"
|
||||
return new_key
|
||||
|
||||
@@ -46,9 +47,10 @@ async def create_new_encryption_key(db_session, user_id: int = None) -> KnownKey
|
||||
async def create_encrypted_content(
|
||||
db_session, decrypted_content: StoredContent,
|
||||
) -> StoredContent:
|
||||
encrypted_content = db_session.query(StoredContent).filter(
|
||||
from sqlalchemy import select
|
||||
encrypted_content = (await db_session.execute(select(StoredContent).where(
|
||||
StoredContent.id == decrypted_content.decrypted_content_id
|
||||
).first()
|
||||
))).scalars().first()
|
||||
if encrypted_content:
|
||||
make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): already exist (found by decrypted content)", level="debug")
|
||||
return encrypted_content
|
||||
@@ -57,10 +59,8 @@ async def create_encrypted_content(
|
||||
if decrypted_content.key is None:
|
||||
key = await create_new_encryption_key(db_session, user_id=decrypted_content.user_id)
|
||||
decrypted_content.key_id = key.id
|
||||
db_session.commit()
|
||||
decrypted_content = db_session.query(StoredContent).filter(
|
||||
StoredContent.id == decrypted_content.id
|
||||
).first()
|
||||
await db_session.commit()
|
||||
decrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == decrypted_content.id))).scalars().first()
|
||||
assert decrypted_content.key_id, "Key not assigned"
|
||||
|
||||
decrypted_path = os.path.join(UPLOADS_DIR, decrypted_content.hash)
|
||||
@@ -72,9 +72,7 @@ async def create_encrypted_content(
|
||||
encrypted_bin = cipher.encrypt(decrypted_bin)
|
||||
encrypted_hash_bin = sha256(encrypted_bin).digest()
|
||||
encrypted_hash = b58encode(encrypted_hash_bin).decode()
|
||||
encrypted_content = db_session.query(StoredContent).filter(
|
||||
StoredContent.hash == encrypted_hash
|
||||
).first()
|
||||
encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash))).scalars().first()
|
||||
if encrypted_content:
|
||||
make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): already exist (found by encrypted_hash)", level="debug")
|
||||
return encrypted_content
|
||||
@@ -99,19 +97,16 @@ async def create_encrypted_content(
|
||||
created=datetime.now(),
|
||||
)
|
||||
db_session.add(encrypted_content)
|
||||
db_session.commit()
|
||||
await db_session.commit()
|
||||
|
||||
encrypted_path = os.path.join(UPLOADS_DIR, encrypted_hash)
|
||||
async with aiofiles.open(encrypted_path, mode='wb') as file:
|
||||
await file.write(encrypted_bin)
|
||||
|
||||
encrypted_content = db_session.query(StoredContent).filter(
|
||||
StoredContent.hash == encrypted_hash
|
||||
).first()
|
||||
encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.hash == encrypted_hash))).scalars().first()
|
||||
assert encrypted_content, "Content not created"
|
||||
make_log("create_encrypted_content", f"(d={decrypted_content.cid.serialize_v2()}) => (e={encrypted_content.cid.serialize_v2()}): created new content/bin", level="debug")
|
||||
return encrypted_content
|
||||
|
||||
|
||||
|
||||
|
||||
+27
-19
@@ -1,4 +1,6 @@
|
||||
import asyncio
|
||||
from os import getenv, urandom
|
||||
import os
|
||||
|
||||
from nacl.bindings import crypto_sign_seed_keypair
|
||||
from tonsdk.utils import Address
|
||||
@@ -7,38 +9,44 @@ from app.core._blockchain.ton.wallet_v3cr3 import WalletV3CR3
|
||||
from app.core.models._config import ServiceConfig
|
||||
from app.core.storage import db_session
|
||||
from app.core.logger import make_log
|
||||
import os
|
||||
|
||||
|
||||
def load_hot_pair():
|
||||
with db_session() as session:
|
||||
async def load_hot_pair_async():
|
||||
async with db_session() as session:
|
||||
service_config = ServiceConfig(session)
|
||||
hot_seed = service_config.get('private_key')
|
||||
hot_seed = await service_config.get('private_key')
|
||||
if hot_seed is None:
|
||||
make_log("HotWallet", "No seed found, generating new one", level='info')
|
||||
hot_seed = os.getenv("TON_INIT_HOT_SEED")
|
||||
if not hot_seed:
|
||||
hot_seed = urandom(32)
|
||||
hot_seed_env = os.getenv("TON_INIT_HOT_SEED")
|
||||
if not hot_seed_env:
|
||||
hot_seed_bytes = urandom(32)
|
||||
make_log("HotWallet", f"Generated random seed")
|
||||
else:
|
||||
hot_seed = bytes.fromhex(hot_seed)
|
||||
hot_seed_bytes = bytes.fromhex(hot_seed_env)
|
||||
make_log("HotWallet", f"Loaded seed from env")
|
||||
|
||||
service_config.set('private_key', hot_seed.hex())
|
||||
return load_hot_pair()
|
||||
await service_config.set('private_key', hot_seed_bytes.hex())
|
||||
hot_seed = hot_seed_bytes.hex()
|
||||
|
||||
hot_seed = bytes.fromhex(hot_seed)
|
||||
public_key, private_key = crypto_sign_seed_keypair(hot_seed)
|
||||
return hot_seed, public_key, private_key
|
||||
hot_seed_bytes = bytes.fromhex(hot_seed)
|
||||
public_key, private_key = crypto_sign_seed_keypair(hot_seed_bytes)
|
||||
return hot_seed_bytes, public_key, private_key
|
||||
|
||||
|
||||
_extra_ton_wallet_options = {}
|
||||
if getenv('TON_CUSTOM_WALLET_ADDRESS'):
|
||||
_extra_ton_wallet_options['address'] = Address(getenv('TON_CUSTOM_WALLET_ADDRESS'))
|
||||
|
||||
hot_seed, hot_pubkey, hot_privkey = load_hot_pair()
|
||||
service_wallet = WalletV3CR3(
|
||||
private_key=hot_privkey,
|
||||
public_key=hot_pubkey,
|
||||
**_extra_ton_wallet_options
|
||||
)
|
||||
|
||||
def _init_wallet():
|
||||
# Safe to call at import time; Sanic event loop not running yet
|
||||
hot_seed, hot_pubkey, hot_privkey = asyncio.run(load_hot_pair_async())
|
||||
wallet = WalletV3CR3(
|
||||
private_key=hot_privkey,
|
||||
public_key=hot_pubkey,
|
||||
**_extra_ton_wallet_options
|
||||
)
|
||||
return hot_seed, hot_pubkey, hot_privkey, wallet
|
||||
|
||||
|
||||
hot_seed, hot_pubkey, hot_privkey, service_wallet = _init_wallet()
|
||||
@@ -1,10 +1,12 @@
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from app.core.models import BlockchainTask
|
||||
from app.core.models.base import AlchemyBase
|
||||
|
||||
|
||||
def create_maria_tables(engine):
|
||||
"""Create all tables in the database."""
|
||||
async def create_db_tables(engine: AsyncEngine):
|
||||
"""Create all tables in the database (PostgreSQL, async)."""
|
||||
# ensure model import side-effects initialize mappers
|
||||
BlockchainTask()
|
||||
AlchemyBase.metadata.create_all(engine)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(AlchemyBase.metadata.create_all)
|
||||
|
||||
|
||||
+3
-2
@@ -56,9 +56,10 @@ class AuthenticationMixin:
|
||||
},
|
||||
created=datetime.fromtimestamp(init_ts)
|
||||
)
|
||||
from sqlalchemy import select
|
||||
db_session.add(new_key)
|
||||
db_session.commit()
|
||||
new_key = db_session.query(KnownKey).filter(KnownKey.seed_hash == new_key.seed_hash).first()
|
||||
await db_session.commit()
|
||||
new_key = (await db_session.execute(select(KnownKey).where(KnownKey.seed_hash == new_key.seed_hash))).scalars().first()
|
||||
assert new_key, "Key not created"
|
||||
make_log("auth", f"[new-K] User {user_id} created new {token_type} key {new_key.id}")
|
||||
return {
|
||||
|
||||
@@ -6,7 +6,7 @@ import json
|
||||
import shutil
|
||||
import magic # python-magic for MIME detection
|
||||
from base58 import b58decode, b58encode
|
||||
from sqlalchemy import and_, or_
|
||||
from sqlalchemy import and_, or_, select
|
||||
from app.core.models.node_storage import StoredContent
|
||||
from app.core.models._telegram import Wrapped_CBotChat
|
||||
from app.core._utils.send_status import send_status
|
||||
@@ -19,9 +19,9 @@ from app.core.content.content_id import ContentId
|
||||
|
||||
|
||||
async def convert_loop(memory):
|
||||
with db_session() as session:
|
||||
async with db_session() as session:
|
||||
# Query for unprocessed encrypted content
|
||||
unprocessed_encrypted_content = session.query(StoredContent).filter(
|
||||
unprocessed_encrypted_content = (await session.execute(select(StoredContent).where(
|
||||
and_(
|
||||
StoredContent.type == "onchain/content",
|
||||
or_(
|
||||
@@ -29,15 +29,15 @@ async def convert_loop(memory):
|
||||
StoredContent.ipfs_cid == None,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
))).scalars().first()
|
||||
if not unprocessed_encrypted_content:
|
||||
make_log("ConvertProcess", "No content to convert", level="debug")
|
||||
return
|
||||
|
||||
# Достаем расшифрованный файл
|
||||
decrypted_content = session.query(StoredContent).filter(
|
||||
decrypted_content = (await session.execute(select(StoredContent).where(
|
||||
StoredContent.id == unprocessed_encrypted_content.decrypted_content_id
|
||||
).first()
|
||||
))).scalars().first()
|
||||
if not decrypted_content:
|
||||
make_log("ConvertProcess", "Decrypted content not found", level="error")
|
||||
return
|
||||
@@ -78,7 +78,7 @@ async def convert_loop(memory):
|
||||
option_name: decrypted_content.hash for option_name in ['high', 'low', 'low_preview']
|
||||
}
|
||||
}
|
||||
session.commit()
|
||||
await session.commit()
|
||||
return
|
||||
|
||||
# ==== Конвертация для видео или аудио: оригинальная логика ====
|
||||
@@ -171,9 +171,7 @@ async def convert_loop(memory):
|
||||
file_hash = b58encode(bytes.fromhex(file_hash)).decode()
|
||||
|
||||
# Save new StoredContent if not exists
|
||||
if not session.query(StoredContent).filter(
|
||||
StoredContent.hash == file_hash
|
||||
).first():
|
||||
if not (await session.execute(select(StoredContent).where(StoredContent.hash == file_hash))).scalars().first():
|
||||
new_content = StoredContent(
|
||||
type="local/content_bin",
|
||||
hash=file_hash,
|
||||
@@ -183,7 +181,7 @@ async def convert_loop(memory):
|
||||
created=datetime.now(),
|
||||
)
|
||||
session.add(new_content)
|
||||
session.commit()
|
||||
await session.commit()
|
||||
|
||||
save_path = os.path.join(UPLOADS_DIR, file_hash)
|
||||
try:
|
||||
@@ -233,13 +231,13 @@ async def convert_loop(memory):
|
||||
**unprocessed_encrypted_content.meta,
|
||||
'converted_content': converted_content
|
||||
}
|
||||
session.commit()
|
||||
await session.commit()
|
||||
|
||||
# Notify user if needed
|
||||
if not unprocessed_encrypted_content.meta.get('upload_notify_msg_id'):
|
||||
wallet_owner_connection = session.query(WalletConnection).filter(
|
||||
wallet_owner_connection = (await session.execute(select(WalletConnection).where(
|
||||
WalletConnection.wallet_address == unprocessed_encrypted_content.owner_address
|
||||
).order_by(WalletConnection.id.desc()).first()
|
||||
).order_by(WalletConnection.id.desc()))).scalars().first()
|
||||
if wallet_owner_connection:
|
||||
wallet_owner_user = wallet_owner_connection.user
|
||||
bot = Wrapped_CBotChat(
|
||||
@@ -249,7 +247,7 @@ async def convert_loop(memory):
|
||||
db_session=session
|
||||
)
|
||||
unprocessed_encrypted_content.meta['upload_notify_msg_id'] = await bot.send_content(session, unprocessed_encrypted_content)
|
||||
session.commit()
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def main_fn(memory):
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.core._utils.resolve_content import resolve_content
|
||||
from app.core.models.wallet_connection import WalletConnection
|
||||
from app.core._keyboards import get_inline_keyboard
|
||||
from app.core.models._telegram import Wrapped_CBotChat
|
||||
from sqlalchemy import select, and_, desc
|
||||
from app.core.storage import db_session
|
||||
import os
|
||||
import traceback
|
||||
@@ -33,7 +34,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
platform_found = True
|
||||
|
||||
make_log("Indexer", "Service running", level="debug")
|
||||
with db_session() as session:
|
||||
async with db_session() as session:
|
||||
try:
|
||||
result = await toncenter.run_get_method('EQD8TJ8xEWB1SpnRE4d89YO3jl0W0EiBnNS4IBaHaUmdfizE', 'get_pool_data')
|
||||
assert result['exit_code'] == 0, f"Error in get-method: {result}"
|
||||
@@ -41,21 +42,21 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
assert result['stack'][1][0] == 'num', f"get second element is not num"
|
||||
usdt_per_ton = (int(result['stack'][0][1], 16) * 1e3) / int(result['stack'][1][1], 16)
|
||||
ton_per_star = 0.014 / usdt_per_ton
|
||||
ServiceConfig(session).set('live_tonPerStar', [ton_per_star, datetime.utcnow().timestamp()])
|
||||
await ServiceConfig(session).set('live_tonPerStar', [ton_per_star, datetime.utcnow().timestamp()])
|
||||
make_log("TON_Daemon", f"TON per STAR price: {ton_per_star}", level="DEBUG")
|
||||
except BaseException as e:
|
||||
make_log("TON_Daemon", f"Error while saving TON per STAR price: {e}" + '\n' + traceback.format_exc(), level="ERROR")
|
||||
|
||||
new_licenses = session.query(UserContent).filter(
|
||||
new_licenses = (await session.execute(select(UserContent).where(
|
||||
and_(
|
||||
~UserContent.meta.contains({'notification_sent': True}),
|
||||
UserContent.type == 'nft/listen'
|
||||
)
|
||||
).all()
|
||||
))).scalars().all()
|
||||
for new_license in new_licenses:
|
||||
licensed_content = session.query(StoredContent).filter(
|
||||
licensed_content = (await session.execute(select(StoredContent).where(
|
||||
StoredContent.id == new_license.content_id
|
||||
).first()
|
||||
))).scalars().first()
|
||||
if not licensed_content:
|
||||
make_log("Indexer", f"Licensed content not found: {new_license.content_id}", level="error")
|
||||
|
||||
@@ -70,10 +71,12 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
session, licensed_content
|
||||
)
|
||||
|
||||
wallet_owner_connection = session.query(WalletConnection).filter_by(
|
||||
wallet_address=licensed_content.owner_address,
|
||||
invalidated=False
|
||||
).order_by(desc(WalletConnection.id)).first()
|
||||
wallet_owner_connection = (await session.execute(
|
||||
select(WalletConnection).where(
|
||||
WalletConnection.wallet_address == licensed_content.owner_address,
|
||||
WalletConnection.invalidated == False
|
||||
).order_by(desc(WalletConnection.id))
|
||||
)).scalars().first()
|
||||
wallet_owner_user = wallet_owner_connection.user
|
||||
if wallet_owner_user.telegram_id:
|
||||
wallet_owner_bot = Wrapped_CBotChat(memory._telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
|
||||
@@ -89,21 +92,19 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
make_log("IndexerSendNewLicense", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
|
||||
new_license.meta = {**new_license.meta, 'notification_sent': True}
|
||||
session.commit()
|
||||
await session.commit()
|
||||
|
||||
content_without_cid = session.query(StoredContent).filter(
|
||||
StoredContent.content_id == None
|
||||
)
|
||||
content_without_cid = (await session.execute(select(StoredContent).where(StoredContent.content_id == None))).scalars().all()
|
||||
for target_content in content_without_cid:
|
||||
target_cid = target_content.cid.serialize_v2()
|
||||
make_log("Indexer", f"Content without CID: {target_content.hash}, setting CID: {target_cid}", level="debug")
|
||||
target_content.content_id = target_cid
|
||||
|
||||
session.commit()
|
||||
await session.commit()
|
||||
|
||||
last_known_index_ = session.query(StoredContent).filter(
|
||||
StoredContent.onchain_index != None
|
||||
).order_by(StoredContent.onchain_index.desc()).first()
|
||||
last_known_index_ = (await session.execute(
|
||||
select(StoredContent).where(StoredContent.onchain_index != None).order_by(StoredContent.onchain_index.desc())
|
||||
)).scalars().first()
|
||||
last_known_index = last_known_index_.onchain_index if last_known_index_ else 0
|
||||
last_known_index = max(last_known_index, 0)
|
||||
make_log("Indexer", f"Last known index: {last_known_index}", level="debug")
|
||||
@@ -196,14 +197,13 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
|
||||
user_wallet_connection = None
|
||||
if item_owner_address:
|
||||
user_wallet_connection = session.query(WalletConnection).filter(
|
||||
user_wallet_connection = (await session.execute(select(WalletConnection).where(
|
||||
WalletConnection.wallet_address == item_owner_address.to_string(1, 1, 1)
|
||||
).first()
|
||||
))).scalars().first()
|
||||
|
||||
encrypted_stored_content = session.query(StoredContent).filter(
|
||||
StoredContent.hash == item_content_hash_str,
|
||||
# StoredContent.type.like("local%")
|
||||
).first()
|
||||
encrypted_stored_content = (await session.execute(select(StoredContent).where(
|
||||
StoredContent.hash == item_content_hash_str
|
||||
))).scalars().first()
|
||||
if encrypted_stored_content:
|
||||
is_duplicate = encrypted_stored_content.type.startswith("onchain") \
|
||||
and encrypted_stored_content.onchain_index != item_index
|
||||
@@ -234,14 +234,15 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
)
|
||||
|
||||
try:
|
||||
for hint_message in session.query(KnownTelegramMessage).filter(
|
||||
result = await session.execute(select(KnownTelegramMessage).where(
|
||||
and_(
|
||||
KnownTelegramMessage.chat_id == user.telegram_id,
|
||||
KnownTelegramMessage.type == 'hint',
|
||||
cast(KnownTelegramMessage.meta['encrypted_content_hash'], String) == encrypted_stored_content.hash,
|
||||
KnownTelegramMessage.deleted == False
|
||||
)
|
||||
).all():
|
||||
))
|
||||
for hint_message in result.scalars().all():
|
||||
await user_uploader_wrapper.delete_message(hint_message.message_id)
|
||||
except BaseException as e:
|
||||
make_log("Indexer", f"Error while deleting hint messages: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
@@ -260,7 +261,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
**item_metadata_packed
|
||||
}
|
||||
|
||||
session.commit()
|
||||
await session.commit()
|
||||
return platform_found, seqno
|
||||
else:
|
||||
item_metadata_packed['copied_from'] = encrypted_stored_content.id
|
||||
@@ -282,7 +283,7 @@ async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
updated=datetime.now()
|
||||
)
|
||||
session.add(onchain_stored_content)
|
||||
session.commit()
|
||||
await session.commit()
|
||||
make_log("Indexer", f"Item indexed: {item_content_hash_str}", level="info")
|
||||
last_known_index += 1
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ from base64 import b64decode
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from base58 import b58encode
|
||||
from sqlalchemy import and_, or_
|
||||
from sqlalchemy import and_, or_, select, desc
|
||||
from tonsdk.boc import Cell
|
||||
from tonsdk.utils import Address
|
||||
|
||||
@@ -27,7 +27,7 @@ import traceback
|
||||
|
||||
async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
make_log("LicenseIndex", "Service running", level="debug")
|
||||
with db_session() as session:
|
||||
async with db_session() as session:
|
||||
async def check_telegram_stars_transactions():
|
||||
# Проверка звездных telegram транзакций, обновление paid
|
||||
offset = {'desc': 'Статичное число заранее известного количества транзакций, которое даже не знает наш бот', 'value': 1}['value'] + \
|
||||
@@ -45,19 +45,19 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
|
||||
continue
|
||||
|
||||
try:
|
||||
existing_invoice = session.query(StarsInvoice).filter(
|
||||
existing_invoice = (await session.execute(select(StarsInvoice).where(
|
||||
StarsInvoice.external_id == star_payment.source.invoice_payload
|
||||
).first()
|
||||
))).scalars().first()
|
||||
if not existing_invoice:
|
||||
continue
|
||||
|
||||
if star_payment.amount == existing_invoice.amount:
|
||||
if not existing_invoice.paid:
|
||||
existing_invoice.paid = True
|
||||
session.commit()
|
||||
await session.commit()
|
||||
|
||||
licensed_content = session.query(StoredContent).filter(StoredContent.hash == existing_invoice.content_hash).first()
|
||||
user = session.query(User).filter(User.id == existing_invoice.user_id).first()
|
||||
licensed_content = (await session.execute(select(StoredContent).where(StoredContent.hash == existing_invoice.content_hash))).scalars().first()
|
||||
user = (await session.execute(select(User).where(User.id == existing_invoice.user_id))).scalars().first()
|
||||
|
||||
await (Wrapped_CBotChat(memory._client_telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)).send_content(
|
||||
session, licensed_content
|
||||
@@ -73,9 +73,10 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
|
||||
make_log("StarsProcessing", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
|
||||
# Проверка кошельков пользователей на появление новых NFT, добавление их в базу как неопознанные
|
||||
for user in session.query(User).filter(
|
||||
users = (await session.execute(select(User).where(
|
||||
User.last_use > datetime.now() - timedelta(hours=4)
|
||||
).order_by(User.updated.asc()).all():
|
||||
).order_by(User.updated.asc()))).scalars().all()
|
||||
for user in users:
|
||||
user_wallet_address = user.wallet_address(session)
|
||||
if not user_wallet_address:
|
||||
make_log("LicenseIndex", f"User {user.id} has no wallet address", level="info")
|
||||
@@ -91,17 +92,17 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
|
||||
try:
|
||||
await user.scan_owned_user_content(session)
|
||||
user.meta = {**user.meta, 'last_updated_licenses': datetime.now().isoformat()}
|
||||
session.commit()
|
||||
await session.commit()
|
||||
except BaseException as e:
|
||||
make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
|
||||
# Проверка NFT на актуальность данных, в том числе уже проверенные
|
||||
process_content = session.query(UserContent).filter(
|
||||
process_content = (await session.execute(select(UserContent).where(
|
||||
and_(
|
||||
UserContent.type.startswith('nft/'),
|
||||
UserContent.updated < (datetime.now() - timedelta(minutes=60)),
|
||||
)
|
||||
).order_by(UserContent.updated.asc()).first()
|
||||
).order_by(UserContent.updated.asc()))).scalars().first()
|
||||
if process_content:
|
||||
make_log("LicenseIndex", f"Syncing content with blockchain: {process_content.id}", level="info")
|
||||
try:
|
||||
@@ -110,7 +111,7 @@ async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool,
|
||||
make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
finally:
|
||||
process_content.updated = datetime.now()
|
||||
session.commit()
|
||||
await session.commit()
|
||||
|
||||
return platform_found, seqno
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ async def main_fn(memory):
|
||||
sw_seqno_value = await get_sw_seqno()
|
||||
make_log("TON", f"Service running ({sw_seqno_value})", level="debug")
|
||||
|
||||
with db_session() as session:
|
||||
async with db_session() as session:
|
||||
# Проверка отправленных сообщений
|
||||
await send_status("ton_daemon", f"working: processing in-txs (seqno={sw_seqno_value})")
|
||||
async def process_incoming_transaction(transaction: dict):
|
||||
@@ -142,14 +142,17 @@ async def main_fn(memory):
|
||||
in_msg_created_at = in_msg_slice.read_uint(64)
|
||||
in_msg_epoch = int(in_msg_created_at // (60 * 60))
|
||||
in_msg_seqno = HighloadQueryId.from_query_id(in_msg_query_id).to_seqno()
|
||||
from sqlalchemy import select
|
||||
in_msg_blockchain_task = (
|
||||
session.query(BlockchainTask).filter(
|
||||
and_(
|
||||
BlockchainTask.seqno == in_msg_seqno,
|
||||
BlockchainTask.epoch == in_msg_epoch,
|
||||
await session.execute(
|
||||
select(BlockchainTask).where(
|
||||
and_(
|
||||
BlockchainTask.seqno == in_msg_seqno,
|
||||
BlockchainTask.epoch == in_msg_epoch,
|
||||
)
|
||||
)
|
||||
)
|
||||
).first()
|
||||
).scalars().first()
|
||||
if not in_msg_blockchain_task:
|
||||
return
|
||||
|
||||
@@ -157,7 +160,7 @@ async def main_fn(memory):
|
||||
in_msg_blockchain_task.status = 'done'
|
||||
in_msg_blockchain_task.transaction_hash = transaction_hash
|
||||
in_msg_blockchain_task.transaction_lt = transaction_lt
|
||||
session.commit()
|
||||
await session.commit()
|
||||
|
||||
for blockchain_message in [transaction['in_msg']]:
|
||||
try:
|
||||
@@ -177,11 +180,11 @@ async def main_fn(memory):
|
||||
|
||||
await send_status("ton_daemon", f"working: processing out-txs (seqno={sw_seqno_value})")
|
||||
# Отправка подписанных сообщений
|
||||
for blockchain_task in (
|
||||
session.query(BlockchainTask).filter(
|
||||
BlockchainTask.status == 'processing',
|
||||
).order_by(BlockchainTask.updated.asc()).all()
|
||||
):
|
||||
from sqlalchemy import select
|
||||
_processing = (await session.execute(select(BlockchainTask).where(
|
||||
BlockchainTask.status == 'processing'
|
||||
).order_by(BlockchainTask.updated.asc()))).scalars().all()
|
||||
for blockchain_task in _processing:
|
||||
make_log("TON_Daemon", f"Processing task (processing) {blockchain_task.id}")
|
||||
query_boc = bytes.fromhex(blockchain_task.meta['signed_message'])
|
||||
errors_list = []
|
||||
@@ -210,23 +213,22 @@ async def main_fn(memory):
|
||||
# or sum([int("terminating vm with exit code 36" in e) for e in errors_list]) > 0:
|
||||
make_log("TON_Daemon", f"Task {blockchain_task.id} done", level="DEBUG")
|
||||
blockchain_task.status = 'done'
|
||||
session.commit()
|
||||
await session.commit()
|
||||
continue
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await send_status("ton_daemon", f"working: creating new messages (seqno={sw_seqno_value})")
|
||||
# Создание новых подписей
|
||||
for blockchain_task in (
|
||||
session.query(BlockchainTask).filter(BlockchainTask.status == 'wait').all()
|
||||
):
|
||||
_waiting = (await session.execute(select(BlockchainTask).where(BlockchainTask.status == 'wait'))).scalars().all()
|
||||
for blockchain_task in _waiting:
|
||||
try:
|
||||
# Check processing tasks in current epoch < 3_000_000
|
||||
if (
|
||||
session.query(BlockchainTask).filter(
|
||||
BlockchainTask.epoch == blockchain_task.epoch,
|
||||
).count() > 3_000_000
|
||||
):
|
||||
from sqlalchemy import func
|
||||
_cnt = (await session.execute(select(func.count()).select_from(BlockchainTask).where(
|
||||
BlockchainTask.epoch == blockchain_task.epoch
|
||||
))).scalar() or 0
|
||||
if _cnt > 3_000_000:
|
||||
make_log("TON", f"Too many processing tasks in epoch {blockchain_task.epoch}", level="error")
|
||||
await send_status("ton_daemon", f"working: too many tasks in epoch {blockchain_task.epoch}")
|
||||
await asyncio.sleep(5)
|
||||
@@ -235,10 +237,11 @@ async def main_fn(memory):
|
||||
sign_created = int(datetime.utcnow().timestamp()) - 60
|
||||
try:
|
||||
current_epoch = int(datetime.utcnow().timestamp() // (60 * 60))
|
||||
from sqlalchemy import func
|
||||
max_epoch_seqno = (
|
||||
session.query(func.max(BlockchainTask.seqno)).filter(
|
||||
(await session.execute(select(func.max(BlockchainTask.seqno)).where(
|
||||
BlockchainTask.epoch == current_epoch
|
||||
).scalar() or 0
|
||||
))).scalar() or 0
|
||||
)
|
||||
current_epoch_shift = 3_000_000 if current_epoch % 2 == 0 else 0
|
||||
current_seqno = max_epoch_seqno + 1 + (current_epoch_shift if max_epoch_seqno == 0 else 0)
|
||||
@@ -266,7 +269,7 @@ async def main_fn(memory):
|
||||
'sign_created': sign_created,
|
||||
'signed_message': query_boc.hex(),
|
||||
}
|
||||
session.commit()
|
||||
await session.commit()
|
||||
make_log("TON", f"Created signed message for task {blockchain_task.id}" + '\n' + traceback.format_exc(), level="info")
|
||||
except BaseException as e:
|
||||
make_log("TON", f"Error processing task {blockchain_task.id}: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
@@ -287,4 +290,3 @@ async def main_fn(memory):
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -26,7 +26,9 @@ async def create_new_content(
|
||||
content_hash_bin = sha256(content_bin).digest()
|
||||
content_hash_b58 = b58encode(content_hash_bin).decode()
|
||||
|
||||
new_content = db_session.query(StoredContent).filter(StoredContent.hash == content_hash_b58).first()
|
||||
from sqlalchemy import select
|
||||
result = await db_session.execute(select(StoredContent).where(StoredContent.hash == content_hash_b58))
|
||||
new_content = result.scalars().first()
|
||||
if new_content:
|
||||
return new_content, False
|
||||
|
||||
@@ -38,8 +40,9 @@ async def create_new_content(
|
||||
|
||||
)
|
||||
db_session.add(new_content)
|
||||
db_session.commit()
|
||||
new_content = db_session.query(StoredContent).filter(StoredContent.hash == content_hash_b58).first()
|
||||
await db_session.commit()
|
||||
result = await db_session.execute(select(StoredContent).where(StoredContent.hash == content_hash_b58))
|
||||
new_content = result.scalars().first()
|
||||
assert new_content, "Content not created (through utils)"
|
||||
content_filepath = os.path.join(UPLOADS_DIR, content_hash_b58)
|
||||
async with aiofiles.open(content_filepath, 'wb') as file:
|
||||
|
||||
+12
-14
@@ -1,6 +1,6 @@
|
||||
|
||||
from app.core.models.base import AlchemyBase
|
||||
from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
|
||||
from sqlalchemy import Column, Integer, String, JSON, select
|
||||
|
||||
|
||||
class ServiceConfigValue(AlchemyBase):
|
||||
@@ -19,20 +19,18 @@ class ServiceConfig:
|
||||
def __init__(self, session):
|
||||
self.session = session
|
||||
|
||||
def get(self, key, default=None):
|
||||
result = self.session.query(ServiceConfigValue).filter(ServiceConfigValue.key == key).first()
|
||||
async def get(self, key, default=None):
|
||||
result = (await self.session.execute(select(ServiceConfigValue).where(ServiceConfigValue.key == key))).scalars().first()
|
||||
return (result.value if result else None) or default
|
||||
|
||||
def set(self, key, value):
|
||||
config_value = self.session.query(ServiceConfigValue).filter(
|
||||
ServiceConfigValue.key == key
|
||||
).first()
|
||||
if not config_value:
|
||||
config_value = ServiceConfigValue(key=key)
|
||||
self.session.add(config_value)
|
||||
self.session.commit()
|
||||
return self.set(key, value)
|
||||
async def set(self, key, value):
|
||||
result = (await self.session.execute(select(ServiceConfigValue).where(ServiceConfigValue.key == key))).scalars().first()
|
||||
if not result:
|
||||
result = ServiceConfigValue(key=key)
|
||||
self.session.add(result)
|
||||
await self.session.commit()
|
||||
return await self.set(key, value)
|
||||
|
||||
config_value.packed_value = {'value': value}
|
||||
self.session.commit()
|
||||
result.packed_value = {'value': value}
|
||||
await self.session.commit()
|
||||
return
|
||||
@@ -1,4 +1,4 @@
|
||||
from sqlalchemy import and_
|
||||
from sqlalchemy import and_, select
|
||||
from app.core.models.node_storage import StoredContent
|
||||
from app.core.models.content.user_content import UserContent, UserAction
|
||||
from app.core.logger import make_log
|
||||
@@ -27,17 +27,17 @@ class PlayerTemplates:
|
||||
if not content.encrypted:
|
||||
local_content = content
|
||||
else:
|
||||
local_content = db_session.query(StoredContent).filter_by(
|
||||
id=content.decrypted_content_id
|
||||
).first()
|
||||
local_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == content.decrypted_content_id))).scalars().first()
|
||||
# TODO: add check decrypted_content by .format_json()['content_cid']
|
||||
if local_content:
|
||||
cd_log += f"Decrypted: {local_content.hash}. "
|
||||
else:
|
||||
cd_log += "Can't decrypt content. "
|
||||
|
||||
user_wallet_address = self.user.wallet_address(self.db_session)
|
||||
user_existing_license = self.db_session.query(UserContent).filter_by(user_id=self.user.id, content_id=content.id).first()
|
||||
user_wallet_address = await self.user.wallet_address_async(self.db_session)
|
||||
user_existing_license = (await self.db_session.execute(select(UserContent).where(
|
||||
and_(UserContent.user_id == self.user.id, UserContent.content_id == content.id)
|
||||
))).scalars().first()
|
||||
|
||||
if local_content:
|
||||
content_meta = content.json_format()
|
||||
@@ -48,12 +48,12 @@ class PlayerTemplates:
|
||||
except:
|
||||
content_type, content_encoding = 'application', 'x-binary'
|
||||
|
||||
content_metadata = StoredContent.from_cid(db_session, content_meta.get('metadata_cid') or None)
|
||||
content_metadata = await StoredContent.from_cid_async(db_session, content_meta.get('metadata_cid') or None)
|
||||
with open(content_metadata.filepath, 'r') as f:
|
||||
content_metadata_json = json.loads(f.read())
|
||||
|
||||
try:
|
||||
cover_content = StoredContent.from_cid(self.db_session, content_meta.get('cover_cid') or None)
|
||||
cover_content = await StoredContent.from_cid_async(self.db_session, content_meta.get('cover_cid') or None)
|
||||
cd_log += f"Cover content: {cover_content.cid.serialize_v2()}. "
|
||||
except BaseException as e:
|
||||
cd_log += f"Can't get cover content: {e}. "
|
||||
@@ -88,12 +88,15 @@ class PlayerTemplates:
|
||||
<blockquote><a href="{content_share_link['url']}">🔴 «открыть в MY»</a></blockquote>"""
|
||||
|
||||
make_log("TG-Player", f"Send content {content_type} ({content_encoding}) to chat {self._chat_id}. {cd_log}")
|
||||
for kmsg in self.db_session.query(KnownTelegramMessage).filter_by(
|
||||
content_id=content.id,
|
||||
chat_id=self._chat_id,
|
||||
type=f'content/{content_type}',
|
||||
deleted=False
|
||||
).all():
|
||||
kmsgs = (await self.db_session.execute(select(KnownTelegramMessage).where(
|
||||
and_(
|
||||
KnownTelegramMessage.content_id == content.id,
|
||||
KnownTelegramMessage.chat_id == self._chat_id,
|
||||
KnownTelegramMessage.type == f'content/{content_type}',
|
||||
KnownTelegramMessage.deleted == False
|
||||
)
|
||||
))).scalars().all()
|
||||
for kmsg in kmsgs:
|
||||
await self.delete_message(kmsg.message_id)
|
||||
|
||||
r = await tg_process_template(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from aiogram import Bot, types
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import and_
|
||||
from sqlalchemy import and_, select
|
||||
|
||||
from app.core.logger import make_log
|
||||
from app.core.models.messages import KnownTelegramMessage
|
||||
@@ -46,14 +46,15 @@ class Wrapped_CBotChat(T, PlayerTemplates):
|
||||
if self.db_session:
|
||||
if message_type == 'common':
|
||||
ci = 0
|
||||
for oc_msg in self.db_session.query(KnownTelegramMessage).filter(
|
||||
result = await self.db_session.execute(select(KnownTelegramMessage).where(
|
||||
and_(
|
||||
KnownTelegramMessage.type == 'common',
|
||||
KnownTelegramMessage.bot_id == self.bot_id,
|
||||
KnownTelegramMessage.chat_id == self._chat_id,
|
||||
KnownTelegramMessage.deleted == False
|
||||
)
|
||||
).all():
|
||||
))
|
||||
for oc_msg in result.scalars().all():
|
||||
make_log(self, f"Delete old message {oc_msg.message_id} {oc_msg.type} {oc_msg.bot_id} {oc_msg.chat_id}")
|
||||
await self.delete_message(oc_msg.message_id)
|
||||
ci += 1
|
||||
@@ -75,7 +76,7 @@ class Wrapped_CBotChat(T, PlayerTemplates):
|
||||
content_id=content_id
|
||||
)
|
||||
)
|
||||
self.db_session.commit()
|
||||
await self.db_session.commit()
|
||||
else:
|
||||
make_log(self, f"Unknown result type: {type(result)}", level='warning')
|
||||
|
||||
@@ -127,14 +128,16 @@ class Wrapped_CBotChat(T, PlayerTemplates):
|
||||
message_id
|
||||
)):
|
||||
if self.db_session:
|
||||
known_message = self.db_session.query(KnownTelegramMessage).filter(
|
||||
KnownTelegramMessage.bot_id == self.bot_id,
|
||||
KnownTelegramMessage.chat_id == self._chat_id,
|
||||
KnownTelegramMessage.message_id == message_id
|
||||
).first()
|
||||
known_message = (await self.db_session.execute(select(KnownTelegramMessage).where(
|
||||
and_(
|
||||
KnownTelegramMessage.bot_id == self.bot_id,
|
||||
KnownTelegramMessage.chat_id == self._chat_id,
|
||||
KnownTelegramMessage.message_id == message_id
|
||||
)
|
||||
))).scalars().first()
|
||||
if known_message:
|
||||
known_message.deleted = True
|
||||
self.db_session.commit()
|
||||
await self.db_session.commit()
|
||||
except Exception as e:
|
||||
make_log(self, f"Error deleting message {self._chat_id}/{message_id}. Error: {e}", level='warning')
|
||||
return None
|
||||
|
||||
+14
-15
@@ -29,22 +29,21 @@ class Asset(AlchemyBase):
|
||||
AlchemyBase.metadata.create_all(engine)
|
||||
|
||||
@classmethod
|
||||
def find(cls, session, **kwargs):
|
||||
async def find_async(cls, session, **kwargs):
|
||||
from sqlalchemy import select, func
|
||||
if 'symbol' in kwargs:
|
||||
kwargs['symbol'] = kwargs['symbol'].upper()
|
||||
|
||||
result = session.query(cls).filter_by(**kwargs)
|
||||
results_count = result.count()
|
||||
if results_count == 0:
|
||||
any_count = session.query(cls).count()
|
||||
if any_count == 0:
|
||||
init_asset = cls(**DEFAULT_ASSET_INITOBJ)
|
||||
session.add(init_asset)
|
||||
session.commit()
|
||||
return cls.find(session, **kwargs)
|
||||
result = await session.execute(select(cls).filter_by(**kwargs))
|
||||
row = result.scalars().first()
|
||||
if row:
|
||||
return row
|
||||
|
||||
raise Exception(f"Asset not found: {kwargs}")
|
||||
elif results_count == 1:
|
||||
return result.first()
|
||||
else:
|
||||
raise Exception(f"Multiple assets found: {results_count}")
|
||||
any_count = (await session.execute(select(func.count()).select_from(cls))).scalar() or 0
|
||||
if any_count == 0:
|
||||
init_asset = cls(**DEFAULT_ASSET_INITOBJ)
|
||||
session.add(init_asset)
|
||||
await session.commit()
|
||||
return await cls.find_async(session, **kwargs)
|
||||
|
||||
raise Exception(f"Asset not found: {kwargs}")
|
||||
@@ -1,7 +1,7 @@
|
||||
import traceback
|
||||
|
||||
import base58
|
||||
from sqlalchemy import and_
|
||||
from sqlalchemy import and_, select
|
||||
|
||||
from app.core.logger import make_log
|
||||
from app.core.models import StoredContent
|
||||
@@ -57,13 +57,9 @@ class UserContentIndexationMixin:
|
||||
values_slice = cc_indexator_data['values'].begin_parse()
|
||||
content_hash_b58 = base58.b58encode(bytes.fromhex(hex(values_slice.read_uint(256))[2:])).decode()
|
||||
make_log("UserContent", f"License ({self.onchain_address}) content hash: {content_hash_b58}", level="info")
|
||||
stored_content = db_session.query(StoredContent).filter(
|
||||
and_(
|
||||
StoredContent.type == 'onchain/content',
|
||||
StoredContent.hash == content_hash_b58,
|
||||
|
||||
)
|
||||
).first()
|
||||
stored_content = (await db_session.execute(select(StoredContent).where(
|
||||
and_(StoredContent.type == 'onchain/content', StoredContent.hash == content_hash_b58)
|
||||
))).scalars().first()
|
||||
trusted_cop_address_result = await toncenter.run_get_method(stored_content.meta['item_address'], 'get_nft_address_by_index', [['num', cc_indexator_data['index']]])
|
||||
assert trusted_cop_address_result.get('exit_code', -1) == 0, "Trusted cop address error"
|
||||
trusted_cop_address = Cell.one_from_boc(b64decode(trusted_cop_address_result['stack'][0][1]['bytes'])).begin_parse().read_msg_addr().to_string(1, 1, 1)
|
||||
@@ -72,7 +68,7 @@ class UserContentIndexationMixin:
|
||||
self.owner_address = cc_indexator_data['owner_address']
|
||||
self.type = 'nft/listen'
|
||||
self.content_id = stored_content.id
|
||||
db_session.commit()
|
||||
await db_session.commit()
|
||||
except BaseException as e:
|
||||
errored = True
|
||||
make_log("UserContent", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
@@ -80,7 +76,6 @@ class UserContentIndexationMixin:
|
||||
if errored is True:
|
||||
self.type = 'nft/unknown'
|
||||
self.content_id = None
|
||||
db_session.commit()
|
||||
|
||||
await db_session.commit()
|
||||
|
||||
|
||||
@@ -96,6 +96,30 @@ class StoredContent(AlchemyBase, AudioContentMixin):
|
||||
make_log("NodeStorage.open_content", f"Can't open content: {self.id} {e}", level='warning')
|
||||
raise e
|
||||
|
||||
async def open_content_async(self, db_session, content_type=None):
|
||||
from sqlalchemy import select
|
||||
try:
|
||||
decrypted_content = self if not self.encrypted else None
|
||||
encrypted_content = self if self.encrypted else None
|
||||
if not decrypted_content:
|
||||
decrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.id == self.decrypted_content_id))).scalars().first()
|
||||
else:
|
||||
encrypted_content = (await db_session.execute(select(StoredContent).where(StoredContent.decrypted_content_id == self.id))).scalars().first()
|
||||
|
||||
assert decrypted_content, "Can't get decrypted content"
|
||||
assert encrypted_content, "Can't get encrypted content"
|
||||
_ct = content_type or decrypted_content.json_format()['content_type']
|
||||
content_type = _ct.split('/')[0] if _ct else 'application'
|
||||
|
||||
return {
|
||||
'encrypted_content': encrypted_content,
|
||||
'decrypted_content': decrypted_content,
|
||||
'content_type': content_type or 'application/x-binary'
|
||||
}
|
||||
except BaseException as e:
|
||||
make_log("NodeStorage.open_content_async", f"Can't open content: {self.id} {e}", level='warning')
|
||||
raise e
|
||||
|
||||
def json_format(self):
|
||||
extra_fields = {}
|
||||
if self.type.startswith('local'):
|
||||
@@ -155,3 +179,15 @@ class StoredContent(AlchemyBase, AudioContentMixin):
|
||||
assert content, "Content not found"
|
||||
return content
|
||||
|
||||
@classmethod
|
||||
async def from_cid_async(cls, db_session, content_id):
|
||||
from sqlalchemy import select
|
||||
if isinstance(content_id, str):
|
||||
cid = ContentId.deserialize(content_id)
|
||||
else:
|
||||
cid = content_id
|
||||
|
||||
result = await db_session.execute(select(StoredContent).where(StoredContent.hash == cid.content_hash_b58))
|
||||
content = result.scalars().first()
|
||||
assert content, "Content not found"
|
||||
return content
|
||||
@@ -10,18 +10,21 @@ from httpx import AsyncClient
|
||||
|
||||
|
||||
class WalletMixin:
|
||||
def wallet_connection(self, db_session):
|
||||
return db_session.query(WalletConnection).filter(
|
||||
WalletConnection.user_id == self.id,
|
||||
WalletConnection.invalidated == False
|
||||
).order_by(WalletConnection.created.desc()).first()
|
||||
async def wallet_connection_async(self, db_session):
|
||||
from sqlalchemy import select, and_, desc
|
||||
result = await db_session.execute(
|
||||
select(WalletConnection)
|
||||
.where(and_(WalletConnection.user_id == self.id, WalletConnection.invalidated == False))
|
||||
.order_by(WalletConnection.created.desc())
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
def wallet_address(self, db_session):
|
||||
wallet_connection = self.wallet_connection(db_session)
|
||||
return wallet_connection.wallet_address if wallet_connection else None
|
||||
async def wallet_address_async(self, db_session):
|
||||
wc = await self.wallet_connection_async(db_session)
|
||||
return wc.wallet_address if wc else None
|
||||
|
||||
async def scan_owned_user_content(self, db_session):
|
||||
user_wallet_address = self.wallet_address(db_session)
|
||||
user_wallet_address = await self.wallet_address_async(db_session)
|
||||
|
||||
async def get_nft_items_list():
|
||||
try:
|
||||
@@ -40,9 +43,8 @@ class WalletMixin:
|
||||
item_address = Address(nft_item['address']).to_string(1, 1, 1)
|
||||
owner_address = Address(nft_item['owner']['address']).to_string(1, 1, 1)
|
||||
|
||||
user_content = db_session.query(UserContent).filter(
|
||||
UserContent.onchain_address == item_address
|
||||
).first()
|
||||
from sqlalchemy import select
|
||||
user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
|
||||
if user_content:
|
||||
continue
|
||||
|
||||
@@ -57,18 +59,18 @@ class WalletMixin:
|
||||
created=datetime.now(),
|
||||
meta={},
|
||||
user_id=self.id,
|
||||
wallet_connection_id=self.wallet_connection(db_session).id,
|
||||
wallet_connection_id=(await self.wallet_connection_async(db_session)).id,
|
||||
status="active"
|
||||
)
|
||||
db_session.add(user_content)
|
||||
db_session.commit()
|
||||
await db_session.commit()
|
||||
make_log(self, f"New onchain NFT found: {item_address}", level='info')
|
||||
|
||||
async def ____scan_owned_user_content(self, db_session):
|
||||
page_id = -1
|
||||
page_size = 100
|
||||
have_next_page = True
|
||||
user_wallet_address = self.wallet_address(db_session)
|
||||
user_wallet_address = await self.wallet_address_async(db_session)
|
||||
while have_next_page:
|
||||
page_id += 1
|
||||
nfts_list = await toncenter.get_nft_items(limit=100, offset=page_id * page_size, owner_address=user_wallet_address)
|
||||
@@ -81,9 +83,8 @@ class WalletMixin:
|
||||
item_address = Address(nft_item['address']).to_string(1, 1, 1)
|
||||
owner_address = Address(nft_item['owner_address']).to_string(1, 1, 1)
|
||||
|
||||
user_content = db_session.query(UserContent).filter(
|
||||
UserContent.onchain_address == item_address
|
||||
).first()
|
||||
from sqlalchemy import select
|
||||
user_content = (await db_session.execute(select(UserContent).where(UserContent.onchain_address == item_address))).scalars().first()
|
||||
if user_content:
|
||||
continue
|
||||
|
||||
@@ -105,11 +106,11 @@ class WalletMixin:
|
||||
'metadata_uri': nft_content,
|
||||
},
|
||||
user_id=self.id,
|
||||
wallet_connection_id=self.wallet_connection(db_session).id,
|
||||
wallet_connection_id=(await self.wallet_connection_async(db_session)).id,
|
||||
status="active"
|
||||
)
|
||||
db_session.add(user_content)
|
||||
db_session.commit()
|
||||
await db_session.commit()
|
||||
|
||||
make_log(self, f"New onchain NFT found: {item_address}", level='info')
|
||||
except BaseException as e:
|
||||
@@ -122,6 +123,6 @@ class WalletMixin:
|
||||
except BaseException as e:
|
||||
make_log(self, f"Error while scanning user content: {e}", level='error')
|
||||
|
||||
return self.db_session.query(UserContent).filter(
|
||||
UserContent.user_id == self.id
|
||||
).offset(offset).limit(limit).all()
|
||||
from sqlalchemy import select
|
||||
result = await db_session.execute(select(UserContent).where(UserContent.user_id == self.id).offset(offset).limit(limit))
|
||||
return result.scalars().all()
|
||||
+46
-34
@@ -1,45 +1,57 @@
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.sql import text
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
|
||||
|
||||
from app.core._config import MYSQL_URI, MYSQL_DATABASE
|
||||
from app.core._config import DATABASE_URL
|
||||
from app.core.logger import make_log
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
engine = create_engine(MYSQL_URI, poolclass=NullPool) #, echo=True)
|
||||
Session = sessionmaker(bind=engine)
|
||||
|
||||
|
||||
database_initialized = False
|
||||
while not database_initialized:
|
||||
def _to_async_dsn(url: str) -> str:
|
||||
# Convert psycopg2 DSN to asyncpg DSN
|
||||
# postgresql+psycopg2://user:pass@host:5432/db -> postgresql+asyncpg://user:pass@host:5432/db
|
||||
return url.replace("+psycopg2", "+asyncpg")
|
||||
|
||||
|
||||
# Async engine for PostgreSQL
|
||||
engine = create_async_engine(
|
||||
_to_async_dsn(DATABASE_URL),
|
||||
pool_size=10,
|
||||
max_overflow=20,
|
||||
pool_timeout=30,
|
||||
pool_recycle=1800,
|
||||
pool_pre_ping=True,
|
||||
)
|
||||
|
||||
AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
|
||||
|
||||
|
||||
async def wait_db_ready():
|
||||
ready = False
|
||||
while not ready:
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
await conn.execute(text("SELECT 1"))
|
||||
ready = True
|
||||
except Exception as e:
|
||||
make_log("SQL", 'PostgreSQL is not ready yet: ' + str(e), level='debug')
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def db_session(auto_commit: bool = False):
|
||||
session: AsyncSession = AsyncSessionLocal()
|
||||
try:
|
||||
with Session() as session:
|
||||
databases_list = session.execute(text("SHOW DATABASES;"))
|
||||
databases_list = [row[0] for row in databases_list]
|
||||
make_log("SQL", 'Database list: ' + str(databases_list), level='debug')
|
||||
assert MYSQL_DATABASE in databases_list, 'Database not found'
|
||||
database_initialized = True
|
||||
except Exception as e:
|
||||
make_log("SQL", 'MariaDB is not ready yet: ' + str(e), level='debug')
|
||||
time.sleep(1)
|
||||
|
||||
engine = create_engine(f"{MYSQL_URI}/{MYSQL_DATABASE}", poolclass=NullPool)
|
||||
Session = sessionmaker(bind=engine)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def db_session(auto_commit=False):
|
||||
_session = Session()
|
||||
try:
|
||||
yield _session
|
||||
if auto_commit is True:
|
||||
_session.commit()
|
||||
yield session
|
||||
if auto_commit:
|
||||
await session.commit()
|
||||
except BaseException as e:
|
||||
_session.rollback()
|
||||
await session.rollback()
|
||||
raise e
|
||||
finally:
|
||||
_session.close()
|
||||
await session.close()
|
||||
|
||||
|
||||
def new_session() -> AsyncSession:
|
||||
return AsyncSessionLocal()
|
||||
+15
-18
@@ -1,19 +1,19 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import select, and_, func
|
||||
from app.core.logger import make_log
|
||||
from app.core.models import Memory, User, UserBalance, Asset, InternalTransaction
|
||||
from app.core.storage import db_session
|
||||
|
||||
|
||||
def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
|
||||
async def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
|
||||
assert user, "No user"
|
||||
assert asset, "No asset"
|
||||
result = session.query(UserBalance).filter(
|
||||
UserBalance.user_id == user.id,
|
||||
UserBalance.asset_id == asset.id
|
||||
)
|
||||
results_count = result.count()
|
||||
if results_count == 0:
|
||||
result = await session.execute(select(UserBalance).where(
|
||||
and_(UserBalance.user_id == user.id, UserBalance.asset_id == asset.id)
|
||||
))
|
||||
row = result.scalars().first()
|
||||
if not row:
|
||||
user_balance = UserBalance(
|
||||
user_id=user.id,
|
||||
asset_id=asset.id,
|
||||
@@ -21,12 +21,9 @@ def get_user_balance(session, user: User, asset: Asset) -> UserBalance:
|
||||
created=datetime.now(),
|
||||
)
|
||||
session.add(user_balance)
|
||||
session.commit()
|
||||
return get_user_balance(session, user, asset)
|
||||
elif results_count == 1:
|
||||
return result.first()
|
||||
else:
|
||||
raise Exception(f"Multiple user balances found: {results_count}")
|
||||
await session.commit()
|
||||
return await get_user_balance(session, user, asset)
|
||||
return row
|
||||
|
||||
|
||||
async def make_internal_transaction(
|
||||
@@ -46,13 +43,13 @@ async def make_internal_transaction(
|
||||
raise Exception(f"Invalid amount: {amount}")
|
||||
|
||||
abs_amount = abs(amount)
|
||||
with db_session(auto_commit=False) as session:
|
||||
async with db_session(auto_commit=False) as session:
|
||||
async with memory.transaction():
|
||||
user = session.query(User).filter_by(id=user_id).first()
|
||||
user = (await session.execute(select(User).where(User.id == user_id))).scalars().first()
|
||||
assert user, "No user"
|
||||
asset = session.query(Asset).filter_by(id=asset_id).first()
|
||||
asset = (await session.execute(select(Asset).where(Asset.id == asset_id))).scalars().first()
|
||||
assert asset, "No asset"
|
||||
user_balance = get_user_balance(session, user, asset)
|
||||
user_balance = await get_user_balance(session, user, asset)
|
||||
assert user_balance, "No user balance"
|
||||
if is_spent is True:
|
||||
if abs_amount > user_balance.balance:
|
||||
@@ -71,6 +68,6 @@ async def make_internal_transaction(
|
||||
created=datetime.now(),
|
||||
)
|
||||
session.add(internal_transaction)
|
||||
session.commit()
|
||||
await session.commit()
|
||||
|
||||
make_log(user, f"Made internal transaction: {'-' if is_spent else ''}{abs_amount} {asset.symbol}, type: {type}")
|
||||
Reference in new issue
Block a user