my-dev
/
uploader-bot
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

big update

This commit is contained in:
user 2024-04-05 02:35:15 +03:00
parent 7d05a3f2c0
commit f45d593a84
32 changed files with 726 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -7,3 +7,4 @@ playground
alembic.ini
.DS_Store
messages.pot
activeConfig

30
alembic/versions/5c3d7b5ae3fb_add_rates_to_asset.py
View File

@ -1,30 +0,0 @@
"""add rates to Asset
Revision ID: 5c3d7b5ae3fb
Revises:
Create Date: 2024-02-16 15:59:08.740548
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '5c3d7b5ae3fb'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('assets', sa.Column('rates', sa.JSON(), nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('assets', 'rates')
# ### end Alembic commands ###

30
alembic/versions/9749eb810999_add_new_field.py
View File

@ -1,30 +0,0 @@
"""add new field
Revision ID: 9749eb810999
Revises: 5c3d7b5ae3fb
Create Date: 2024-02-16 16:14:11.380132
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '9749eb810999'
down_revision: Union[str, None] = '5c3d7b5ae3fb'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('wallet_connections', sa.Column('without_pk', sa.Boolean(), nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('wallet_connections', 'without_pk')
# ### end Alembic commands ###

26
alembic/versions/a7c1357e8d15_create_service_config.py Normal file
View File

@ -0,0 +1,26 @@
"""create service config
Revision ID: a7c1357e8d15
Revises:
Create Date: 2024-03-30 02:08:49.367910
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = 'a7c1357e8d15'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass

40
app/__main__.py
View File

@ -1,18 +1,36 @@
import asyncio
import sys
import os
import time
import traceback
from asyncio import sleep
from datetime import datetime
startup_target = '__main__'
try:
startup_target = sys.argv[1]
except BaseException:
pass
from app.core._utils.create_maria_tables import create_maria_tables
from app.core.storage import engine
if startup_target == '__main__':
create_maria_tables(engine)
else:
time.sleep(7)
from app.api import app
from app.bot import dp as uploader_bot_dp
from app.client_bot import dp as client_bot_dp
from app.core._config import SANIC_PORT, MYSQL_URI, PROJECT_HOST
from app.core._utils.create_maria_tables import create_maria_tables
from app.core.logger import make_log
if int(os.getenv("SANIC_MAINTENANCE", '0')) == 1:
make_log("Global", "Application is in maintenance mode")
while True:
time.sleep(1)
from app.core.models import Memory
from app.core.storage import engine
async def queue_daemon(app):
@ -29,11 +47,11 @@ async def queue_daemon(app):
async def execute_queue(app):
await create_maria_tables(engine)
telegram_bot_username = (await app.ctx.memory._telegram_bot.get_me()).username
client_telegram_bot_username = (await app.ctx.memory._client_telegram_bot.get_me()).username
make_log(None, f"Application normally started. HTTP port: {SANIC_PORT}")
make_log(None, f"Telegram bot: https://t.me/{telegram_bot_username}")
make_log(None, f"Client Telegram bot: https://t.me/{client_telegram_bot_username}")
make_log(None, f"MariaDB host: {MYSQL_URI.split('@')[1].split('/')[0].replace('/', '')}")
make_log(None, f"API host: {PROJECT_HOST}")
while True:
@ -61,12 +79,6 @@ async def execute_queue(app):
if __name__ == '__main__':
startup_target = '__main__'
try:
startup_target = sys.argv[1]
except BaseException:
pass
main_memory = Memory()
if startup_target == '__main__':
app.ctx.memory = main_memory
@ -82,15 +94,21 @@ if __name__ == '__main__':
app.run(host='0.0.0.0', port=SANIC_PORT)
else:
time.sleep(3)
time.sleep(2)
startup_fn = None
if startup_target == 'indexer':
from app.core.background.indexer_service import main_fn as target_fn
time.sleep(1)
elif startup_target == 'uploader':
from app.core.background.uploader_service import main_fn as target_fn
time.sleep(3)
elif startup_target == 'ton_daemon':
from app.core.background.ton_service import main_fn as target_fn
time.sleep(5)
elif startup_target == 'license_index':
from app.core.background.license_service import main_fn as target_fn
time.sleep(7)
startup_fn = startup_fn or target_fn
assert startup_fn

4
app/api/middleware.py
View File

@ -66,8 +66,8 @@ async def try_authorization(request):
request.ctx.user = user
request.ctx.user_key = known_key
request.ctx.user_uploader_wrapper = Wrapped_CBotChat(request.app.ctx.memory._telegram_bot, chat_id=user.telegram_id, db_session=request.ctx.db_session)
request.ctx.user_client_wrapper = Wrapped_CBotChat(request.app.ctx.memory._client_telegram_bot, chat_id=user.telegram_id, db_session=request.ctx.db_session)
request.ctx.user_uploader_wrapper = Wrapped_CBotChat(request.app.ctx.memory._telegram_bot, chat_id=user.telegram_id, db_session=request.ctx.db_session, user=user)
request.ctx.user_client_wrapper = Wrapped_CBotChat(request.app.ctx.memory._client_telegram_bot, chat_id=user.telegram_id, db_session=request.ctx.db_session, user=user)
async def try_service_authorization(request):

74
app/bot/__init__.py
View File

@ -1,80 +1,12 @@
# @MY_UploaderRobot
from datetime import datetime
from aiogram import BaseMiddleware, Dispatcher
from aiogram import Dispatcher
from aiogram.fsm.storage.memory import MemoryStorage
from app.bot.middleware import UserDataMiddleware
from app.bot.routers.index import main_router
from app.core.logger import logger
from app.core.models._telegram import Wrapped_CBotChat
from app.core.models.user import User
from app.core.storage import db_session
dp = Dispatcher(storage=MemoryStorage())
class UserDataMiddleware(BaseMiddleware):
async def __call__(self, handler, event, data):
update_body = event.message or event.callback_query
if not update_body:
return
if update_body.from_user.is_bot is True:
return
user_id = update_body.from_user.id
assert user_id >= 1
# TODO: maybe make users cache
with db_session(auto_commit=False) as session:
try:
user = session.query(User).filter_by(telegram_id=user_id).first()
except BaseException as e:
logger.error(f"Error when middleware getting user: {e}")
user = None
if user is None:
logger.debug(f"User {user_id} not found. Creating new user")
user = User(
telegram_id=user_id,
username=update_body.from_user.username,
lang_code='en',
last_use=datetime.now(),
meta=dict(first_name=update_body.from_user.first_name or '',
last_name=update_body.from_user.last_name or '', username=update_body.from_user.username,
language_code=update_body.from_user.language_code,
is_premium=update_body.from_user.is_premium),
created=datetime.now()
)
session.add(user)
session.commit()
else:
if user.username != update_body.from_user.username:
user.username = update_body.from_user.username
updated_meta_fields = {}
if user.meta.get('first_name') != update_body.from_user.first_name:
updated_meta_fields['first_name'] = update_body.from_user.first_name
if user.meta.get('last_name') != update_body.from_user.last_name:
updated_meta_fields['last_name'] = update_body.from_user.last_name
user.meta = {
**user.meta,
**updated_meta_fields
}
user.last_use = datetime.now()
session.commit()
data['user'] = user
data['db_session'] = session
data['chat_wrap'] = Wrapped_CBotChat(data['bot'], chat_id=user_id, db_session=session)
data['memory'] = dp._s_memory
result = await handler(event, data)
return result
dp.update.outer_middleware(UserDataMiddleware())
dp.include_router(main_router)

97
app/bot/middleware.py Normal file
View File

@ -0,0 +1,97 @@
from app.core.logger import make_log, logger
from app.core.models._telegram import Wrapped_CBotChat
from app.core.models.user import User
from app.core.storage import db_session
from aiogram import BaseMiddleware, types
from app.core.models.messages import KnownTelegramMessage
from datetime import datetime
class UserDataMiddleware(BaseMiddleware):
async def __call__(self, handler, event, data):
update_body = event.message or event.callback_query
if not update_body:
return
if update_body.from_user.is_bot is True:
return
user_id = update_body.from_user.id
assert user_id >= 1
# TODO: maybe make users cache
with db_session(auto_commit=False) as session:
try:
user = session.query(User).filter_by(telegram_id=user_id).first()
except BaseException as e:
logger.error(f"Error when middleware getting user: {e}")
user = None
if user is None:
logger.debug(f"User {user_id} not found. Creating new user")
user = User(
telegram_id=user_id,
username=update_body.from_user.username,
lang_code='en',
last_use=datetime.now(),
meta=dict(first_name=update_body.from_user.first_name or '',
last_name=update_body.from_user.last_name or '', username=update_body.from_user.username,
language_code=update_body.from_user.language_code,
is_premium=update_body.from_user.is_premium),
created=datetime.now()
)
session.add(user)
session.commit()
else:
if user.username != update_body.from_user.username:
user.username = update_body.from_user.username
updated_meta_fields = {}
if user.meta.get('first_name') != update_body.from_user.first_name:
updated_meta_fields['first_name'] = update_body.from_user.first_name
if user.meta.get('last_name') != update_body.from_user.last_name:
updated_meta_fields['last_name'] = update_body.from_user.last_name
user.meta = {
**user.meta,
**updated_meta_fields
}
user.last_use = datetime.now()
session.commit()
data['user'] = user
data['db_session'] = session
data['chat_wrap'] = Wrapped_CBotChat(data['bot'], chat_id=user_id, db_session=session, user=user)
data['memory'] = data['dispatcher']._s_memory
if isinstance(update_body, types.Message):
message_type = 'common'
if update_body.text.startswith('/start'):
message_type = 'start_command'
if session.query(KnownTelegramMessage).filter_by(
chat_id=update_body.chat.id,
message_id=update_body.message_id,
from_user=True
).first():
make_log("UserDataMiddleware", f"Message {update_body.message_id} already processed", level='debug')
return
new_message = KnownTelegramMessage(
type=message_type,
bot_id=data['chat_wrap'].bot_id,
chat_id=update_body.chat.id,
message_id=update_body.message_id,
from_user=True,
from_telegram_id=user_id,
created=datetime.now(),
meta={}
)
session.add(new_message)
session.commit()
result = await handler(event, data)
return result

75
app/client_bot/__init__.py
View File

@ -1,80 +1,11 @@
# @MY_UploaderRobot
# @MY_Web3Bot
from datetime import datetime
from aiogram import BaseMiddleware, Dispatcher
from aiogram import Dispatcher
from aiogram.fsm.storage.memory import MemoryStorage
from app.bot.middleware import UserDataMiddleware
from app.client_bot.routers.index import main_router
from app.core.logger import logger
from app.core.models._telegram import Wrapped_CBotChat
from app.core.models.user import User
from app.core.storage import db_session
dp = Dispatcher(storage=MemoryStorage())
class UserDataMiddleware(BaseMiddleware):
async def __call__(self, handler, event, data):
update_body = event.message or event.callback_query
if not update_body:
return
if update_body.from_user.is_bot is True:
return
user_id = update_body.from_user.id
assert user_id >= 1
# TODO: maybe make users cache
with db_session(auto_commit=False) as session:
try:
user = session.query(User).filter_by(telegram_id=user_id).first()
except BaseException as e:
logger.error(f"Error when middleware getting user: {e}")
user = None
if user is None:
logger.debug(f"User {user_id} not found. Creating new user")
user = User(
telegram_id=user_id,
username=update_body.from_user.username,
lang_code='en',
last_use=datetime.now(),
meta=dict(first_name=update_body.from_user.first_name or '',
last_name=update_body.from_user.last_name or '', username=update_body.from_user.username,
language_code=update_body.from_user.language_code,
is_premium=update_body.from_user.is_premium),
created=datetime.now()
)
session.add(user)
session.commit()
else:
if user.username != update_body.from_user.username:
user.username = update_body.from_user.username
updated_meta_fields = {}
if user.meta.get('first_name') != update_body.from_user.first_name:
updated_meta_fields['first_name'] = update_body.from_user.first_name
if user.meta.get('last_name') != update_body.from_user.last_name:
updated_meta_fields['last_name'] = update_body.from_user.last_name
user.meta = {
**user.meta,
**updated_meta_fields
}
user.last_use = datetime.now()
session.commit()
data['user'] = user
data['db_session'] = session
data['chat_wrap'] = Wrapped_CBotChat(data['bot'], chat_id=user_id)
data['memory'] = dp._s_memory
result = await handler(event, data)
return result
dp.update.outer_middleware(UserDataMiddleware())
dp.include_router(main_router)

16
app/client_bot/routers/content.py Normal file
View File

@ -0,0 +1,16 @@
import os
import sys
import traceback
from aiogram import types, Router
from app.client_bot.routers.home import router as home_router
from app.client_bot.routers.tonconnect import router as tonconnect_router
from app.core.logger import logger
router = Router()
# router.message.register(t_, Command('dev_tonconnect'))
# router.callback_query.register(t_callback_init_tonconnect, F.data.startswith('initTonconnect_'))
# router.callback_query.register(t_callback_disconnect_wallet, F.data == 'disconnectWallet')

9
app/client_bot/routers/home.py
View File

@ -6,6 +6,7 @@ from app.core._blockchain.ton.connect import TonConnect
from app.core._keyboards import get_inline_keyboard
from app.core._utils.tg_process_template import tg_process_template
from app.core.models.wallet_connection import WalletConnection
from app.core.models.node_storage import StoredContent
main_router = Router()
@ -71,6 +72,14 @@ async def t_home_menu(__msg, **extra):
if not wallet_connection:
return await send_connect_wallets_list(db_session, chat_wrap, user, message_id=message_id)
args = []
if isinstance(__msg, types.Message):
args = __msg.text.split(' ')[1:]
if args[0].startswith('C'):
content = StoredContent.from_cid(db_session, args[0][1:])
return await chat_wrap.send_content(content, message_id=message_id)
return await send_home_menu(chat_wrap, user, wallet_connection, message_id=message_id)

5
app/client_bot/routers/index.py
View File

@ -5,10 +5,15 @@ import traceback
from aiogram import types, Router
from app.client_bot.routers.home import router as home_router
from app.client_bot.routers.tonconnect import router as tonconnect_router
from app.client_bot.routers.content import router as content_router
from app.core.logger import logger
main_router = Router()
main_router.include_routers(home_router)
main_router.include_routers(tonconnect_router)
main_router.include_routers(content_router)
closing_router = Router()

134
app/client_bot/routers/tonconnect.py Normal file
View File

@ -0,0 +1,134 @@
import asyncio
import json
from datetime import datetime, timedelta
from aiogram import types, Router, F
from aiogram.filters import Command
from app.client_bot.routers.home import send_connect_wallets_list, send_home_menu
from app.core._blockchain.ton.connect import TonConnect, unpack_wallet_info
from app.core._keyboards import get_inline_keyboard
from app.core._utils.tg_process_template import tg_process_template
from app.core.logger import make_log
from app.core.models.wallet_connection import WalletConnection
router = Router()
async def pause_ton_connection(ton_connect: TonConnect):
if ton_connect.connected:
ton_connect._sdk_client.pause_connection()
async def t_tonconnect_dev_menu(message: types.Message, memory=None, user=None, db_session=None, chat_wrap=None,
**extra):
try:
command_args = message.text.split(" ")[1:]
except BaseException as e:
command_args = []
make_log("TonConnect_DevMenu", f"Command args: {command_args}", level='info')
wallet_app_name = 'tonkeeper'
if len(command_args) > 0:
wallet_app_name = command_args[0].lower()
keyboard = []
ton_connect, ton_connection = TonConnect.by_user(db_session, user, callback_fn=())
make_log("TonConnect_DevMenu", f"Available wallets: {ton_connect._sdk_client.get_wallets()}", level='debug')
await ton_connect.restore_connection()
make_log("TonConnect_DevMenu", f"SDK connected?: {ton_connect.connected}", level='info')
if not ton_connect.connected:
if ton_connection:
make_log("TonConnect_DevMenu", f"Invalidating old connection", level='debug')
ton_connection.invalidated = True
db_session.commit()
message_text = f"""<b>Wallet is not connected</b>
Use /dev_tonconnect <code>{wallet_app_name}</code> for connect to wallet."""
connection_link = await ton_connect.new_connection(wallet_app_name)
ton_connect.connected
make_log("TonConnect_DevMenu", f"New connection link for {wallet_app_name}: {connection_link}", level='debug')
keyboard.append([
{
'text': 'Connect',
'url': connection_link
}
])
else:
wallet_info_text = json.dumps(unpack_wallet_info(ton_connect._sdk_client._wallet), indent=4, ensure_ascii=False)
message_text = f"""<b>Wallet is connected</b>
<pre>{wallet_info_text}</pre>"""
memory.add_task(pause_ton_connection, ton_connect, delay_s=60 * 3)
return await tg_process_template(
chat_wrap, message_text,
keyboard=get_inline_keyboard(keyboard) if keyboard else None
)
async def t_callback_init_tonconnect(query: types.CallbackQuery, memory=None, user=None, db_session=None,
chat_wrap=None, **extra):
wallet_app_name = query.data.split("_")[1]
ton_connect, ton_connection = TonConnect.by_user(db_session, user)
await ton_connect.restore_connection()
connection_link = await ton_connect.new_connection(wallet_app_name)
ton_connect.connected
memory.add_task(pause_ton_connection, ton_connect, delay_s=60 * 3)
make_log("TonConnect_Init", f"New connection link for {wallet_app_name}: {connection_link}", level='debug')
message_text = user.translated("tonconnectInit_menu")
r = await tg_process_template(
chat_wrap, message_text,
keyboard=get_inline_keyboard([
[
{
'text': user.translated('tonconnectOpenWallet_button'),
'url': connection_link
}
],
[
{
'text': user.translated('home_button'),
'callback_data': 'home'
}
]
]), message_id=query.message.message_id
)
start_ts = datetime.now()
while datetime.now() - start_ts < timedelta(seconds=180):
new_connection = db_session.query(WalletConnection).filter(
WalletConnection.user_id == user.id,
WalletConnection.invalidated == False
).first()
if new_connection:
await tg_process_template(
chat_wrap, user.translated('p_successConnectWallet')
)
await send_home_menu(chat_wrap, user, new_connection)
break
await asyncio.sleep(1)
return r
async def t_callback_disconnect_wallet(query: types.CallbackQuery, memory=None, user=None, db_session=None,
chat_wrap=None, **extra):
wallet_connections = db_session.query(WalletConnection).filter(
WalletConnection.user_id == user.id,
WalletConnection.invalidated == False
).all()
for wallet_connection in wallet_connections:
wallet_connection.invalidated = True
db_session.commit()
return await send_connect_wallets_list(db_session, chat_wrap, user, message_id=query.message.message_id)
router.message.register(t_tonconnect_dev_menu, Command('dev_tonconnect'))
router.callback_query.register(t_callback_init_tonconnect, F.data.startswith('initTonconnect_'))
router.callback_query.register(t_callback_disconnect_wallet, F.data == 'disconnectWallet')

14
app/core/_blockchain/ton/toncenter.py
View File

@ -9,12 +9,13 @@ from app.core.logger import make_log
class TonCenter:
def __init__(self, host: str, api_key: str = None, testnet: bool = False):
def __init__(self, host: str, api_key: str = None, v3_host: str = None, testnet: bool = False):
self.host = host
self.api_key = api_key
self.v3_host = v3_host
self.last_used = time.time()
async def request(self, method: str, endpoint: str, *args, tries_count=0, **kwargs) -> dict:
async def request(self, method: str, endpoint: str, *args, v3: bool=False, tries_count=0, **kwargs) -> dict:
if tries_count > 3:
raise Exception(f'Error while toncenter request {endpoint}: {tries_count}')
@ -30,7 +31,7 @@ class TonCenter:
await asyncio.sleep(0.1)
self.last_used = time.time()
response = await client.request(method, f"{self.host}{endpoint}", *args, **kwargs)
response = await client.request(method, f"{self.host_v3 if v3 is True else self.host}{endpoint}", *args, **kwargs)
try:
if response.status_code != 200:
raise Exception(f'Error while toncenter request {endpoint}: {response.text}')
@ -80,5 +81,12 @@ class TonCenter:
}
)).get('result', {})
async def get_nft_items(self, limit: int = 100, offset: int = 0, **search_options):
return (await self.request(
"GET", 'nft/items',
params={**search_options, 'limit': limit, 'offset': offset},
v3=True
)).get('nft_items', [])
toncenter = TonCenter(TONCENTER_HOST, TONCENTER_API_KEY)

2
app/core/_config.py
View File

@ -42,7 +42,7 @@ ALLOWED_CONTENT_TYPES = [
TESTNET = bool(int(os.getenv('TESTNET', '0')))
TONCENTER_HOST = os.getenv('TONCENTER_HOST', 'https://toncenter.com/api/v1/')
TONCENTER_HOST = os.getenv('TONCENTER_HOST', 'https://toncenter.com/api/v2/')
TONCENTER_API_KEY = os.getenv('TONCENTER_API_KEY')
MY_PLATFORM_CONTRACT = 'EQAGbwW0sFghy9N4MQ0Ozp8YOIr0lcMI8J5kbbydFnQtheMY'

23
app/core/_secrets.py
View File

@ -4,21 +4,24 @@ from nacl.bindings import crypto_sign_seed_keypair
from tonsdk.utils import Address
from app.core._blockchain.ton.wallet_v3cr3 import WalletV3CR3
from app.core.active_config import active_config
from app.core.models._config import ServiceConfig
from app.core.storage import db_session
from app.core.logger import make_log
def load_hot_pair():
hot_seed = active_config.get('private_key')
if hot_seed is None:
make_log("HotWallet", "No seed found, generating new one", level='info')
hot_seed = urandom(32)
active_config.set('private_key', hot_seed.hex())
return load_hot_pair()
with db_session() as session:
service_config = ServiceConfig(session)
hot_seed = service_config.get('private_key')
if hot_seed is None:
make_log("HotWallet", "No seed found, generating new one", level='info')
hot_seed = urandom(32)
service_config.set('private_key', hot_seed.hex())
return load_hot_pair()
hot_seed = bytes.fromhex(hot_seed)
public_key, private_key = crypto_sign_seed_keypair(hot_seed)
return hot_seed, public_key, private_key
hot_seed = bytes.fromhex(hot_seed)
public_key, private_key = crypto_sign_seed_keypair(hot_seed)
return hot_seed, public_key, private_key
_extra_ton_wallet_options = {}

2
app/core/_utils/create_maria_tables.py
View File

@ -2,7 +2,7 @@ from app.core.models import Asset
from app.core.models.base import AlchemyBase
async def create_maria_tables(engine):
def create_maria_tables(engine):
"""Create all tables in the database."""
Asset()
AlchemyBase.metadata.create_all(engine)

4
app/core/_utils/tg_process_template.py
View File

@ -1,9 +1,9 @@
async def tg_process_template(
chat_wrap: 'Wrapped_CBot',
text, keyboard=None, message_id=None,
photo=None, video=None, document=None, **kwargs
photo=None, audio=None, video=None, document=None, **kwargs
):
if (photo or video or document) and message_id:
if (photo or video or audio or document) and message_id:
await chat_wrap.delete_message(message_id)
message_id = None

10
app/core/active_config.py
View File

@ -1,10 +0,0 @@
import os
from app.core._config import CONFIG_FILE
from app.core.models._config import ConfigFile
if not os.path.exists(CONFIG_FILE):
with open(CONFIG_FILE, 'w') as f:
f.write('{}')
active_config = ConfigFile(CONFIG_FILE)

50
app/core/background/license_service.py Normal file
View File

@ -0,0 +1,50 @@
import asyncio
from base64 import b64decode
from datetime import datetime
from base58 import b58encode
from tonsdk.boc import Cell
from tonsdk.utils import Address
from app.core._blockchain.ton.platform import platform
from app.core._blockchain.ton.toncenter import toncenter
from app.core._utils.send_status import send_status
from app.core.logger import make_log
from app.core.models.node_storage import StoredContent
from app.core._utils.resolve_content import resolve_content
from app.core.models.wallet_connection import WalletConnection
from app.core._keyboards import get_inline_keyboard
from app.core.models._telegram import Wrapped_CBotChat
from app.core.storage import db_session
import os
import traceback
async def license_index_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
make_log("LicenseIndex", "Service running", level="debug")
with db_session() as session:
pass
return platform_found, seqno
async def main_fn(memory, ):
make_log("LicenseIndex", "Service started", level="info")
platform_found = False
seqno = 0
while True:
try:
platform_found, seqno = await license_index_loop(memory, platform_found, seqno)
except BaseException as e:
make_log("LicenseIndex", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
if platform_found:
await send_status("LicenseIndex", f"working (seqno={seqno})")
await asyncio.sleep(5)
seqno += 1
# if __name__ == '__main__':
# loop = asyncio.get_event_loop()
# loop.run_until_complete(main())
# loop.close()

6
app/core/background/ton_service.py
View File

@ -38,7 +38,7 @@ async def main_fn(memory):
)['message'].to_boc(False)
)
await asyncio.sleep(5)
return await main_fn()
return await main_fn(memory)
if os.getenv("TON_BEGIN_COMMAND_WITHDRAW"):
await toncenter.send_boc(
@ -53,7 +53,7 @@ async def main_fn(memory):
)
make_log("TON", "Withdraw command sent", level="info")
await asyncio.sleep(10)
return await main_fn()
return await main_fn(memory)
platform_state = await toncenter.get_account(platform.address.to_string(1, 1, 1))
if not platform_state.get('code'):
@ -72,7 +72,7 @@ async def main_fn(memory):
await send_status("ton_daemon", "working: deploying platform")
await asyncio.sleep(15)
return await main_fn()
return await main_fn(memory)
while True:
try:

3
app/core/models/__init__.py
View File

@ -1,4 +1,3 @@
from app.core.models.asset import Asset
from app.core.models.base import AlchemyBase
from app.core.models.keys import KnownKey
from app.core.models.memory import Memory
@ -9,3 +8,5 @@ from app.core.models.wallet_connection import WalletConnection
from app.core.models.messages import KnownTelegramMessage
from app.core.models.user_activity import UserActivity
from app.core.models.content.user_content import UserContent
from app.core.models._config import ServiceConfigValue
from app.core.models.asset import Asset

78
app/core/models/_config.py
View File

@ -1,64 +1,38 @@
import string
from json import dumps as json_dumps
from json import loads as json_loads
from random import choice
from subprocess import Popen, PIPE
from app.core.logger import make_log
from app.core.models.base import AlchemyBase
from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
class ConfigFile:
def __init__(self, filepath: str):
self.filepath = filepath
self.values
class ServiceConfigValue(AlchemyBase):
__tablename__ = 'service_config'
id = Column(Integer, autoincrement=True, primary_key=True)
key = Column(String(128), nullable=False, unique=True)
packed_value = Column(JSON, nullable=False, default={})
@property
def values(self):
with open(self.filepath, 'r') as file:
return json_loads(file.read())
def value(self):
return self.packed_value['value']
assert isinstance(self.values, dict)
class ServiceConfig:
def __init__(self, session):
self.session = session
def get(self, key, default=None):
return self.values.get(key, default)
result = self.session.query(ServiceConfigValue).filter(ServiceConfigValue.key == key).first()
return (result.value if result else None) or default
def set(self, key, value):
random_part = choice(string.ascii_lowercase)
app_cached_values = self.values
app_cached_values[key] = value
with open(f"{'/'.join(self.filepath.split('/')[:-1])}/.backup_{self.filepath.split('/')[-1]}_{random_part}", 'w') as file:
file.write(
json_dumps(
app_cached_values,
indent=4,
sort_keys=True
)
)
with open(self.filepath, 'w') as file:
file.write(
json_dumps(
app_cached_values,
indent=4,
sort_keys=True
)
)
make_log("ConfigFile", f"Edited {key}", level="debug")
p1 = Popen(["md5sum", self.filepath], stdout=PIPE, stderr=PIPE)
p1.wait()
out1, err1 = p1.communicate()
p2 = Popen(["md5sum", f"{'/'.join(self.filepath.split('/')[:-1])}/.backup_{self.filepath.split('/')[-1]}_{random_part}"], stdout=PIPE, stderr=PIPE)
p2.wait()
out2, err2 = p2.communicate()
if err1 or err2:
make_log("ConfigFile", f"Error when editing {key} (check md5sum): {err1} {err2}", level="error")
return self.set(key, value)
fingerprint1 = out1.split()[0].strip()
fingerprint2 = out2.split()[0].strip()
if fingerprint1 != fingerprint2:
make_log("ConfigFile", f"Error when editing {key} (check md5sum): {fingerprint1} {fingerprint2}", level="error")
config_value = self.session.query(ServiceConfigValue).filter(
ServiceConfigValue.key == key
).first()
if not config_value:
config_value = ServiceConfigValue(key=key)
self.session.add(config_value)
self.session.commit()
return self.set(key, value)
config_value.packed_value = {'value': value}
self.session.commit()
return

70
app/core/models/_telegram/templates/player.py Normal file
View File

@ -0,0 +1,70 @@
from app.core.models.node_storage import StoredContent
from app.core.models.content.user_content import UserContent
from app.core.logger import make_log
from app.core._utils.tg_process_template import tg_process_template
from app.core._config import PROJECT_HOST
from app.core._keyboards import get_inline_keyboard
class PlayerTemplates:
async def send_content(self, content: StoredContent, extra_buttons=None, message_id=None):
assert content.type.startswith('onchain/content'), "Invalid nodeStorage content type"
inline_keyboard_array = []
cd_log = f"Content (SHA256: {content.hash}), Encrypted: {content.encrypted}, TelegramCID: {content.telegram_cid}. "
if not content.encrypted:
local_content = content
else:
local_content = content.decrypted_content
# TODO: add check decrypted_content by .format_json()['content_cid']
if local_content:
cd_log += f"Decrypted: {local_content.hash}. "
else:
cd_log += "Can't decrypt content. "
if local_content:
content_meta = content.json_format()
local_content_meta = local_content.json_format()
try:
content_type, content_encoding = local_content_meta["content_type"].split('/')
except:
content_type, content_encoding = 'application', 'x-binary'
try:
cover_content = StoredContent.from_cid(self.db_session, content_meta.get('cover_cid') or None)
except BaseException as e:
cd_log += f"Can't get cover content: {e}. "
cover_content = None
local_content_cid = local_content.cid
local_content_cid.content_type = 'audio/mpeg'
local_content_url = f"{PROJECT_HOST}/api/v1/storage/{local_content_cid.serialize_v2(include_accept_type=True)}"
template_kwargs = {}
if content_type[0] == 'audio':
template_kwargs['title'] = 'title'
template_kwargs['performer'] = 'performer'
template_kwargs['protect_content'] = True
template_kwargs['audio'] = local_content_url
if cover_content:
template_kwargs['thumbnail'] = cover_content.web_url
else:
local_content = None
if not local_content:
text = self.user.translated('p_playerContext_unsupportedContent').format(
content_type=content_type,
content_encoding=content_encoding
)
inline_keyboard_array = []
extra_buttons = []
make_log("TG-Player", f"Send content {content_type} ({content_encoding}) to chat {self._chat_id}. {cd_log}")
return await tg_process_template(
self, text, message_id=message_id, **template_kwargs,
keyboard=get_inline_keyboard([*inline_keyboard_array, *extra_buttons]) if inline_keyboard_array else None,
message_type=f'content/{content_type}', message_meta={'content_sha256': content_meta['hash']}
)

64
app/core/models/_telegram/wrapped_bot.py
View File

@ -3,10 +3,16 @@ from datetime import datetime, timedelta
from app.core.logger import make_log
from app.core.models.messages import KnownTelegramMessage
from app.core._config import TELEGRAM_API_KEY, CLIENT_TELEGRAM_API_KEY
from app.core.models._telegram.templates.player import PlayerTemplates
class Wrapped_CBotChat:
def __init__(self, api_key: str, chat_id: int = None, db_session=None, **kwargs):
class T: pass
class Wrapped_CBotChat(T, PlayerTemplates):
def __init__(self, api_key: str, chat_id: int = None, db_session=None, user=None, **kwargs):
if isinstance(api_key, Bot):
self._bot_key = api_key.token
self._bot = api_key
@ -18,6 +24,7 @@ class Wrapped_CBotChat:
self._chat_id = chat_id
self.db_session = db_session
self.user = user
self.options = kwargs
def __repr__(self):
@ -26,13 +33,21 @@ class Wrapped_CBotChat:
return "Bot"
async def return_result(self, result, message_type='common'):
@property
def bot_id(self):
return {
TELEGRAM_API_KEY: 0,
CLIENT_TELEGRAM_API_KEY: 1
}[self._bot_key]
async def return_result(self, result, message_type='common', message_meta={}):
if self.db_session:
if message_type == 'common':
ci = 0
for oc_msg in self.db_session.query(KnownTelegramMessage).filter(
KnownTelegramMessage.type == 'common',
KnownTelegramMessage.chat_id == self._chat_id,
KnownTelegramMessage.deleted == False
).all():
await self.delete_message(oc_msg.message_id)
ci += 1
@ -45,9 +60,12 @@ class Wrapped_CBotChat:
self.db_session.add(
KnownTelegramMessage(
type=message_type,
bot_id=self.bot_id,
chat_id=self._chat_id,
message_id=message_id,
created=datetime.now()
from_user=False,
created=datetime.now(),
meta=message_meta or {}
)
)
self.db_session.commit()
@ -56,7 +74,7 @@ class Wrapped_CBotChat:
return result
async def send_message(self, text: str, message_type='common', **kwargs):
async def send_message(self, text: str, message_type='common', message_meta={}, **kwargs):
assert self._chat_id, "No chat_id"
try:
make_log(self, f"Send message to {self._chat_id}. Text len: {len(text)}", level='debug')
@ -67,7 +85,7 @@ class Wrapped_CBotChat:
disable_web_page_preview=True,
**kwargs
)
return await self.return_result(r, message_type=message_type)
return await self.return_result(r, message_type=message_type, message_meta=message_meta)
except BaseException as e:
make_log(self, f"Error sending message to {self._chat_id}. Error: {e}", level='warning')
return None
@ -104,7 +122,7 @@ class Wrapped_CBotChat:
make_log(self, f"Error deleting message {self._chat_id}/{message_id}. Error: {e}", level='warning')
return None
async def send_photo(self, file_id, message_type='common', **kwargs):
async def send_photo(self, file_id, message_type='common', message_meta={}, **kwargs):
assert self._chat_id, "No chat_id"
try:
make_log(self, f"Send photo to {self._chat_id}. File: {file_id}", level='debug')
@ -113,12 +131,12 @@ class Wrapped_CBotChat:
file_id,
**kwargs
)
return await self.return_result(r, message_type=message_type)
return await self.return_result(r, message_type=message_type, message_meta=message_meta)
except Exception as e:
make_log(self, f"Error sending photo to {self._chat_id}. Error: {e}", level='warning')
return None
async def send_document(self, file_id, message_type='common', **kwargs):
async def send_document(self, file_id, message_type='common', message_meta={}, **kwargs):
assert self._chat_id, "No chat_id"
try:
make_log(self, f"Send document to {self._chat_id}. File: {file_id}", level='debug')
@ -127,12 +145,12 @@ class Wrapped_CBotChat:
file_id,
**kwargs
)
return await self.return_result(r, message_type=message_type)
return await self.return_result(r, message_type=message_type, message_meta=message_meta)
except Exception as e:
make_log(self, f"Error sending document to {self._chat_id}. Error: {e}", level='warning')
return None
async def send_video(self, file_id, message_type='common', **kwargs):
async def send_video(self, file_id, message_type='common', message_meta={}, **kwargs):
assert self._chat_id, "No chat_id"
try:
make_log(self, f"Send video to {self._chat_id}. File: {file_id}", level='debug')
@ -141,12 +159,26 @@ class Wrapped_CBotChat:
file_id,
**kwargs
)
return await self.return_result(r, message_type=message_type)
return await self.return_result(r, message_type=message_type, message_meta=message_meta)
except Exception as e:
make_log(self, f"Error sending video to {self._chat_id}. Error: {e}", level='warning')
return None
async def copy_message(self, from_chat_id, message_id, message_type='common', **kwargs):
async def send_audio(self, file_id, message_type='common', message_meta={}, **kwargs):
assert self._chat_id, "No chat_id"
try:
make_log(self, f"Send audio to {self._chat_id}. File: {file_id}", level='debug')
r = await self._bot.send_audio(
self._chat_id,
file_id,
**kwargs
)
return await self.return_result(r, message_type=message_type, message_meta=message_meta)
except Exception as e:
make_log(self, f"Error sending audio to {self._chat_id}. Error: {e}", level='warning')
return None
async def copy_message(self, from_chat_id, message_id, message_type='common', message_meta={}, **kwargs):
assert self._chat_id, "No chat_id"
try:
make_log(self, f"Copy message from {from_chat_id}/{message_id} to {self._chat_id}", level='debug')
@ -156,12 +188,12 @@ class Wrapped_CBotChat:
message_id,
**kwargs
)
return await self.return_result(r, message_type=message_type)
return await self.return_result(r, message_type=message_type, message_meta=message_meta)
except Exception as e:
make_log(self, f"Error copying message from {from_chat_id}/{message_id} to {self._chat_id}. Error: {e}", level='warning')
return None
async def forward_message(self, from_chat_id, message_id, message_type='common', **kwargs):
async def forward_message(self, from_chat_id, message_id, message_type='common', message_meta={}, **kwargs):
assert self._chat_id, "No chat_id"
try:
make_log(self, f"Forward message from {from_chat_id}/{message_id} to {self._chat_id}", level='debug')
@ -171,7 +203,7 @@ class Wrapped_CBotChat:
message_id,
**kwargs
)
return await self.return_result(r, message_type=message_type)
return await self.return_result(r, message_type=message_type, message_meta=message_meta)
except Exception as e:
make_log(self, f"Error forwarding message from {from_chat_id}/{message_id} to {self._chat_id}. Error: {e}", level='warning')
return None

44
app/core/models/content/indexation_mixins.py Normal file
View File

@ -0,0 +1,44 @@
from app.core.models.wallet_connection import WalletConnection
from app.core._blockchain.ton.toncenter import toncenter
from tonsdk.boc import Cell
from base64 import b64decode
def unpack_item_indexator_data(item_get_data_result):
result = {}
assert item_get_data_result['stack'][0][0] == 'num', "Type is not a number"
result['type'] = int(item_get_data_result['stack'][0][1], 16)
result['address'] = Cell.one_from_boc(
b64decode(item_get_data_result['stack'][1][1]['bytes'])).begin_parse().read_msg_addr().to_string(1, 1, 1)
assert item_get_data_result['stack'][2][0] == 'num', "Index is not a number"
result['index'] = int(item_get_data_result['stack'][2][1], 16)
result['platform_address'] = Cell.one_from_boc(
b64decode(item_get_data_result['stack'][3][1]['bytes'])).begin_parse().read_msg_addr().to_string(1, 1, 1)
assert item_get_data_result['stack'][4][0] == 'num', "License type is not a number"
result['license_type'] = int(item_get_data_result['stack'][4][1], 16)
result['owner_address'] = Cell.one_from_boc(
b64decode(item_get_data_result['stack'][5][1]["bytes"])).begin_parse().read_msg_addr().to_string(1, 1, 1)
result['values'] = Cell.one_from_boc(b64decode(item_get_data_result['stack'][6][1]['bytes']))
result['derivates'] = Cell.one_from_boc(b64decode(item_get_data_result['stack'][7][1]['bytes']))
result['platform_variables'] = Cell.one_from_boc(b64decode(item_get_data_result['stack'][8][1]['bytes']))
result['distribution'] = Cell.one_from_boc(b64decode(item_get_data_result['stack'][9][1]['bytes']))
return result
class NodeStorageIndexationMixin:
pass # async def fetch_onchain_metadata(self):
class UserContentIndexationMixin:
pass

8
app/core/models/content/user_content.py
View File

@ -2,14 +2,18 @@
from sqlalchemy import Column, BigInteger, Integer, String, ForeignKey, DateTime, JSON, Boolean
from sqlalchemy.orm import relationship
from app.core.models.base import AlchemyBase
from app.core.models.content.indexation_mixins import UserContentIndexationMixin
class UserContent(AlchemyBase):
class UserContent(AlchemyBase, UserContentIndexationMixin):
__tablename__ = 'users_content'
id = Column(Integer, autoincrement=True, primary_key=True)
type = Column(String(128), nullable=False) # 'license_issuer', 'license_listen'
type = Column(String(128), nullable=False) # 'license/issuer', 'license/listen', 'nft/unknown'
onchain_address = Column(String(1024), nullable=True) # bind by this
owner_address = Column(String(1024), nullable=True)
code_hash = Column(String(128), nullable=True)
data_hash = Column(String(128), nullable=True)
updated = Column(DateTime, nullable=False, default=0)
content_id = Column(Integer, ForeignKey('node_storage.id'), nullable=True)

7
app/core/models/messages.py
View File

@ -1,5 +1,5 @@
from base58 import b58decode
from sqlalchemy import Column, Integer, String, DateTime, JSON, BigInteger
from sqlalchemy import Column, Integer, String, DateTime, JSON, BigInteger, Boolean
from .base import AlchemyBase
@ -12,6 +12,11 @@ class KnownTelegramMessage(AlchemyBase):
id = Column(Integer, autoincrement=True, primary_key=True)
type = Column(String(64), nullable=True)
bot_id = Column(Integer, nullable=False, default=1) # 0 – uploader, 1 – client
chat_id = Column(BigInteger, nullable=False)
message_id = Column(BigInteger, nullable=False)
from_user = Column(Boolean, nullable=False)
from_telegram_id = Column(BigInteger, nullable=True)
created = Column(DateTime, nullable=False, default=0)
deleted = Column(Boolean, nullable=True, default=False)
meta = Column(JSON, nullable=False, default={})

26
app/core/models/node_storage.py
View File

@ -4,11 +4,14 @@ from sqlalchemy.orm import relationship
from datetime import datetime
from app.core.logger import make_log
from app.core._config import UPLOADS_DIR, PROJECT_HOST
import os
from app.core.content.content_id import ContentId
from app.core.models.content.indexation_mixins import NodeStorageIndexationMixin
from .base import AlchemyBase
class StoredContent(AlchemyBase):
class StoredContent(AlchemyBase, NodeStorageIndexationMixin):
__tablename__ = 'node_storage'
id = Column(Integer, autoincrement=True, primary_key=True)
@ -50,6 +53,14 @@ class StoredContent(AlchemyBase):
accept_type=self.meta.get('content_type', 'image/jpeg')
)
@property
def filepath(self) -> str:
return os.path.join(UPLOADS_DIR, file_hash)
@property
def web_url(self) -> str:
return f"{PROJECT_HOST}/api/v1/storage/{self.cid.serialize_v2()}"
@property
def decrypt_possible(self) -> bool:
if self.encrypted is False:
@ -80,7 +91,20 @@ class StoredContent(AlchemyBase):
**extra_fields,
"hash": self.hash,
"cid": self.cid.serialize_v2(),
"content_type": self.meta.get('content_type', 'application/x-binary'),
"status": self.status,
"updated": self.updated.isoformat() if isinstance(self.updated, datetime) else (make_log("Content.json_format", f"Invalid Content.updated: {self.updated} ({type(self.updated)})", level="error") or None),
"created": self.created.isoformat() if isinstance(self.created, datetime) else (make_log("Content.json_format", f"Invalid Content.created: {self.created} ({type(self.created)})", level="error") or None),
}
@classmethod
def from_cid(cls, db_session, content_id):
if isinstance(content_id, str):
cid = ContentId.deserialize(content_id)
else:
cid = content_id
content = db_session.query(StoredContent).filter(StoredContent.hash == cid.content_hash_b58).first()
assert content, "Content not found"
return content

3
app/core/models/user/__init__.py
View File

@ -3,11 +3,12 @@ from sqlalchemy.orm import relationship
from app.core.auth_v1 import AuthenticationMixin as AuthenticationMixin_V1
from app.core.models.user.display_mixin import DisplayMixin
from app.core.models.user.wallet_mixin import WalletMixin
from app.core.translation import TranslationCore
from ..base import AlchemyBase
class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1):
class User(AlchemyBase, DisplayMixin, TranslationCore, AuthenticationMixin_V1, WalletMixin):
LOCALE_DOMAIN = 'sanic_telegram_bot'
__tablename__ = 'users'

74
app/core/models/user/wallet_mixin.py Normal file
View File

@ -0,0 +1,74 @@
from app.core.models.content.user_content import UserContent
from app.core.models.wallet_connection import WalletConnection
from app.core._blockchain.ton.toncenter import toncenter
from tonsdk.utils import Address
from datetime import datetime, timedelta
from app.core.logger import make_log
class WalletMixin:
def wallet_connection(self, db_session):
return db_session.query(WalletConnection).filter(
WalletConnection.user_id == self.id,
WalletConnection.invalidated == False
).order_by(WalletConnection.created.desc()).first()
def wallet_address(self, db_session):
wallet_connection = self.wallet_connection(db_session)
return wallet_connection.wallet_address if wallet_connection else None
async def scan_owned_user_content(self, db_session):
page_id = -1
page_size = 100
have_next_page = True
while have_next_page:
page_id += 1
nfts_list = await toncenter.get_nft_items(limit=100, offset=page_id * page_size)
if len(nfts_list) >= page_size:
have_next_page = True
for nft_item in nfts_list:
item_address = Address(nft_item['address']).to_string(1, 1, 1)
owner_address = Address(nft_item['owner_address']).to_string(1, 1, 1)
user_content = db_session.query(UserContent).filter(
UserContent.onchain_address == item_address
).first()
if user_content:
continue
try:
nft_content = nft_item['content']['uri']
except KeyError:
nft_content = None
user_content = UserContent(
type='nft/unknown',
onchain_address=item_address,
owner_address=owner_address,
code_hash=nft_item['code_hash'],
data_hash=nft_item['data_hash'],
updated=datetime.fromtimestamp(0),
content_id=None, # not resolved yet
created=datetime.now(),
meta={
'metadata_uri': nft_content,
},
user_id=self.id,
wallet_connection_id=self.wallet_connection(db_session).id,
status="active"
)
db_session.add(user_content)
db_session.commit()
make_log(self, f"New onchain NFT found: {item_address}", level='info')
async def get_user_content(self, db_session, limit=100, offset=0):
try:
await self.scan_owned_user_content(db_session)
except BaseException as e:
make_log(self, f"Error while scanning user content: {e}", level='error')
return self.db_session.query(UserContent).filter(
UserContent.user_id == self.id
).offset(offset).limit(limit).all()

17
docker-compose.yml
View File

@ -84,3 +84,20 @@ services:
maria_db:
condition: service_healthy
license_index:
build:
context: .
dockerfile: Dockerfile
command: python -m app license_index
env_file:
- .env
links:
- maria_db
volumes:
- ./logs:/app/logs
- ./storedContent:/app/data
- ./activeConfig:/app/config
depends_on:
maria_db:
condition: service_healthy