relayers
This commit is contained in:
user committed
1 parent
21964fa986
commit
797f379648
68 files changed
+23871
-1271
No files matched your search
+150
-127
@@ -1,135 +1,158 @@
|
||||
"""
|
||||
Main application entry point for my-uploader-bot.
|
||||
Handles startup, shutdown, and application lifecycle management.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import signal
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from asyncio import sleep
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
startup_target = '__main__'
|
||||
try:
|
||||
startup_target = sys.argv[1]
|
||||
except BaseException:
|
||||
pass
|
||||
from app.api import create_app
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import init_database, close_database
|
||||
from app.core.logging import get_logger, setup_logging
|
||||
from app.core.background.ton_service import cleanup_ton_service
|
||||
|
||||
from app.core._utils.create_maria_tables import create_maria_tables
|
||||
from app.core.storage import engine
|
||||
if startup_target == '__main__':
|
||||
create_maria_tables(engine)
|
||||
else:
|
||||
time.sleep(7)
|
||||
# Setup logging first
|
||||
setup_logging()
|
||||
logger = get_logger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
from app.api import app
|
||||
from app.bot import dp as uploader_bot_dp
|
||||
from app.client_bot import dp as client_bot_dp
|
||||
from app.core._config import SANIC_PORT, MYSQL_URI, PROJECT_HOST
|
||||
from app.core.logger import make_log
|
||||
|
||||
if int(os.getenv("SANIC_MAINTENANCE", '0')) == 1:
|
||||
make_log("Global", "Application is in maintenance mode")
|
||||
while True:
|
||||
time.sleep(1)
|
||||
|
||||
from app.core.models import Memory
|
||||
|
||||
|
||||
async def queue_daemon(app):
|
||||
await sleep(3)
|
||||
|
||||
while True:
|
||||
delayed_list = {k: v for k, v in app.ctx.memory._delayed_queue.items()}
|
||||
for _execute_ts in delayed_list:
|
||||
if _execute_ts <= datetime.now().timestamp():
|
||||
del app.ctx.memory._delayed_queue[_execute_ts]
|
||||
app.ctx.memory._execute_queue.append(delayed_list[_execute_ts])
|
||||
|
||||
await sleep(.7)
|
||||
|
||||
|
||||
async def execute_queue(app):
|
||||
telegram_bot_username = (await app.ctx.memory._telegram_bot.get_me()).username
|
||||
client_telegram_bot_username = (await app.ctx.memory._client_telegram_bot.get_me()).username
|
||||
make_log(None, f"Application normally started. HTTP port: {SANIC_PORT}")
|
||||
make_log(None, f"Telegram bot: https://t.me/{telegram_bot_username}")
|
||||
make_log(None, f"Client Telegram bot: https://t.me/{client_telegram_bot_username}")
|
||||
make_log(None, f"MariaDB host: {MYSQL_URI.split('@')[1].split('/')[0].replace('/', '')}")
|
||||
make_log(None, f"API host: {PROJECT_HOST}")
|
||||
while True:
|
||||
class ApplicationManager:
|
||||
"""Manages application lifecycle and graceful shutdown."""
|
||||
|
||||
def __init__(self):
|
||||
self.app = None
|
||||
self.shutdown_event = asyncio.Event()
|
||||
self.tasks = []
|
||||
|
||||
async def startup(self):
|
||||
"""Initialize application and all services."""
|
||||
try:
|
||||
_cmd = app.ctx.memory._execute_queue.pop(0)
|
||||
except IndexError:
|
||||
await sleep(.05)
|
||||
continue
|
||||
|
||||
_fn = _cmd.pop(0)
|
||||
assert _fn
|
||||
_args = _cmd.pop(0)
|
||||
assert type(_args) is tuple
|
||||
await logger.ainfo("Starting my-uploader-bot application", version="2.0.0")
|
||||
|
||||
# Initialize database connections
|
||||
await logger.ainfo("Initializing database connections")
|
||||
await init_database()
|
||||
|
||||
# Create Sanic application
|
||||
await logger.ainfo("Creating Sanic application")
|
||||
self.app = await create_app()
|
||||
|
||||
# Setup signal handlers for graceful shutdown
|
||||
self._setup_signal_handlers()
|
||||
|
||||
await logger.ainfo(
|
||||
"Application startup completed",
|
||||
host=settings.HOST,
|
||||
port=settings.PORT,
|
||||
debug=settings.DEBUG,
|
||||
workers=settings.WORKERS
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Application startup failed", error=str(e))
|
||||
await self.shutdown()
|
||||
sys.exit(1)
|
||||
|
||||
async def run(self):
|
||||
"""Run the application server."""
|
||||
try:
|
||||
_kwargs = _cmd.pop(0)
|
||||
assert type(_kwargs) is dict
|
||||
except IndexError:
|
||||
_kwargs = {}
|
||||
|
||||
try:
|
||||
make_log("Queue.execute", f"{_fn} {_args} {_kwargs}", level='debug')
|
||||
await _fn(*_args, **_kwargs)
|
||||
except BaseException as e:
|
||||
make_log("Queue.execute", f"{_fn} {_args} {_kwargs} => Error: {e}" + '\n' + str(traceback.format_exc()))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main_memory = Memory()
|
||||
if startup_target == '__main__':
|
||||
app.ctx.memory = main_memory
|
||||
for _target in [uploader_bot_dp, client_bot_dp]:
|
||||
_target._s_memory = app.ctx.memory
|
||||
|
||||
app.ctx.memory._app = app
|
||||
|
||||
app.add_task(execute_queue(app))
|
||||
app.add_task(queue_daemon(app))
|
||||
app.add_task(uploader_bot_dp.start_polling(app.ctx.memory._telegram_bot))
|
||||
app.add_task(client_bot_dp.start_polling(app.ctx.memory._client_telegram_bot))
|
||||
|
||||
app.run(host='0.0.0.0', port=SANIC_PORT)
|
||||
else:
|
||||
time.sleep(2)
|
||||
|
||||
startup_fn = None
|
||||
if startup_target == 'indexer':
|
||||
from app.core.background.indexer_service import main_fn as target_fn
|
||||
time.sleep(1)
|
||||
elif startup_target == 'uploader':
|
||||
from app.core.background.uploader_service import main_fn as target_fn
|
||||
time.sleep(3)
|
||||
elif startup_target == 'ton_daemon':
|
||||
from app.core.background.ton_service import main_fn as target_fn
|
||||
time.sleep(5)
|
||||
elif startup_target == 'license_index':
|
||||
from app.core.background.license_service import main_fn as target_fn
|
||||
time.sleep(7)
|
||||
elif startup_target == 'convert_process':
|
||||
from app.core.background.convert_service import main_fn as target_fn
|
||||
time.sleep(9)
|
||||
|
||||
startup_fn = startup_fn or target_fn
|
||||
assert startup_fn
|
||||
|
||||
async def wrapped_startup_fn(*args):
|
||||
try:
|
||||
await startup_fn(*args)
|
||||
except BaseException as e:
|
||||
make_log(startup_target[0].upper() + startup_target[1:], f"Error: {e}" + '\n' + str(traceback.format_exc()),
|
||||
level='error')
|
||||
sys.exit(1)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
loop.run_until_complete(wrapped_startup_fn(main_memory))
|
||||
except BaseException as e:
|
||||
make_log(startup_target[0].upper() + startup_target[1:], f"Error: {e}" + '\n' + str(traceback.format_exc()),
|
||||
level='error')
|
||||
sys.exit(0)
|
||||
if not self.app:
|
||||
await self.startup()
|
||||
|
||||
# Run the Sanic server
|
||||
await logger.ainfo("Starting HTTP server")
|
||||
|
||||
server_config = {
|
||||
"host": settings.HOST,
|
||||
"port": settings.PORT,
|
||||
"debug": settings.DEBUG,
|
||||
"access_log": settings.DEBUG,
|
||||
"auto_reload": settings.AUTO_RELOAD,
|
||||
"workers": settings.WORKERS if not settings.DEBUG else 1,
|
||||
}
|
||||
|
||||
# Start server
|
||||
await self.app.create_server(**server_config, return_asyncio_server=True)
|
||||
|
||||
# Wait for shutdown signal
|
||||
await self.shutdown_event.wait()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
await logger.ainfo("Received keyboard interrupt, shutting down")
|
||||
except Exception as e:
|
||||
await logger.aerror("Server error", error=str(e))
|
||||
finally:
|
||||
loop.close()
|
||||
await self.shutdown()
|
||||
|
||||
async def shutdown(self):
|
||||
"""Gracefully shutdown the application."""
|
||||
await logger.ainfo("Initiating graceful shutdown")
|
||||
|
||||
try:
|
||||
# Cancel all background tasks
|
||||
for task in self.tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# Cleanup services
|
||||
await logger.ainfo("Cleaning up services")
|
||||
|
||||
# Cleanup TON service
|
||||
await cleanup_ton_service()
|
||||
|
||||
# Close database connections
|
||||
await close_database()
|
||||
|
||||
await logger.ainfo("Graceful shutdown completed")
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Error during shutdown", error=str(e))
|
||||
|
||||
def _setup_signal_handlers(self):
|
||||
"""Setup signal handlers for graceful shutdown."""
|
||||
def signal_handler(signum, frame):
|
||||
signal_name = signal.Signals(signum).name
|
||||
logger.info(f"Received {signal_name}, initiating shutdown")
|
||||
self.shutdown_event.set()
|
||||
|
||||
# Register signal handlers
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
# Unix-specific signals
|
||||
if hasattr(signal, 'SIGHUP'):
|
||||
signal.signal(signal.SIGHUP, signal_handler)
|
||||
|
||||
async def main():
|
||||
"""Main application entry point."""
|
||||
app_manager = ApplicationManager()
|
||||
|
||||
try:
|
||||
await app_manager.run()
|
||||
except Exception as e:
|
||||
logger.error(f"Application failed: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Ensure we're running from the correct directory
|
||||
app_root = Path(__file__).parent.parent
|
||||
if app_root.exists():
|
||||
import os
|
||||
os.chdir(app_root)
|
||||
|
||||
# Run the application
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\nApplication interrupted by user")
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(f"Failed to start application: {e}")
|
||||
sys.exit(1)
|
||||
+441
-70
@@ -1,80 +1,451 @@
|
||||
import traceback
|
||||
"""
|
||||
Enhanced Sanic API application with async support and monitoring
|
||||
"""
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from sanic import Sanic, response
|
||||
from sanic import Sanic, Request, HTTPResponse
|
||||
from sanic.response import json as json_response
|
||||
from sanic.exceptions import SanicException
|
||||
import structlog
|
||||
|
||||
from app.core.logger import make_log
|
||||
from app.core.config import settings
|
||||
from app.core.database import init_database, close_database
|
||||
from app.core.logging import get_logger, AsyncContextLogger
|
||||
from app.api.middleware import (
|
||||
request_middleware,
|
||||
response_middleware,
|
||||
exception_middleware,
|
||||
maintenance_middleware
|
||||
)
|
||||
|
||||
app = Sanic(__name__)
|
||||
|
||||
from app.api.middleware import attach_user_to_request, close_db_session, close_request_handler
|
||||
|
||||
app.register_middleware(attach_user_to_request, "request")
|
||||
app.register_middleware(close_db_session, "response")
|
||||
|
||||
from app.api.routes._index import s_index, s_favicon
|
||||
from app.api.routes._system import s_api_v1_node, s_api_system_version, s_api_system_send_status, s_api_v1_node_friendly
|
||||
from app.api.routes.auth import s_api_v1_auth_twa, s_api_v1_auth_select_wallet, s_api_v1_auth_me
|
||||
from app.api.routes.statics import s_api_tonconnect_manifest, s_api_platform_metadata
|
||||
from app.api.routes.node_storage import s_api_v1_storage_post, s_api_v1_storage_get, \
|
||||
s_api_v1_storage_decode_cid
|
||||
from app.api.routes.progressive_storage import s_api_v1_5_storage_get, s_api_v1_5_storage_post
|
||||
from app.api.routes.account import s_api_v1_account_get
|
||||
from app.api.routes._blockchain import s_api_v1_blockchain_send_new_content_message, \
|
||||
s_api_v1_blockchain_send_purchase_content_message
|
||||
from app.api.routes.content import s_api_v1_content_list, s_api_v1_content_view, s_api_v1_content_friendly_list, s_api_v1_5_content_list
|
||||
from app.api.routes.tonconnect import s_api_v1_tonconnect_new, s_api_v1_tonconnect_logout
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
app.add_route(s_index, "/", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_favicon, "/favicon.ico", methods=["GET", "OPTIONS"])
|
||||
|
||||
app.add_route(s_api_v1_node, "/api/v1/node", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_v1_node_friendly, "/api/v1/nodeFriendly", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_system_version, "/api/system.version", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_system_send_status, "/api/system.sendStatus", methods=["POST", "OPTIONS"])
|
||||
|
||||
app.add_route(s_api_tonconnect_manifest, "/api/tonconnect-manifest.json", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_platform_metadata, "/api/platform-metadata.json", methods=["GET", "OPTIONS"])
|
||||
|
||||
app.add_route(s_api_v1_auth_twa, "/api/v1/auth.twa", methods=["POST", "OPTIONS"])
|
||||
app.add_route(s_api_v1_auth_me, "/api/v1/auth.me", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_v1_auth_select_wallet, "/api/v1/auth.selectWallet", methods=["POST", "OPTIONS"])
|
||||
|
||||
app.add_route(s_api_v1_tonconnect_new, "/api/v1/tonconnect.new", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_v1_tonconnect_logout, "/api/v1/tonconnect.logout", methods=["POST", "OPTIONS"])
|
||||
|
||||
app.add_route(s_api_v1_5_storage_post, "/api/v1.5/storage", methods=["POST", "OPTIONS"])
|
||||
app.add_route(s_api_v1_5_storage_get, "/api/v1.5/storage/<file_hash>", methods=["GET", "OPTIONS"])
|
||||
|
||||
app.add_route(s_api_v1_storage_post, "/api/v1/storage", methods=["POST", "OPTIONS"])
|
||||
app.add_route(s_api_v1_storage_get, "/api/v1/storage/<file_hash>", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_v1_storage_decode_cid, "/api/v1/storage.decodeContentId/<content_id>", methods=["GET", "OPTIONS"])
|
||||
|
||||
app.add_route(s_api_v1_account_get, "/api/v1/account", methods=["GET", "OPTIONS"])
|
||||
|
||||
app.add_route(s_api_v1_blockchain_send_new_content_message, "/api/v1/blockchain.sendNewContentMessage", methods=["POST", "OPTIONS"])
|
||||
app.add_route(s_api_v1_blockchain_send_purchase_content_message, "/api/v1/blockchain.sendPurchaseContentMessage", methods=["POST", "OPTIONS"])
|
||||
|
||||
app.add_route(s_api_v1_content_list, "/api/v1/content.list", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_v1_content_view, "/api/v1/content.view/<content_address>", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_v1_content_friendly_list, "/api/v1/content.friendlyList", methods=["GET", "OPTIONS"])
|
||||
app.add_route(s_api_v1_5_content_list, "/api/v1.5/content.list", methods=["GET", "OPTIONS"])
|
||||
class EnhancedSanic(Sanic):
|
||||
"""Enhanced Sanic application with additional features"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ctx.startup_tasks = []
|
||||
self.ctx.shutdown_tasks = []
|
||||
self.ctx.background_tasks = set()
|
||||
|
||||
async def startup(self):
|
||||
"""Application startup tasks"""
|
||||
async with AsyncContextLogger("app_startup"):
|
||||
# Initialize database
|
||||
await init_database()
|
||||
logger.info("Database initialized")
|
||||
|
||||
# Initialize cache connections
|
||||
from app.core.database import get_cache
|
||||
cache = await get_cache()
|
||||
await cache.redis.ping()
|
||||
logger.info("Redis cache initialized")
|
||||
|
||||
# Run custom startup tasks
|
||||
for task in self.ctx.startup_tasks:
|
||||
try:
|
||||
await task()
|
||||
except Exception as e:
|
||||
logger.error("Startup task failed", task=task.__name__, error=str(e))
|
||||
|
||||
logger.info("Application startup completed")
|
||||
|
||||
async def shutdown(self):
|
||||
"""Application shutdown tasks"""
|
||||
async with AsyncContextLogger("app_shutdown"):
|
||||
# Cancel background tasks
|
||||
for task in self.ctx.background_tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
# Wait for tasks to complete
|
||||
if self.ctx.background_tasks:
|
||||
await asyncio.gather(*self.ctx.background_tasks, return_exceptions=True)
|
||||
|
||||
# Run custom shutdown tasks
|
||||
for task in self.ctx.shutdown_tasks:
|
||||
try:
|
||||
await task()
|
||||
except Exception as e:
|
||||
logger.error("Shutdown task failed", task=task.__name__, error=str(e))
|
||||
|
||||
# Close database connections
|
||||
await close_database()
|
||||
logger.info("Database connections closed")
|
||||
|
||||
logger.info("Application shutdown completed")
|
||||
|
||||
def add_startup_task(self, task):
|
||||
"""Add startup task"""
|
||||
self.ctx.startup_tasks.append(task)
|
||||
|
||||
def add_shutdown_task(self, task):
|
||||
"""Add shutdown task"""
|
||||
self.ctx.shutdown_tasks.append(task)
|
||||
|
||||
def add_background_task(self, coro):
|
||||
"""Add background task"""
|
||||
task = asyncio.create_task(coro)
|
||||
self.ctx.background_tasks.add(task)
|
||||
task.add_done_callback(self.ctx.background_tasks.discard)
|
||||
return task
|
||||
|
||||
|
||||
@app.exception(BaseException)
|
||||
async def s_handle_exception(request, exception):
|
||||
response_buffer = response.json({"error": "An internal server error occurred"}, status=500)
|
||||
# Create Sanic app instance
|
||||
app = EnhancedSanic(
|
||||
name="my_uploader_bot",
|
||||
configure_logging=False # We handle logging ourselves
|
||||
)
|
||||
|
||||
# Configure app settings
|
||||
app.config.update({
|
||||
"REQUEST_MAX_SIZE": settings.MAX_FILE_SIZE,
|
||||
"REQUEST_TIMEOUT": 60,
|
||||
"RESPONSE_TIMEOUT": 60,
|
||||
"KEEP_ALIVE_TIMEOUT": 5,
|
||||
"KEEP_ALIVE": True,
|
||||
"ACCESS_LOG": False, # We handle access logging in middleware
|
||||
"AUTO_RELOAD": settings.DEBUG,
|
||||
"DEBUG": settings.DEBUG,
|
||||
})
|
||||
|
||||
|
||||
# Register middleware
|
||||
app.register_middleware(maintenance_middleware, "request")
|
||||
app.register_middleware(request_middleware, "request")
|
||||
app.register_middleware(response_middleware, "response")
|
||||
|
||||
|
||||
# Global exception handler
|
||||
@app.exception(Exception)
|
||||
async def handle_exception(request: Request, exception: Exception):
|
||||
"""Global exception handler"""
|
||||
return await exception_middleware(request, exception)
|
||||
|
||||
|
||||
# Health check endpoint
|
||||
@app.get("/health")
|
||||
async def health_check(request: Request):
|
||||
"""Health check endpoint"""
|
||||
try:
|
||||
raise exception
|
||||
except AssertionError as e:
|
||||
response_buffer = response.json({"error": str(e)}, status=400)
|
||||
except BaseException as e:
|
||||
make_log("sanic_exception", f"Exception: {e}" + '\n' + str(traceback.format_exc()), level='error')
|
||||
# Check database connection
|
||||
from app.core.database import db_manager
|
||||
async with db_manager.get_session() as session:
|
||||
await session.execute("SELECT 1")
|
||||
|
||||
# Check Redis connection
|
||||
from app.core.database import get_cache
|
||||
cache = await get_cache()
|
||||
await cache.redis.ping()
|
||||
|
||||
return json_response({
|
||||
"status": "healthy",
|
||||
"version": settings.PROJECT_VERSION,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Health check failed", error=str(e))
|
||||
return json_response({
|
||||
"status": "unhealthy",
|
||||
"error": str(e),
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}, status=503)
|
||||
|
||||
response_buffer = await close_db_session(request, response_buffer)
|
||||
response_buffer.headers["Access-Control-Allow-Origin"] = "*"
|
||||
response_buffer.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
|
||||
response_buffer.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site"
|
||||
response_buffer.headers["Access-Control-Allow-Credentials"] = "true"
|
||||
return response_buffer
|
||||
|
||||
# Metrics endpoint (if enabled)
|
||||
if settings.METRICS_ENABLED:
|
||||
@app.get("/metrics")
|
||||
async def metrics_endpoint(request: Request):
|
||||
"""Prometheus metrics endpoint"""
|
||||
try:
|
||||
from prometheus_client import generate_latest, CONTENT_TYPE_LATEST
|
||||
metrics_data = generate_latest()
|
||||
return HTTPResponse(
|
||||
body=metrics_data,
|
||||
headers={"Content-Type": CONTENT_TYPE_LATEST},
|
||||
status=200
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Metrics generation failed", error=str(e))
|
||||
return json_response({"error": "Metrics unavailable"}, status=503)
|
||||
|
||||
|
||||
# System info endpoint
|
||||
@app.get("/api/system/info")
|
||||
async def system_info(request: Request):
|
||||
"""System information endpoint"""
|
||||
try:
|
||||
import psutil
|
||||
import sys
|
||||
|
||||
# Get system metrics
|
||||
memory = psutil.virtual_memory()
|
||||
disk = psutil.disk_usage('/')
|
||||
|
||||
info = {
|
||||
"application": {
|
||||
"name": settings.PROJECT_NAME,
|
||||
"version": settings.PROJECT_VERSION,
|
||||
"python_version": sys.version,
|
||||
"debug": settings.DEBUG,
|
||||
},
|
||||
"system": {
|
||||
"cpu_percent": psutil.cpu_percent(),
|
||||
"memory": {
|
||||
"total": memory.total,
|
||||
"available": memory.available,
|
||||
"percent": memory.percent
|
||||
},
|
||||
"disk": {
|
||||
"total": disk.total,
|
||||
"free": disk.free,
|
||||
"percent": (disk.used / disk.total) * 100
|
||||
}
|
||||
},
|
||||
"services": {
|
||||
"database": "connected",
|
||||
"redis": "connected",
|
||||
"indexer": "running" if settings.INDEXER_ENABLED else "disabled",
|
||||
"ton_daemon": "running" if settings.TON_DAEMON_ENABLED else "disabled",
|
||||
}
|
||||
}
|
||||
|
||||
return json_response(info)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("System info failed", error=str(e))
|
||||
return json_response({"error": "System info unavailable"}, status=500)
|
||||
|
||||
|
||||
# Register API routes
|
||||
def register_routes():
|
||||
"""Register all API routes"""
|
||||
from app.api.routes import (
|
||||
auth_routes,
|
||||
content_routes,
|
||||
storage_routes,
|
||||
blockchain_routes,
|
||||
admin_routes,
|
||||
user_routes,
|
||||
system_routes
|
||||
)
|
||||
|
||||
# Register route blueprints
|
||||
app.blueprint(auth_routes.bp)
|
||||
app.blueprint(content_routes.bp)
|
||||
app.blueprint(storage_routes.bp)
|
||||
app.blueprint(blockchain_routes.bp)
|
||||
app.blueprint(admin_routes.bp)
|
||||
app.blueprint(user_routes.bp)
|
||||
app.blueprint(system_routes.bp)
|
||||
|
||||
# Попробовать добавить MY Network маршруты
|
||||
try:
|
||||
from app.api.routes import my_network_sanic, my_monitoring_sanic
|
||||
|
||||
# Создать MY Network blueprint'ы
|
||||
app.blueprint(my_network_sanic.bp)
|
||||
app.blueprint(my_monitoring_sanic.bp)
|
||||
|
||||
logger.info("MY Network routes registered")
|
||||
|
||||
except ImportError as e:
|
||||
logger.warning("MY Network routes not available", error=str(e))
|
||||
|
||||
logger.info("API routes registered")
|
||||
|
||||
|
||||
# Application lifecycle hooks
|
||||
@app.before_server_start
|
||||
async def before_server_start(app, loop):
|
||||
"""Tasks to run before server starts"""
|
||||
await app.startup()
|
||||
|
||||
|
||||
@app.after_server_stop
|
||||
async def after_server_stop(app, loop):
|
||||
"""Tasks to run after server stops"""
|
||||
await app.shutdown()
|
||||
|
||||
|
||||
# Background task management
|
||||
class BackgroundTaskManager:
|
||||
"""Manager for background tasks"""
|
||||
|
||||
def __init__(self, app: EnhancedSanic):
|
||||
self.app = app
|
||||
self.tasks: Dict[str, asyncio.Task] = {}
|
||||
|
||||
async def start_service(self, name: str, service_func, *args, **kwargs):
|
||||
"""Start a background service"""
|
||||
if name in self.tasks:
|
||||
logger.warning("Service already running", service=name)
|
||||
return
|
||||
|
||||
logger.info("Starting background service", service=name)
|
||||
task = self.app.add_background_task(service_func(*args, **kwargs))
|
||||
self.tasks[name] = task
|
||||
|
||||
# Add error handling
|
||||
task.add_done_callback(lambda t: self._handle_task_completion(name, t))
|
||||
|
||||
def _handle_task_completion(self, name: str, task: asyncio.Task):
|
||||
"""Handle background task completion"""
|
||||
if name in self.tasks:
|
||||
del self.tasks[name]
|
||||
|
||||
if task.cancelled():
|
||||
logger.info("Background service cancelled", service=name)
|
||||
elif task.exception():
|
||||
logger.error("Background service failed", service=name, error=str(task.exception()))
|
||||
else:
|
||||
logger.info("Background service completed", service=name)
|
||||
|
||||
async def stop_service(self, name: str):
|
||||
"""Stop a background service"""
|
||||
if name not in self.tasks:
|
||||
logger.warning("Service not running", service=name)
|
||||
return
|
||||
|
||||
logger.info("Stopping background service", service=name)
|
||||
task = self.tasks[name]
|
||||
task.cancel()
|
||||
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def stop_all_services(self):
|
||||
"""Stop all background services"""
|
||||
for name in list(self.tasks.keys()):
|
||||
await self.stop_service(name)
|
||||
|
||||
def get_service_status(self) -> Dict[str, str]:
|
||||
"""Get status of all services"""
|
||||
status = {}
|
||||
for name, task in self.tasks.items():
|
||||
if task.done():
|
||||
if task.cancelled():
|
||||
status[name] = "cancelled"
|
||||
elif task.exception():
|
||||
status[name] = "failed"
|
||||
else:
|
||||
status[name] = "completed"
|
||||
else:
|
||||
status[name] = "running"
|
||||
return status
|
||||
|
||||
|
||||
# Initialize background task manager
|
||||
task_manager = BackgroundTaskManager(app)
|
||||
app.ctx.task_manager = task_manager
|
||||
|
||||
|
||||
# Service startup functions
|
||||
async def start_background_services():
|
||||
"""Start all background services"""
|
||||
from app.core.background import (
|
||||
indexer_service,
|
||||
ton_service,
|
||||
license_service,
|
||||
convert_service,
|
||||
uploader_service
|
||||
)
|
||||
|
||||
if settings.INDEXER_ENABLED:
|
||||
await task_manager.start_service("indexer", indexer_service.main_fn)
|
||||
|
||||
if settings.TON_DAEMON_ENABLED:
|
||||
await task_manager.start_service("ton_daemon", ton_service.main_fn)
|
||||
|
||||
if settings.LICENSE_SERVICE_ENABLED:
|
||||
await task_manager.start_service("license_service", license_service.main_fn)
|
||||
|
||||
if settings.CONVERT_SERVICE_ENABLED:
|
||||
await task_manager.start_service("convert_service", convert_service.main_fn)
|
||||
|
||||
# Попробовать запустить MY Network сервис
|
||||
try:
|
||||
await start_my_network_service()
|
||||
except Exception as e:
|
||||
logger.warning("MY Network service not started", error=str(e))
|
||||
|
||||
logger.info("Background services started")
|
||||
|
||||
|
||||
async def start_my_network_service():
|
||||
"""Запустить MY Network сервис."""
|
||||
try:
|
||||
from app.core.my_network.node_service import NodeService
|
||||
|
||||
# Создать и запустить сервис ноды
|
||||
node_service = NodeService()
|
||||
|
||||
# Добавить как фоновую задачу
|
||||
async def my_network_task():
|
||||
await node_service.start()
|
||||
|
||||
# Держать сервис активным
|
||||
try:
|
||||
while True:
|
||||
await asyncio.sleep(60) # Проверять каждую минуту
|
||||
|
||||
# Проверить состояние сервиса
|
||||
if not node_service.is_running:
|
||||
logger.warning("MY Network service stopped unexpectedly")
|
||||
break
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.info("MY Network service shutdown requested")
|
||||
await node_service.stop()
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("MY Network service error", error=str(e))
|
||||
await node_service.stop()
|
||||
raise
|
||||
|
||||
await task_manager.start_service("my_network", my_network_task)
|
||||
logger.info("MY Network service started")
|
||||
|
||||
except ImportError as e:
|
||||
logger.info("MY Network modules not available", error=str(e))
|
||||
except Exception as e:
|
||||
logger.error("Failed to start MY Network service", error=str(e))
|
||||
raise
|
||||
|
||||
|
||||
# Add startup task
|
||||
app.add_startup_task(start_background_services)
|
||||
app.add_shutdown_task(task_manager.stop_all_services)
|
||||
|
||||
# Register routes
|
||||
register_routes()
|
||||
|
||||
|
||||
# Main application factory
|
||||
def create_app() -> EnhancedSanic:
|
||||
"""Application factory"""
|
||||
return app
|
||||
|
||||
|
||||
# Development server runner
|
||||
async def run_dev_server():
|
||||
"""Run development server"""
|
||||
await app.create_server(
|
||||
host="0.0.0.0",
|
||||
port=settings.SANIC_PORT,
|
||||
debug=settings.DEBUG,
|
||||
auto_reload=settings.DEBUG,
|
||||
access_log=False
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger.info("Starting development server")
|
||||
asyncio.run(run_dev_server())
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
"""OpenAPI documentation configuration for my-uploader-bot API."""
|
||||
|
||||
from typing import Dict, Any
|
||||
|
||||
# API metadata
|
||||
API_TITLE = "My Uploader Bot API"
|
||||
API_VERSION = "2.0.0"
|
||||
API_DESCRIPTION = """
|
||||
# My Uploader Bot API
|
||||
|
||||
A comprehensive file upload and management system with blockchain integration.
|
||||
|
||||
## Features
|
||||
|
||||
- **File Upload & Management**: Chunked uploads, multiple storage backends, file processing
|
||||
- **User Authentication**: JWT tokens, API keys, sessions management
|
||||
- **Blockchain Integration**: TON blockchain wallet management, transactions
|
||||
- **Content Management**: Version control, metadata, search functionality
|
||||
- **Security**: Rate limiting, CORS, input validation, file encryption
|
||||
- **Monitoring**: Prometheus metrics, structured logging, health checks
|
||||
|
||||
## Authentication
|
||||
|
||||
The API supports multiple authentication methods:
|
||||
|
||||
1. **JWT Bearer Token**: Use `Authorization: Bearer <token>` header
|
||||
2. **API Key**: Use `X-API-Key: <api_key>` header
|
||||
3. **Session Cookie**: Browser-based authentication
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
API endpoints are rate-limited based on user tier:
|
||||
- Free tier: 100 requests per hour
|
||||
- Premium tier: 1000 requests per hour
|
||||
- Enterprise tier: 10000 requests per hour
|
||||
|
||||
## File Upload Process
|
||||
|
||||
1. **Initiate Upload**: POST `/api/v1/storage/upload/initiate` with file metadata
|
||||
2. **Upload Chunks**: POST `/api/v1/storage/upload/chunk` for each chunk
|
||||
3. **Complete Upload**: POST `/api/v1/storage/upload/complete` to finalize
|
||||
4. **Processing**: File is automatically processed in the background
|
||||
|
||||
## Error Handling
|
||||
|
||||
All errors follow RFC 7807 Problem Details format:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "https://api.myuploader.com/errors/validation",
|
||||
"title": "Validation Error",
|
||||
"status": 422,
|
||||
"detail": "The request body contains invalid data",
|
||||
"instance": "/api/v1/content/upload",
|
||||
"errors": [
|
||||
{
|
||||
"field": "file_size",
|
||||
"message": "File size exceeds maximum limit"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Webhook Events
|
||||
|
||||
The API can send webhook notifications for:
|
||||
- File upload completion
|
||||
- Processing status updates
|
||||
- Blockchain transaction confirmations
|
||||
- User subscription changes
|
||||
|
||||
## SDKs and Examples
|
||||
|
||||
- Python SDK: `pip install myuploader-python`
|
||||
- JavaScript SDK: `npm install @myuploader/js-sdk`
|
||||
- Examples: https://github.com/myuploader/examples
|
||||
|
||||
## Support
|
||||
|
||||
- Documentation: https://docs.myuploader.com
|
||||
- Support: support@myuploader.com
|
||||
- Status: https://status.myuploader.com
|
||||
"""
|
||||
|
||||
# OpenAPI tags
|
||||
TAGS_METADATA = [
|
||||
{
|
||||
"name": "Authentication",
|
||||
"description": "User authentication and session management endpoints",
|
||||
},
|
||||
{
|
||||
"name": "Users",
|
||||
"description": "User profile and account management",
|
||||
},
|
||||
{
|
||||
"name": "Content",
|
||||
"description": "Content management, search, and metadata operations",
|
||||
},
|
||||
{
|
||||
"name": "Storage",
|
||||
"description": "File upload, download, and storage operations",
|
||||
},
|
||||
{
|
||||
"name": "Blockchain",
|
||||
"description": "TON blockchain wallet and transaction management",
|
||||
},
|
||||
{
|
||||
"name": "System",
|
||||
"description": "System health, metrics, and administrative endpoints",
|
||||
},
|
||||
]
|
||||
|
||||
# Response examples
|
||||
RESPONSE_EXAMPLES = {
|
||||
"user_profile": {
|
||||
"summary": "User profile example",
|
||||
"value": {
|
||||
"id": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"username": "john_doe",
|
||||
"email": "john@example.com",
|
||||
"first_name": "John",
|
||||
"last_name": "Doe",
|
||||
"is_active": True,
|
||||
"is_verified": True,
|
||||
"avatar_url": "https://cdn.myuploader.com/avatars/john_doe.jpg",
|
||||
"bio": "Software developer and blockchain enthusiast",
|
||||
"created_at": "2024-01-01T00:00:00Z",
|
||||
"updated_at": "2024-01-01T00:00:00Z"
|
||||
}
|
||||
},
|
||||
"content_item": {
|
||||
"summary": "Content item example",
|
||||
"value": {
|
||||
"id": "123e4567-e89b-12d3-a456-426614174001",
|
||||
"title": "My Awesome Video",
|
||||
"description": "A great video about blockchain development",
|
||||
"content_type": "video",
|
||||
"file_path": "uploads/user123/video_2024_01_01.mp4",
|
||||
"file_size": 104857600,
|
||||
"mime_type": "video/mp4",
|
||||
"is_public": True,
|
||||
"view_count": 1250,
|
||||
"download_count": 95,
|
||||
"like_count": 42,
|
||||
"tags": ["blockchain", "tutorial", "development"],
|
||||
"thumbnail_url": "https://cdn.myuploader.com/thumbnails/video_thumb.jpg",
|
||||
"status": "published",
|
||||
"created_at": "2024-01-01T00:00:00Z",
|
||||
"updated_at": "2024-01-01T00:00:00Z"
|
||||
}
|
||||
},
|
||||
"upload_session": {
|
||||
"summary": "Upload session example",
|
||||
"value": {
|
||||
"session_id": "upload_123e4567-e89b-12d3-a456-426614174002",
|
||||
"filename": "large_video.mp4",
|
||||
"file_size": 1073741824,
|
||||
"chunk_size": 1048576,
|
||||
"total_chunks": 1024,
|
||||
"uploaded_chunks": 512,
|
||||
"status": "uploading",
|
||||
"progress": 50.0,
|
||||
"expires_at": "2024-01-01T01:00:00Z",
|
||||
"upload_urls": [
|
||||
"https://api.myuploader.com/api/v1/storage/upload/chunk"
|
||||
]
|
||||
}
|
||||
},
|
||||
"wallet_info": {
|
||||
"summary": "Wallet information example",
|
||||
"value": {
|
||||
"id": "123e4567-e89b-12d3-a456-426614174003",
|
||||
"address": "EQD6M8aVGx1fF6Z5q5q5q5q5q5q5q5q5q5q5q5q5q5q5q5q5q",
|
||||
"network": "mainnet",
|
||||
"balance": "10.50000000",
|
||||
"is_active": True,
|
||||
"is_primary": True,
|
||||
"created_at": "2024-01-01T00:00:00Z",
|
||||
"transactions": [
|
||||
{
|
||||
"tx_hash": "abc123def456ghi789jkl012mno345pqr678stu901vwx234yz",
|
||||
"amount": "5.00000000",
|
||||
"status": "confirmed",
|
||||
"created_at": "2024-01-01T00:30:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"error_validation": {
|
||||
"summary": "Validation error example",
|
||||
"value": {
|
||||
"type": "https://api.myuploader.com/errors/validation",
|
||||
"title": "Validation Error",
|
||||
"status": 422,
|
||||
"detail": "The request contains invalid data",
|
||||
"instance": "/api/v1/content/upload",
|
||||
"errors": [
|
||||
{
|
||||
"field": "file_size",
|
||||
"message": "File size must be less than 100MB"
|
||||
},
|
||||
{
|
||||
"field": "content_type",
|
||||
"message": "Content type is required"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"error_auth": {
|
||||
"summary": "Authentication error example",
|
||||
"value": {
|
||||
"type": "https://api.myuploader.com/errors/authentication",
|
||||
"title": "Authentication Required",
|
||||
"status": 401,
|
||||
"detail": "Valid authentication credentials are required",
|
||||
"instance": "/api/v1/content/private"
|
||||
}
|
||||
},
|
||||
"error_forbidden": {
|
||||
"summary": "Permission error example",
|
||||
"value": {
|
||||
"type": "https://api.myuploader.com/errors/forbidden",
|
||||
"title": "Insufficient Permissions",
|
||||
"status": 403,
|
||||
"detail": "You don't have permission to access this resource",
|
||||
"instance": "/api/v1/admin/users"
|
||||
}
|
||||
},
|
||||
"error_not_found": {
|
||||
"summary": "Not found error example",
|
||||
"value": {
|
||||
"type": "https://api.myuploader.com/errors/not-found",
|
||||
"title": "Resource Not Found",
|
||||
"status": 404,
|
||||
"detail": "The requested resource was not found",
|
||||
"instance": "/api/v1/content/nonexistent-id"
|
||||
}
|
||||
},
|
||||
"error_rate_limit": {
|
||||
"summary": "Rate limit error example",
|
||||
"value": {
|
||||
"type": "https://api.myuploader.com/errors/rate-limit",
|
||||
"title": "Rate Limit Exceeded",
|
||||
"status": 429,
|
||||
"detail": "Too many requests. Please try again later",
|
||||
"instance": "/api/v1/content/search",
|
||||
"retry_after": 60
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Security schemes
|
||||
SECURITY_SCHEMES = {
|
||||
"BearerAuth": {
|
||||
"type": "http",
|
||||
"scheme": "bearer",
|
||||
"bearerFormat": "JWT",
|
||||
"description": "JWT token authentication. Get token from /api/v1/auth/login"
|
||||
},
|
||||
"ApiKeyAuth": {
|
||||
"type": "apiKey",
|
||||
"in": "header",
|
||||
"name": "X-API-Key",
|
||||
"description": "API key authentication. Get API key from user dashboard"
|
||||
},
|
||||
"CookieAuth": {
|
||||
"type": "apiKey",
|
||||
"in": "cookie",
|
||||
"name": "session",
|
||||
"description": "Session cookie authentication"
|
||||
}
|
||||
}
|
||||
|
||||
# OpenAPI configuration
|
||||
def get_openapi_config() -> Dict[str, Any]:
|
||||
"""Get OpenAPI configuration."""
|
||||
return {
|
||||
"title": API_TITLE,
|
||||
"version": API_VERSION,
|
||||
"description": API_DESCRIPTION,
|
||||
"terms_of_service": "https://myuploader.com/terms",
|
||||
"contact": {
|
||||
"name": "My Uploader Bot Support",
|
||||
"url": "https://myuploader.com/support",
|
||||
"email": "support@myuploader.com"
|
||||
},
|
||||
"license": {
|
||||
"name": "MIT License",
|
||||
"url": "https://opensource.org/licenses/MIT"
|
||||
},
|
||||
"servers": [
|
||||
{
|
||||
"url": "https://api.myuploader.com",
|
||||
"description": "Production server"
|
||||
},
|
||||
{
|
||||
"url": "https://staging-api.myuploader.com",
|
||||
"description": "Staging server"
|
||||
},
|
||||
{
|
||||
"url": "http://localhost:8000",
|
||||
"description": "Development server"
|
||||
}
|
||||
],
|
||||
"tags": TAGS_METADATA,
|
||||
"components": {
|
||||
"securitySchemes": SECURITY_SCHEMES,
|
||||
"examples": RESPONSE_EXAMPLES,
|
||||
"responses": {
|
||||
"ValidationError": {
|
||||
"description": "Validation error response",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"example": RESPONSE_EXAMPLES["error_validation"]["value"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"AuthError": {
|
||||
"description": "Authentication error response",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"example": RESPONSE_EXAMPLES["error_auth"]["value"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"ForbiddenError": {
|
||||
"description": "Permission error response",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"example": RESPONSE_EXAMPLES["error_forbidden"]["value"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"NotFoundError": {
|
||||
"description": "Not found error response",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"example": RESPONSE_EXAMPLES["error_not_found"]["value"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"RateLimitError": {
|
||||
"description": "Rate limit error response",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"example": RESPONSE_EXAMPLES["error_rate_limit"]["value"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{"BearerAuth": []},
|
||||
{"ApiKeyAuth": []},
|
||||
{"CookieAuth": []}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# Custom OpenAPI schema
|
||||
CUSTOM_OPENAPI_SCHEMA = {
|
||||
"x-logo": {
|
||||
"url": "https://myuploader.com/logo.png",
|
||||
"altText": "My Uploader Bot Logo"
|
||||
},
|
||||
"x-code-samples": [
|
||||
{
|
||||
"lang": "Python",
|
||||
"source": """
|
||||
import requests
|
||||
|
||||
# Upload a file
|
||||
response = requests.post(
|
||||
'https://api.myuploader.com/api/v1/storage/upload/initiate',
|
||||
headers={'Authorization': 'Bearer <your_token>'},
|
||||
json={
|
||||
'filename': 'example.jpg',
|
||||
'file_size': 1024000,
|
||||
'content_type': 'image'
|
||||
}
|
||||
)
|
||||
"""
|
||||
},
|
||||
{
|
||||
"lang": "JavaScript",
|
||||
"source": """
|
||||
// Upload a file
|
||||
const response = await fetch('https://api.myuploader.com/api/v1/storage/upload/initiate', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': 'Bearer <your_token>',
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
filename: 'example.jpg',
|
||||
file_size: 1024000,
|
||||
content_type: 'image'
|
||||
})
|
||||
});
|
||||
"""
|
||||
},
|
||||
{
|
||||
"lang": "cURL",
|
||||
"source": """
|
||||
curl -X POST https://api.myuploader.com/api/v1/storage/upload/initiate \\
|
||||
-H "Authorization: Bearer <your_token>" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d '{
|
||||
"filename": "example.jpg",
|
||||
"file_size": 1024000,
|
||||
"content_type": "image"
|
||||
}'
|
||||
"""
|
||||
}
|
||||
]
|
||||
}
|
||||
+485
-159
@@ -1,168 +1,494 @@
|
||||
from base58 import b58decode
|
||||
from sanic import response as sanic_response
|
||||
|
||||
from app.core._crypto.signer import Signer
|
||||
from app.core._secrets import hot_seed
|
||||
from app.core.logger import make_log
|
||||
from app.core.models.keys import KnownKey
|
||||
from app.core.models._telegram.wrapped_bot import Wrapped_CBotChat
|
||||
from app.core.models.user_activity import UserActivity
|
||||
from app.core.models.user import User
|
||||
from app.core.storage import Session
|
||||
"""
|
||||
Enhanced API middleware with security, rate limiting, and monitoring
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional, Dict, Any, Callable
|
||||
import json
|
||||
|
||||
from sanic import Request, HTTPResponse
|
||||
from sanic.response import json as json_response, text as text_response
|
||||
from sanic.exceptions import Unauthorized, Forbidden, TooManyRequests, BadRequest
|
||||
import structlog
|
||||
|
||||
from app.core.config import settings, SecurityConfig, CACHE_KEYS
|
||||
from app.core.database import get_db_session, get_cache
|
||||
from app.core.logging import request_id_var, user_id_var, operation_var, log_performance
|
||||
from app.core.models.user import User
|
||||
from app.core.models.base import BaseModel
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
def attach_headers(response):
|
||||
response.headers["Access-Control-Allow-Origin"] = "*"
|
||||
response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
|
||||
response.headers["Access-Control-Allow-Headers"] = "Origin, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-file-name, x-last-chunk, x-chunk-start, x-upload-id"
|
||||
# response.headers["Access-Control-Allow-Credentials"] = "true"
|
||||
class SecurityMiddleware:
|
||||
"""Security middleware for request validation and protection"""
|
||||
|
||||
@staticmethod
|
||||
def add_security_headers(response: HTTPResponse) -> HTTPResponse:
|
||||
"""Add security headers to response"""
|
||||
# CORS headers
|
||||
response.headers.update({
|
||||
"Access-Control-Allow-Origin": "*", # Will be restricted based on request
|
||||
"Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
|
||||
"Access-Control-Allow-Headers": (
|
||||
"Origin, Content-Type, Accept, Authorization, "
|
||||
"X-Requested-With, X-API-Key, X-Request-ID"
|
||||
),
|
||||
"Access-Control-Max-Age": "86400",
|
||||
|
||||
# Security headers
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"X-Frame-Options": "DENY",
|
||||
"X-XSS-Protection": "1; mode=block",
|
||||
"Strict-Transport-Security": "max-age=31536000; includeSubDomains",
|
||||
"Referrer-Policy": "strict-origin-when-cross-origin",
|
||||
"Permissions-Policy": "geolocation=(), microphone=(), camera=()",
|
||||
|
||||
# Custom headers
|
||||
"X-API-Version": settings.PROJECT_VERSION,
|
||||
"X-Request-ID": getattr(getattr(Request, 'ctx', None), 'request_id', 'unknown')
|
||||
})
|
||||
|
||||
# CSP header
|
||||
csp_directives = "; ".join([
|
||||
f"{directive} {' '.join(sources)}"
|
||||
for directive, sources in SecurityConfig.CSP_DIRECTIVES.items()
|
||||
])
|
||||
response.headers["Content-Security-Policy"] = csp_directives
|
||||
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def validate_request_size(request: Request) -> None:
|
||||
"""Validate request size limits"""
|
||||
content_length = request.headers.get('content-length')
|
||||
if content_length:
|
||||
size = int(content_length)
|
||||
if size > SecurityConfig.MAX_REQUEST_SIZE:
|
||||
raise BadRequest(f"Request too large: {size} bytes")
|
||||
|
||||
@staticmethod
|
||||
def validate_content_type(request: Request) -> None:
|
||||
"""Validate content type for JSON requests"""
|
||||
if request.method in ['POST', 'PUT', 'PATCH']:
|
||||
content_type = request.headers.get('content-type', '')
|
||||
if 'application/json' in content_type:
|
||||
try:
|
||||
# Validate JSON size
|
||||
if hasattr(request, 'body') and len(request.body) > SecurityConfig.MAX_JSON_SIZE:
|
||||
raise BadRequest("JSON payload too large")
|
||||
except Exception:
|
||||
raise BadRequest("Invalid JSON payload")
|
||||
|
||||
@staticmethod
|
||||
def check_origin(request: Request) -> bool:
|
||||
"""Check if request origin is allowed"""
|
||||
origin = request.headers.get('origin')
|
||||
if not origin:
|
||||
return True # Allow requests without origin (direct API calls)
|
||||
|
||||
return any(
|
||||
origin.startswith(allowed_origin.rstrip('/*'))
|
||||
for allowed_origin in SecurityConfig.CORS_ORIGINS
|
||||
)
|
||||
|
||||
|
||||
class RateLimitMiddleware:
|
||||
"""Rate limiting middleware using Redis"""
|
||||
|
||||
def __init__(self):
|
||||
self.cache = None
|
||||
|
||||
async def get_cache(self):
|
||||
"""Get cache instance"""
|
||||
if not self.cache:
|
||||
self.cache = await get_cache()
|
||||
return self.cache
|
||||
|
||||
async def check_rate_limit(
|
||||
self,
|
||||
request: Request,
|
||||
identifier: str,
|
||||
pattern: str = "api"
|
||||
) -> bool:
|
||||
"""Check rate limit for identifier"""
|
||||
try:
|
||||
cache = await self.get_cache()
|
||||
limits = SecurityConfig.RATE_LIMIT_PATTERNS.get(pattern, {
|
||||
"requests": settings.RATE_LIMIT_REQUESTS,
|
||||
"window": settings.RATE_LIMIT_WINDOW
|
||||
})
|
||||
|
||||
cache_key = CACHE_KEYS["rate_limit"].format(
|
||||
pattern=pattern,
|
||||
identifier=identifier
|
||||
)
|
||||
|
||||
# Get current count
|
||||
current_count = await cache.get(cache_key)
|
||||
if current_count is None:
|
||||
# First request in window
|
||||
await cache.set(cache_key, "1", ttl=limits["window"])
|
||||
return True
|
||||
|
||||
current_count = int(current_count)
|
||||
if current_count >= limits["requests"]:
|
||||
# Rate limit exceeded
|
||||
logger.warning(
|
||||
"Rate limit exceeded",
|
||||
identifier=identifier,
|
||||
pattern=pattern,
|
||||
count=current_count,
|
||||
limit=limits["requests"]
|
||||
)
|
||||
return False
|
||||
|
||||
# Increment counter
|
||||
await cache.incr(cache_key)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Rate limit check failed", error=str(e))
|
||||
return True # Allow request if rate limiting fails
|
||||
|
||||
async def get_rate_limit_info(
|
||||
self,
|
||||
identifier: str,
|
||||
pattern: str = "api"
|
||||
) -> Dict[str, Any]:
|
||||
"""Get rate limit information"""
|
||||
try:
|
||||
cache = await self.get_cache()
|
||||
limits = SecurityConfig.RATE_LIMIT_PATTERNS.get(pattern, {
|
||||
"requests": settings.RATE_LIMIT_REQUESTS,
|
||||
"window": settings.RATE_LIMIT_WINDOW
|
||||
})
|
||||
|
||||
cache_key = CACHE_KEYS["rate_limit"].format(
|
||||
pattern=pattern,
|
||||
identifier=identifier
|
||||
)
|
||||
|
||||
current_count = await cache.get(cache_key) or "0"
|
||||
ttl = await cache.redis.ttl(cache_key)
|
||||
|
||||
return {
|
||||
"limit": limits["requests"],
|
||||
"remaining": max(0, limits["requests"] - int(current_count)),
|
||||
"reset_time": int(time.time()) + max(0, ttl),
|
||||
"window": limits["window"]
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to get rate limit info", error=str(e))
|
||||
return {}
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
"""Authentication middleware for API access"""
|
||||
|
||||
@staticmethod
|
||||
async def extract_token(request: Request) -> Optional[str]:
|
||||
"""Extract authentication token from request"""
|
||||
# Check Authorization header
|
||||
auth_header = request.headers.get('authorization')
|
||||
if auth_header and auth_header.startswith('Bearer '):
|
||||
return auth_header[7:] # Remove 'Bearer ' prefix
|
||||
|
||||
# Check X-API-Key header
|
||||
api_key = request.headers.get('x-api-key')
|
||||
if api_key:
|
||||
return api_key
|
||||
|
||||
# Check query parameter (less secure, for backward compatibility)
|
||||
return request.args.get('token')
|
||||
|
||||
@staticmethod
|
||||
async def validate_token(token: str, session) -> Optional[User]:
|
||||
"""Validate authentication token and return user"""
|
||||
if not token:
|
||||
return None
|
||||
|
||||
try:
|
||||
# For now, implement simple token validation
|
||||
# In production, implement JWT or database token validation
|
||||
|
||||
# Example: if token format is user_id:hash
|
||||
if ':' in token:
|
||||
user_id_str, token_hash = token.split(':', 1)
|
||||
try:
|
||||
user_id = uuid.UUID(user_id_str)
|
||||
user = await User.get_by_id(session, user_id)
|
||||
if user and user.verify_token(token_hash): # Implement in User model
|
||||
return user
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
# Fallback: try to find user by API token
|
||||
# This would require implementing token storage in User model
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Token validation failed", token=token[:8] + "...", error=str(e))
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def check_permissions(user: User, request: Request) -> bool:
|
||||
"""Check if user has required permissions for the endpoint"""
|
||||
# Implement permission checking based on endpoint and user role
|
||||
endpoint = request.path
|
||||
method = request.method
|
||||
|
||||
# Admin endpoints
|
||||
if '/admin/' in endpoint:
|
||||
return user.is_admin
|
||||
|
||||
# Moderator endpoints
|
||||
if '/mod/' in endpoint:
|
||||
return user.is_moderator
|
||||
|
||||
# User-specific endpoints
|
||||
if '/user/' in endpoint and method in ['POST', 'PUT', 'DELETE']:
|
||||
return user.has_permission('user:write')
|
||||
|
||||
# Content upload endpoints
|
||||
if '/upload' in endpoint or '/content' in endpoint and method == 'POST':
|
||||
return user.can_upload_content()
|
||||
|
||||
# Default: allow read access for authenticated users
|
||||
return True
|
||||
|
||||
|
||||
class RequestContextMiddleware:
|
||||
"""Request context middleware for tracking and logging"""
|
||||
|
||||
@staticmethod
|
||||
def generate_request_id() -> str:
|
||||
"""Generate unique request ID"""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
@staticmethod
|
||||
async def add_request_context(request: Request) -> None:
|
||||
"""Add request context for logging and tracking"""
|
||||
# Generate and set request ID
|
||||
request_id = RequestContextMiddleware.generate_request_id()
|
||||
request.ctx.request_id = request_id
|
||||
request_id_var.set(request_id)
|
||||
|
||||
# Set request start time
|
||||
request.ctx.start_time = time.time()
|
||||
|
||||
# Extract client information
|
||||
request.ctx.client_ip = RequestContextMiddleware.get_client_ip(request)
|
||||
request.ctx.user_agent = request.headers.get('user-agent', 'Unknown')
|
||||
|
||||
# Initialize context
|
||||
request.ctx.user = None
|
||||
request.ctx.rate_limit_info = {}
|
||||
|
||||
logger.info(
|
||||
"Request started",
|
||||
method=request.method,
|
||||
path=request.path,
|
||||
client_ip=request.ctx.client_ip,
|
||||
user_agent=request.ctx.user_agent
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_client_ip(request: Request) -> str:
|
||||
"""Get real client IP address"""
|
||||
# Check for forwarded headers
|
||||
forwarded_for = request.headers.get('x-forwarded-for')
|
||||
if forwarded_for:
|
||||
return forwarded_for.split(',')[0].strip()
|
||||
|
||||
real_ip = request.headers.get('x-real-ip')
|
||||
if real_ip:
|
||||
return real_ip
|
||||
|
||||
# Fallback to request IP
|
||||
return getattr(request, 'ip', '127.0.0.1')
|
||||
|
||||
@staticmethod
|
||||
async def log_request_completion(request: Request, response: HTTPResponse) -> None:
|
||||
"""Log request completion with metrics"""
|
||||
duration = time.time() - getattr(request.ctx, 'start_time', time.time())
|
||||
|
||||
logger.info(
|
||||
"Request completed",
|
||||
method=request.method,
|
||||
path=request.path,
|
||||
status_code=response.status,
|
||||
duration_ms=round(duration * 1000, 2),
|
||||
response_size=len(response.body) if response.body else 0,
|
||||
client_ip=getattr(request.ctx, 'client_ip', 'unknown'),
|
||||
user_id=str(request.ctx.user.id) if request.ctx.user else None
|
||||
)
|
||||
|
||||
|
||||
# Initialize middleware instances
|
||||
security_middleware = SecurityMiddleware()
|
||||
rate_limit_middleware = RateLimitMiddleware()
|
||||
auth_middleware = AuthenticationMiddleware()
|
||||
context_middleware = RequestContextMiddleware()
|
||||
|
||||
|
||||
async def request_middleware(request: Request):
|
||||
"""Main request middleware pipeline"""
|
||||
|
||||
# Handle OPTIONS requests for CORS
|
||||
if request.method == 'OPTIONS':
|
||||
response = text_response('OK')
|
||||
return security_middleware.add_security_headers(response)
|
||||
|
||||
# Add request context
|
||||
await context_middleware.add_request_context(request)
|
||||
|
||||
# Security validations
|
||||
try:
|
||||
security_middleware.validate_request_size(request)
|
||||
security_middleware.validate_content_type(request)
|
||||
|
||||
if not security_middleware.check_origin(request):
|
||||
raise Forbidden("Origin not allowed")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Security validation failed", error=str(e))
|
||||
response = json_response({"error": str(e)}, status=400)
|
||||
return security_middleware.add_security_headers(response)
|
||||
|
||||
# Rate limiting
|
||||
if settings.RATE_LIMIT_ENABLED:
|
||||
client_identifier = context_middleware.get_client_ip(request)
|
||||
pattern = "api"
|
||||
|
||||
# Determine rate limit pattern based on endpoint
|
||||
if '/auth/' in request.path:
|
||||
pattern = "auth"
|
||||
elif '/upload' in request.path:
|
||||
pattern = "upload"
|
||||
elif '/admin/' in request.path:
|
||||
pattern = "heavy"
|
||||
|
||||
if not await rate_limit_middleware.check_rate_limit(request, client_identifier, pattern):
|
||||
rate_info = await rate_limit_middleware.get_rate_limit_info(client_identifier, pattern)
|
||||
response = json_response(
|
||||
{
|
||||
"error": "Rate limit exceeded",
|
||||
"rate_limit": rate_info
|
||||
},
|
||||
status=429
|
||||
)
|
||||
return security_middleware.add_security_headers(response)
|
||||
|
||||
# Store rate limit info for response headers
|
||||
request.ctx.rate_limit_info = await rate_limit_middleware.get_rate_limit_info(
|
||||
client_identifier, pattern
|
||||
)
|
||||
|
||||
# Authentication (for protected endpoints)
|
||||
if not request.path.startswith('/api/system') and request.path != '/':
|
||||
async with get_db_session() as session:
|
||||
token = await auth_middleware.extract_token(request)
|
||||
if token:
|
||||
user = await auth_middleware.validate_token(token, session)
|
||||
if user:
|
||||
request.ctx.user = user
|
||||
user_id_var.set(str(user.id))
|
||||
|
||||
# Check permissions
|
||||
if not await auth_middleware.check_permissions(user, request):
|
||||
response = json_response({"error": "Insufficient permissions"}, status=403)
|
||||
return security_middleware.add_security_headers(response)
|
||||
|
||||
# Update user activity
|
||||
user.update_activity()
|
||||
await session.commit()
|
||||
|
||||
# Store session for request handlers
|
||||
request.ctx.db_session = session
|
||||
|
||||
|
||||
async def response_middleware(request: Request, response: HTTPResponse):
|
||||
"""Main response middleware pipeline"""
|
||||
|
||||
# Add security headers
|
||||
response = security_middleware.add_security_headers(response)
|
||||
|
||||
# Add rate limit headers
|
||||
if hasattr(request.ctx, 'rate_limit_info') and request.ctx.rate_limit_info:
|
||||
rate_info = request.ctx.rate_limit_info
|
||||
response.headers.update({
|
||||
"X-RateLimit-Limit": str(rate_info.get('limit', 0)),
|
||||
"X-RateLimit-Remaining": str(rate_info.get('remaining', 0)),
|
||||
"X-RateLimit-Reset": str(rate_info.get('reset_time', 0))
|
||||
})
|
||||
|
||||
# Add request ID to response
|
||||
if hasattr(request.ctx, 'request_id'):
|
||||
response.headers["X-Request-ID"] = request.ctx.request_id
|
||||
|
||||
# Log request completion
|
||||
await context_middleware.log_request_completion(request, response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def try_authorization(request):
|
||||
token = request.headers.get("Authorization")
|
||||
if not token:
|
||||
return
|
||||
|
||||
token_bin = b58decode(token)
|
||||
if len(token_bin) != 57:
|
||||
make_log("auth", "Invalid token length", level="warning")
|
||||
return
|
||||
|
||||
known_key = request.ctx.db_session.query(KnownKey).filter(KnownKey.seed == token).first()
|
||||
if not known_key:
|
||||
make_log("auth", "Unknown key", level="warning")
|
||||
return
|
||||
|
||||
if known_key.type != "USER_API_V1":
|
||||
make_log("auth", "Invalid key type", level="warning")
|
||||
return
|
||||
|
||||
(
|
||||
token_version,
|
||||
user_id,
|
||||
timestamp,
|
||||
randpart
|
||||
) = (
|
||||
int.from_bytes(token_bin[0:1], 'big'),
|
||||
int.from_bytes(token_bin[1:17], 'big'),
|
||||
int.from_bytes(token_bin[17:25], 'big'),
|
||||
token_bin[25:]
|
||||
async def exception_middleware(request: Request, exception: Exception):
|
||||
"""Global exception handling middleware"""
|
||||
|
||||
error_id = str(uuid.uuid4())
|
||||
|
||||
# Log the exception
|
||||
logger.error(
|
||||
"Unhandled exception",
|
||||
error_id=error_id,
|
||||
exception_type=type(exception).__name__,
|
||||
exception_message=str(exception),
|
||||
path=request.path,
|
||||
method=request.method,
|
||||
user_id=str(request.ctx.user.id) if hasattr(request.ctx, 'user') and request.ctx.user else None
|
||||
)
|
||||
assert token_version == 1, "Invalid token version"
|
||||
assert user_id > 0, "Invalid user_id"
|
||||
assert timestamp > 0, "Invalid timestamp"
|
||||
|
||||
if known_key.meta.get('I_user_id', -1) != user_id:
|
||||
make_log("auth", f"User ID mismatch: {known_key.meta.get('I_user_id', -1)} != {user_id}", level="warning")
|
||||
return
|
||||
|
||||
user = request.ctx.db_session.query(User).filter(User.id == known_key.meta['I_user_id']).first()
|
||||
if not user:
|
||||
make_log("auth", "No user from key", level="warning")
|
||||
return
|
||||
|
||||
request.ctx.user = user
|
||||
request.ctx.user_key = known_key
|
||||
|
||||
request.ctx.user_uploader_wrapper = Wrapped_CBotChat(request.app.ctx.memory._telegram_bot, chat_id=user.telegram_id, db_session=request.ctx.db_session, user=user)
|
||||
request.ctx.user_client_wrapper = Wrapped_CBotChat(request.app.ctx.memory._client_telegram_bot, chat_id=user.telegram_id, db_session=request.ctx.db_session, user=user)
|
||||
|
||||
# Handle different exception types
|
||||
if isinstance(exception, Unauthorized):
|
||||
response_data = {"error": "Authentication required", "error_id": error_id}
|
||||
status = 401
|
||||
elif isinstance(exception, Forbidden):
|
||||
response_data = {"error": "Access forbidden", "error_id": error_id}
|
||||
status = 403
|
||||
elif isinstance(exception, TooManyRequests):
|
||||
response_data = {"error": "Rate limit exceeded", "error_id": error_id}
|
||||
status = 429
|
||||
elif isinstance(exception, BadRequest):
|
||||
response_data = {"error": str(exception), "error_id": error_id}
|
||||
status = 400
|
||||
else:
|
||||
# Generic server error
|
||||
response_data = {
|
||||
"error": "Internal server error",
|
||||
"error_id": error_id
|
||||
}
|
||||
status = 500
|
||||
|
||||
if settings.DEBUG:
|
||||
response_data["debug"] = {
|
||||
"type": type(exception).__name__,
|
||||
"message": str(exception)
|
||||
}
|
||||
|
||||
response = json_response(response_data, status=status)
|
||||
return security_middleware.add_security_headers(response)
|
||||
|
||||
|
||||
async def try_service_authorization(request):
|
||||
signature = request.headers.get('X-Service-Signature')
|
||||
if not signature:
|
||||
return
|
||||
|
||||
# TODO: смысл этой проверки если это можно подменить?
|
||||
message_hash_b58 = request.headers.get('X-Message-Hash')
|
||||
if not message_hash_b58:
|
||||
return
|
||||
|
||||
message_hash = b58decode(message_hash_b58)
|
||||
signer = Signer(hot_seed)
|
||||
if signer.verify(message_hash, signature):
|
||||
request.ctx.verified_hash = message_hash
|
||||
|
||||
|
||||
async def save_activity(request):
|
||||
activity_meta = {}
|
||||
try:
|
||||
activity_meta["path"] = request.path
|
||||
if 'system' in activity_meta["path"]:
|
||||
return
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
activity_meta["args"] = dict(request.args)
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
activity_meta["json"] = dict(request.json)
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
activity_meta["method"] = request.method
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
activity_meta["ip"] = (request.headers['X-Forwarded-for'] if 'X-Forwarded-for' in request.headers else None) \
|
||||
or request.remote_addr or request.ip
|
||||
activity_meta["ip"] = activity_meta["ip"].split(",")[0].strip()
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
activity_meta["headers"] = dict(request.headers)
|
||||
except:
|
||||
pass
|
||||
|
||||
new_user_activity = UserActivity(
|
||||
type="API_V1_REQUEST",
|
||||
meta=activity_meta,
|
||||
user_id=request.ctx.user.id if request.ctx.user else None,
|
||||
user_ip=activity_meta.get("ip", "0.0.0.0"),
|
||||
created=datetime.now()
|
||||
)
|
||||
request.ctx.db_session.add(new_user_activity)
|
||||
request.ctx.db_session.commit()
|
||||
|
||||
|
||||
async def attach_user_to_request(request):
|
||||
if request.method == 'OPTIONS':
|
||||
return attach_headers(sanic_response.text("OK"))
|
||||
|
||||
request.ctx.db_session = Session()
|
||||
request.ctx.verified_hash = None
|
||||
request.ctx.user = None
|
||||
request.ctx.user_key = None
|
||||
request.ctx.user_uploader_wrapper = Wrapped_CBotChat(request.app.ctx.memory._telegram_bot, db_session=request.ctx.db_session)
|
||||
request.ctx.user_client_wrapper = Wrapped_CBotChat(request.app.ctx.memory._client_telegram_bot, db_session=request.ctx.db_session)
|
||||
await try_authorization(request)
|
||||
await save_activity(request)
|
||||
await try_service_authorization(request)
|
||||
|
||||
|
||||
async def close_request_handler(request, response):
|
||||
if request.method == 'OPTIONS':
|
||||
response = sanic_response.text("OK")
|
||||
|
||||
try:
|
||||
request.ctx.db_session.close()
|
||||
except BaseException as e:
|
||||
pass
|
||||
|
||||
response = attach_headers(response)
|
||||
|
||||
return request, response
|
||||
|
||||
|
||||
async def close_db_session(request, response):
|
||||
request, response = await close_request_handler(request, response)
|
||||
response = attach_headers(response)
|
||||
return response
|
||||
# Maintenance mode middleware
|
||||
async def maintenance_middleware(request: Request):
|
||||
"""Check for maintenance mode"""
|
||||
if settings.MAINTENANCE_MODE and not request.path.startswith('/api/system'):
|
||||
response = json_response({
|
||||
"error": "Service temporarily unavailable",
|
||||
"message": settings.MAINTENANCE_MESSAGE
|
||||
}, status=503)
|
||||
return security_middleware.add_security_headers(response)
|
||||
@@ -0,0 +1,870 @@
|
||||
"""
|
||||
Authentication and authorization routes with JWT tokens, user management, and security features.
|
||||
Provides user registration, login, token refresh, and account management with comprehensive validation.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sanic import Blueprint, Request, response
|
||||
from sanic.response import JSONResponse
|
||||
from sqlalchemy import select, update, and_
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session, get_cache_manager
|
||||
from app.core.logging import get_logger
|
||||
from app.core.models.user import User, UserSession, UserRole
|
||||
from app.core.security import (
|
||||
hash_password, verify_password, generate_access_token,
|
||||
verify_access_token, generate_refresh_token, generate_api_key,
|
||||
sanitize_input, generate_csrf_token
|
||||
)
|
||||
from app.api.middleware import require_auth, validate_request, rate_limit
|
||||
from app.core.validation import (
|
||||
UserRegistrationSchema, UserLoginSchema, UserUpdateSchema,
|
||||
ApiKeySchema
|
||||
)
|
||||
|
||||
# Initialize blueprint
|
||||
auth_bp = Blueprint("auth", url_prefix="/api/v1/auth")
|
||||
logger = get_logger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
@auth_bp.route("/register", methods=["POST"])
|
||||
@rate_limit(limit=5, window=3600) # 5 registrations per hour
|
||||
@validate_request(UserRegistrationSchema)
|
||||
async def register_user(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Register new user with comprehensive validation and security checks.
|
||||
|
||||
Args:
|
||||
request: Sanic request with user registration data
|
||||
|
||||
Returns:
|
||||
JSONResponse: Registration result with access tokens
|
||||
"""
|
||||
try:
|
||||
data = request.json
|
||||
client_ip = request.headers.get("X-Forwarded-For", request.remote_addr)
|
||||
|
||||
# Sanitize input data
|
||||
username = sanitize_input(data["username"])
|
||||
email = sanitize_input(data["email"])
|
||||
full_name = sanitize_input(data.get("full_name", ""))
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Check if username already exists
|
||||
username_stmt = select(User).where(User.username == username)
|
||||
username_result = await session.execute(username_stmt)
|
||||
if username_result.scalar_one_or_none():
|
||||
return response.json(
|
||||
{"error": "Username already exists", "code": "USERNAME_EXISTS"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Check if email already exists
|
||||
email_stmt = select(User).where(User.email == email)
|
||||
email_result = await session.execute(email_stmt)
|
||||
if email_result.scalar_one_or_none():
|
||||
return response.json(
|
||||
{"error": "Email already registered", "code": "EMAIL_EXISTS"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Check registration rate limiting by IP
|
||||
cache_manager = get_cache_manager()
|
||||
ip_reg_key = f"registration_ip:{client_ip}"
|
||||
ip_registrations = await cache_manager.get(ip_reg_key, default=0)
|
||||
|
||||
if ip_registrations >= 3: # Max 3 registrations per IP per day
|
||||
return response.json(
|
||||
{"error": "Too many registrations from this IP", "code": "IP_LIMIT_EXCEEDED"},
|
||||
status=429
|
||||
)
|
||||
|
||||
# Hash password
|
||||
password_hash = hash_password(data["password"])
|
||||
|
||||
# Create user
|
||||
new_user = User(
|
||||
id=uuid4(),
|
||||
username=username,
|
||||
email=email,
|
||||
password_hash=password_hash,
|
||||
full_name=full_name,
|
||||
is_active=True,
|
||||
email_verified=False, # Require email verification
|
||||
registration_ip=client_ip,
|
||||
last_login_ip=client_ip,
|
||||
settings={"theme": "light", "notifications": True}
|
||||
)
|
||||
|
||||
session.add(new_user)
|
||||
await session.commit()
|
||||
await session.refresh(new_user)
|
||||
|
||||
# Assign default role
|
||||
default_role_stmt = select(UserRole).where(UserRole.name == "user")
|
||||
role_result = await session.execute(default_role_stmt)
|
||||
default_role = role_result.scalar_one_or_none()
|
||||
|
||||
if default_role:
|
||||
new_user.roles.append(default_role)
|
||||
await session.commit()
|
||||
|
||||
# Update IP registration counter
|
||||
await cache_manager.increment(ip_reg_key, ttl=86400) # 24 hours
|
||||
|
||||
# Generate tokens
|
||||
access_token = generate_access_token(
|
||||
{"user_id": str(new_user.id), "username": username},
|
||||
expires_in=settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
|
||||
)
|
||||
|
||||
refresh_token = generate_refresh_token(new_user.id)
|
||||
|
||||
# Create user session
|
||||
session_id = str(uuid4())
|
||||
csrf_token = generate_csrf_token(new_user.id, session_id)
|
||||
|
||||
async with get_async_session() as session:
|
||||
user_session = UserSession(
|
||||
id=UUID(session_id),
|
||||
user_id=new_user.id,
|
||||
refresh_token_hash=hash_password(refresh_token[-32:]), # Hash last 32 chars
|
||||
ip_address=client_ip,
|
||||
user_agent=request.headers.get("User-Agent", ""),
|
||||
expires_at=datetime.utcnow() + timedelta(days=settings.REFRESH_TOKEN_EXPIRE_DAYS)
|
||||
)
|
||||
session.add(user_session)
|
||||
await session.commit()
|
||||
|
||||
await logger.ainfo(
|
||||
"User registered successfully",
|
||||
user_id=str(new_user.id),
|
||||
username=username,
|
||||
email=email,
|
||||
ip=client_ip
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"message": "Registration successful",
|
||||
"user": {
|
||||
"id": str(new_user.id),
|
||||
"username": username,
|
||||
"email": email,
|
||||
"full_name": full_name,
|
||||
"created_at": new_user.created_at.isoformat()
|
||||
},
|
||||
"tokens": {
|
||||
"access_token": access_token,
|
||||
"refresh_token": refresh_token,
|
||||
"token_type": "Bearer",
|
||||
"expires_in": settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
|
||||
},
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"csrf_token": csrf_token
|
||||
}
|
||||
}, status=201)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"User registration failed",
|
||||
username=data.get("username"),
|
||||
email=data.get("email"),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Registration failed", "code": "REGISTRATION_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@auth_bp.route("/login", methods=["POST"])
|
||||
@rate_limit(limit=10, window=900) # 10 login attempts per 15 minutes
|
||||
@validate_request(UserLoginSchema)
|
||||
async def login_user(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Authenticate user and generate access tokens with security logging.
|
||||
|
||||
Args:
|
||||
request: Sanic request with login credentials
|
||||
|
||||
Returns:
|
||||
JSONResponse: Authentication result with tokens
|
||||
"""
|
||||
try:
|
||||
data = request.json
|
||||
username_or_email = sanitize_input(data["username"])
|
||||
password = data["password"]
|
||||
remember_me = data.get("remember_me", False)
|
||||
client_ip = request.headers.get("X-Forwarded-For", request.remote_addr)
|
||||
|
||||
# Check login rate limiting
|
||||
cache_manager = get_cache_manager()
|
||||
login_key = f"login_attempts:{username_or_email}:{client_ip}"
|
||||
attempts = await cache_manager.get(login_key, default=0)
|
||||
|
||||
if attempts >= 5: # Max 5 failed attempts
|
||||
return response.json(
|
||||
{"error": "Too many login attempts", "code": "LOGIN_BLOCKED"},
|
||||
status=429
|
||||
)
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Find user by username or email
|
||||
user_stmt = select(User).where(
|
||||
or_(User.username == username_or_email, User.email == username_or_email)
|
||||
).options(selectinload(User.roles))
|
||||
|
||||
user_result = await session.execute(user_stmt)
|
||||
user = user_result.scalar_one_or_none()
|
||||
|
||||
if not user or not verify_password(password, user.password_hash):
|
||||
# Increment failed attempts
|
||||
await cache_manager.increment(login_key, ttl=900) # 15 minutes
|
||||
|
||||
await logger.awarning(
|
||||
"Failed login attempt",
|
||||
username=username_or_email,
|
||||
ip=client_ip,
|
||||
attempts=attempts + 1
|
||||
)
|
||||
|
||||
return response.json(
|
||||
{"error": "Invalid credentials", "code": "INVALID_CREDENTIALS"},
|
||||
status=401
|
||||
)
|
||||
|
||||
if not user.is_active:
|
||||
return response.json(
|
||||
{"error": "Account deactivated", "code": "ACCOUNT_DEACTIVATED"},
|
||||
status=403
|
||||
)
|
||||
|
||||
# Successful login - clear failed attempts
|
||||
await cache_manager.delete(login_key)
|
||||
|
||||
# Update user login info
|
||||
user.last_login_at = datetime.utcnow()
|
||||
user.last_login_ip = client_ip
|
||||
user.login_count = (user.login_count or 0) + 1
|
||||
|
||||
await session.commit()
|
||||
|
||||
# Generate tokens
|
||||
user_permissions = []
|
||||
for role in user.roles:
|
||||
user_permissions.extend(role.permissions)
|
||||
|
||||
token_payload = {
|
||||
"user_id": str(user.id),
|
||||
"username": user.username,
|
||||
"permissions": list(set(user_permissions)) # Remove duplicates
|
||||
}
|
||||
|
||||
expires_in = settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
|
||||
if remember_me:
|
||||
expires_in *= 24 # 24x longer for remember me
|
||||
|
||||
access_token = generate_access_token(token_payload, expires_in=expires_in)
|
||||
refresh_token = generate_refresh_token(user.id)
|
||||
|
||||
# Create user session
|
||||
session_id = str(uuid4())
|
||||
csrf_token = generate_csrf_token(user.id, session_id)
|
||||
|
||||
refresh_expires = timedelta(days=settings.REFRESH_TOKEN_EXPIRE_DAYS)
|
||||
if remember_me:
|
||||
refresh_expires *= 2 # Longer refresh for remember me
|
||||
|
||||
async with get_async_session() as session:
|
||||
user_session = UserSession(
|
||||
id=UUID(session_id),
|
||||
user_id=user.id,
|
||||
refresh_token_hash=hash_password(refresh_token[-32:]),
|
||||
ip_address=client_ip,
|
||||
user_agent=request.headers.get("User-Agent", ""),
|
||||
expires_at=datetime.utcnow() + refresh_expires,
|
||||
remember_me=remember_me
|
||||
)
|
||||
session.add(user_session)
|
||||
await session.commit()
|
||||
|
||||
await logger.ainfo(
|
||||
"User logged in successfully",
|
||||
user_id=str(user.id),
|
||||
username=user.username,
|
||||
ip=client_ip,
|
||||
remember_me=remember_me
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"message": "Login successful",
|
||||
"user": {
|
||||
"id": str(user.id),
|
||||
"username": user.username,
|
||||
"email": user.email,
|
||||
"full_name": user.full_name,
|
||||
"last_login": user.last_login_at.isoformat() if user.last_login_at else None,
|
||||
"permissions": user_permissions
|
||||
},
|
||||
"tokens": {
|
||||
"access_token": access_token,
|
||||
"refresh_token": refresh_token,
|
||||
"token_type": "Bearer",
|
||||
"expires_in": expires_in
|
||||
},
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"csrf_token": csrf_token
|
||||
}
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Login failed",
|
||||
username=data.get("username"),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Login failed", "code": "LOGIN_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@auth_bp.route("/refresh", methods=["POST"])
|
||||
@rate_limit(limit=50, window=3600) # 50 refresh attempts per hour
|
||||
async def refresh_tokens(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Refresh access token using refresh token with rotation.
|
||||
|
||||
Args:
|
||||
request: Sanic request with refresh token
|
||||
|
||||
Returns:
|
||||
JSONResponse: New access and refresh tokens
|
||||
"""
|
||||
try:
|
||||
refresh_token = request.json.get("refresh_token")
|
||||
if not refresh_token:
|
||||
return response.json(
|
||||
{"error": "Refresh token required", "code": "TOKEN_REQUIRED"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Verify refresh token
|
||||
payload = verify_access_token(refresh_token, token_type="refresh")
|
||||
if not payload:
|
||||
return response.json(
|
||||
{"error": "Invalid refresh token", "code": "INVALID_TOKEN"},
|
||||
status=401
|
||||
)
|
||||
|
||||
user_id = UUID(payload["user_id"])
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Verify session exists and is valid
|
||||
session_stmt = select(UserSession).where(
|
||||
and_(
|
||||
UserSession.user_id == user_id,
|
||||
UserSession.refresh_token_hash == hash_password(refresh_token[-32:]),
|
||||
UserSession.expires_at > datetime.utcnow(),
|
||||
UserSession.is_active == True
|
||||
)
|
||||
)
|
||||
session_result = await session.execute(session_stmt)
|
||||
user_session = session_result.scalar_one_or_none()
|
||||
|
||||
if not user_session:
|
||||
return response.json(
|
||||
{"error": "Session expired or invalid", "code": "SESSION_INVALID"},
|
||||
status=401
|
||||
)
|
||||
|
||||
# Get user with permissions
|
||||
user_stmt = select(User).where(User.id == user_id).options(selectinload(User.roles))
|
||||
user_result = await session.execute(user_stmt)
|
||||
user = user_result.scalar_one_or_none()
|
||||
|
||||
if not user or not user.is_active:
|
||||
return response.json(
|
||||
{"error": "User not found or inactive", "code": "USER_INACTIVE"},
|
||||
status=401
|
||||
)
|
||||
|
||||
# Generate new tokens (token rotation)
|
||||
user_permissions = []
|
||||
for role in user.roles:
|
||||
user_permissions.extend(role.permissions)
|
||||
|
||||
new_access_token = generate_access_token(
|
||||
{
|
||||
"user_id": str(user.id),
|
||||
"username": user.username,
|
||||
"permissions": list(set(user_permissions))
|
||||
},
|
||||
expires_in=settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
|
||||
)
|
||||
|
||||
new_refresh_token = generate_refresh_token(user.id)
|
||||
|
||||
# Update session with new refresh token
|
||||
user_session.refresh_token_hash = hash_password(new_refresh_token[-32:])
|
||||
user_session.last_used_at = datetime.utcnow()
|
||||
|
||||
await session.commit()
|
||||
|
||||
await logger.adebug(
|
||||
"Tokens refreshed",
|
||||
user_id=str(user_id),
|
||||
session_id=str(user_session.id)
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"tokens": {
|
||||
"access_token": new_access_token,
|
||||
"refresh_token": new_refresh_token,
|
||||
"token_type": "Bearer",
|
||||
"expires_in": settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
|
||||
}
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Token refresh failed", error=str(e))
|
||||
return response.json(
|
||||
{"error": "Token refresh failed", "code": "REFRESH_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@auth_bp.route("/logout", methods=["POST"])
|
||||
@require_auth()
|
||||
async def logout_user(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Logout user and invalidate session.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: Logout confirmation
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
session_id = request.headers.get("X-Session-ID")
|
||||
|
||||
if session_id:
|
||||
async with get_async_session() as session:
|
||||
# Invalidate specific session
|
||||
session_stmt = select(UserSession).where(
|
||||
and_(
|
||||
UserSession.id == UUID(session_id),
|
||||
UserSession.user_id == user_id
|
||||
)
|
||||
)
|
||||
session_result = await session.execute(session_stmt)
|
||||
user_session = session_result.scalar_one_or_none()
|
||||
|
||||
if user_session:
|
||||
user_session.is_active = False
|
||||
user_session.logged_out_at = datetime.utcnow()
|
||||
await session.commit()
|
||||
|
||||
await logger.ainfo(
|
||||
"User logged out",
|
||||
user_id=str(user_id),
|
||||
session_id=session_id
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"message": "Logout successful",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Logout failed",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Logout failed", "code": "LOGOUT_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@auth_bp.route("/me", methods=["GET"])
|
||||
@require_auth()
|
||||
async def get_current_user(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Get current user information and permissions.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: Current user data
|
||||
"""
|
||||
try:
|
||||
user = request.ctx.user
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Get user with full details
|
||||
user_stmt = select(User).where(User.id == user.id).options(
|
||||
selectinload(User.roles),
|
||||
selectinload(User.api_keys)
|
||||
)
|
||||
user_result = await session.execute(user_stmt)
|
||||
full_user = user_result.scalar_one_or_none()
|
||||
|
||||
if not full_user:
|
||||
return response.json(
|
||||
{"error": "User not found", "code": "USER_NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
# Get user permissions
|
||||
permissions = []
|
||||
roles = []
|
||||
for role in full_user.roles:
|
||||
roles.append({
|
||||
"name": role.name,
|
||||
"description": role.description
|
||||
})
|
||||
permissions.extend(role.permissions)
|
||||
|
||||
# Get active sessions
|
||||
sessions_stmt = select(UserSession).where(
|
||||
and_(
|
||||
UserSession.user_id == user.id,
|
||||
UserSession.is_active == True,
|
||||
UserSession.expires_at > datetime.utcnow()
|
||||
)
|
||||
)
|
||||
sessions_result = await session.execute(sessions_stmt)
|
||||
active_sessions = sessions_result.scalars().all()
|
||||
|
||||
return response.json({
|
||||
"user": {
|
||||
"id": str(full_user.id),
|
||||
"username": full_user.username,
|
||||
"email": full_user.email,
|
||||
"full_name": full_user.full_name,
|
||||
"bio": full_user.bio,
|
||||
"avatar_url": full_user.avatar_url,
|
||||
"is_active": full_user.is_active,
|
||||
"email_verified": full_user.email_verified,
|
||||
"created_at": full_user.created_at.isoformat(),
|
||||
"last_login_at": full_user.last_login_at.isoformat() if full_user.last_login_at else None,
|
||||
"login_count": full_user.login_count,
|
||||
"settings": full_user.settings
|
||||
},
|
||||
"roles": roles,
|
||||
"permissions": list(set(permissions)),
|
||||
"active_sessions": len(active_sessions),
|
||||
"api_keys": [
|
||||
{
|
||||
"id": str(key.id),
|
||||
"name": key.name,
|
||||
"created_at": key.created_at.isoformat(),
|
||||
"last_used_at": key.last_used_at.isoformat() if key.last_used_at else None,
|
||||
"expires_at": key.expires_at.isoformat() if key.expires_at else None
|
||||
}
|
||||
for key in full_user.api_keys
|
||||
if key.is_active
|
||||
]
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get current user",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to get user information", "code": "USER_INFO_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@auth_bp.route("/me", methods=["PUT"])
|
||||
@require_auth()
|
||||
@validate_request(UserUpdateSchema)
|
||||
async def update_current_user(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Update current user profile information.
|
||||
|
||||
Args:
|
||||
request: Sanic request with update data
|
||||
|
||||
Returns:
|
||||
JSONResponse: Updated user information
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
data = request.json
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Get current user
|
||||
user_stmt = select(User).where(User.id == user_id)
|
||||
user_result = await session.execute(user_stmt)
|
||||
user = user_result.scalar_one_or_none()
|
||||
|
||||
if not user:
|
||||
return response.json(
|
||||
{"error": "User not found", "code": "USER_NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
# Update allowed fields
|
||||
updatable_fields = ["full_name", "bio", "avatar_url", "settings"]
|
||||
for field in updatable_fields:
|
||||
if field in data:
|
||||
if field == "full_name":
|
||||
setattr(user, field, sanitize_input(data[field]))
|
||||
elif field == "bio":
|
||||
setattr(user, field, sanitize_input(data[field], max_length=500))
|
||||
else:
|
||||
setattr(user, field, data[field])
|
||||
|
||||
# Handle email change (requires verification)
|
||||
if "email" in data and data["email"] != user.email:
|
||||
new_email = sanitize_input(data["email"])
|
||||
|
||||
# Check if email is already taken
|
||||
email_stmt = select(User).where(
|
||||
and_(User.email == new_email, User.id != user_id)
|
||||
)
|
||||
email_result = await session.execute(email_stmt)
|
||||
if email_result.scalar_one_or_none():
|
||||
return response.json(
|
||||
{"error": "Email already in use", "code": "EMAIL_IN_USE"},
|
||||
status=400
|
||||
)
|
||||
|
||||
user.email = new_email
|
||||
user.email_verified = False # Require re-verification
|
||||
|
||||
user.updated_at = datetime.utcnow()
|
||||
await session.commit()
|
||||
|
||||
await logger.ainfo(
|
||||
"User profile updated",
|
||||
user_id=str(user_id),
|
||||
updated_fields=list(data.keys())
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"message": "Profile updated successfully",
|
||||
"user": {
|
||||
"id": str(user.id),
|
||||
"username": user.username,
|
||||
"email": user.email,
|
||||
"full_name": user.full_name,
|
||||
"bio": user.bio,
|
||||
"avatar_url": user.avatar_url,
|
||||
"updated_at": user.updated_at.isoformat()
|
||||
}
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to update user profile",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to update profile", "code": "UPDATE_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@auth_bp.route("/api-keys", methods=["POST"])
|
||||
@rate_limit(limit=5, window=3600) # 5 API keys per hour
|
||||
@require_auth(permissions=["api.create"])
|
||||
@validate_request(ApiKeySchema)
|
||||
async def create_api_key(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Create new API key for programmatic access.
|
||||
|
||||
Args:
|
||||
request: Sanic request with API key data
|
||||
|
||||
Returns:
|
||||
JSONResponse: Created API key information
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
data = request.json
|
||||
|
||||
# Generate API key
|
||||
api_key = generate_api_key(
|
||||
user_id=user_id,
|
||||
permissions=data["permissions"],
|
||||
name=data["name"],
|
||||
expires_in=None if not data.get("expires_at") else
|
||||
int((datetime.fromisoformat(data["expires_at"]) - datetime.utcnow()).total_seconds())
|
||||
)
|
||||
|
||||
async with get_async_session() as session:
|
||||
from app.core.models.user import ApiKey
|
||||
|
||||
# Create API key record
|
||||
new_api_key = ApiKey(
|
||||
id=uuid4(),
|
||||
user_id=user_id,
|
||||
name=sanitize_input(data["name"]),
|
||||
key_hash=hash_password(api_key[-32:]), # Hash last 32 chars
|
||||
permissions=data["permissions"],
|
||||
expires_at=datetime.fromisoformat(data["expires_at"]) if data.get("expires_at") else None
|
||||
)
|
||||
|
||||
session.add(new_api_key)
|
||||
await session.commit()
|
||||
await session.refresh(new_api_key)
|
||||
|
||||
await logger.ainfo(
|
||||
"API key created",
|
||||
user_id=str(user_id),
|
||||
api_key_id=str(new_api_key.id),
|
||||
name=data["name"],
|
||||
permissions=data["permissions"]
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"message": "API key created successfully",
|
||||
"api_key": {
|
||||
"id": str(new_api_key.id),
|
||||
"name": new_api_key.name,
|
||||
"key": api_key, # Only returned once
|
||||
"permissions": new_api_key.permissions,
|
||||
"created_at": new_api_key.created_at.isoformat(),
|
||||
"expires_at": new_api_key.expires_at.isoformat() if new_api_key.expires_at else None
|
||||
},
|
||||
"warning": "Save this API key securely. It will not be shown again."
|
||||
}, status=201)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to create API key",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to create API key", "code": "API_KEY_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@auth_bp.route("/sessions", methods=["GET"])
|
||||
@require_auth()
|
||||
async def get_user_sessions(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Get all active user sessions.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: List of active sessions
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
async with get_async_session() as session:
|
||||
sessions_stmt = select(UserSession).where(
|
||||
and_(
|
||||
UserSession.user_id == user_id,
|
||||
UserSession.is_active == True,
|
||||
UserSession.expires_at > datetime.utcnow()
|
||||
)
|
||||
).order_by(UserSession.created_at.desc())
|
||||
|
||||
sessions_result = await session.execute(sessions_stmt)
|
||||
sessions = sessions_result.scalars().all()
|
||||
|
||||
sessions_data = []
|
||||
for sess in sessions:
|
||||
sessions_data.append({
|
||||
"id": str(sess.id),
|
||||
"ip_address": sess.ip_address,
|
||||
"user_agent": sess.user_agent,
|
||||
"created_at": sess.created_at.isoformat(),
|
||||
"last_used_at": sess.last_used_at.isoformat() if sess.last_used_at else None,
|
||||
"expires_at": sess.expires_at.isoformat(),
|
||||
"remember_me": sess.remember_me,
|
||||
"is_current": str(sess.id) == request.headers.get("X-Session-ID")
|
||||
})
|
||||
|
||||
return response.json({
|
||||
"sessions": sessions_data,
|
||||
"total": len(sessions_data)
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get user sessions",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to get sessions", "code": "SESSIONS_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@auth_bp.route("/sessions/<session_id:uuid>", methods=["DELETE"])
|
||||
@require_auth()
|
||||
async def revoke_session(request: Request, session_id: UUID) -> JSONResponse:
|
||||
"""
|
||||
Revoke specific user session.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
session_id: Session UUID to revoke
|
||||
|
||||
Returns:
|
||||
JSONResponse: Revocation status
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
async with get_async_session() as session:
|
||||
session_stmt = select(UserSession).where(
|
||||
and_(
|
||||
UserSession.id == session_id,
|
||||
UserSession.user_id == user_id
|
||||
)
|
||||
)
|
||||
session_result = await session.execute(session_stmt)
|
||||
user_session = session_result.scalar_one_or_none()
|
||||
|
||||
if not user_session:
|
||||
return response.json(
|
||||
{"error": "Session not found", "code": "SESSION_NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
user_session.is_active = False
|
||||
user_session.logged_out_at = datetime.utcnow()
|
||||
await session.commit()
|
||||
|
||||
await logger.ainfo(
|
||||
"Session revoked",
|
||||
user_id=str(user_id),
|
||||
session_id=str(session_id)
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"message": "Session revoked successfully",
|
||||
"session_id": str(session_id)
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to revoke session",
|
||||
user_id=str(request.ctx.user.id),
|
||||
session_id=str(session_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to revoke session", "code": "REVOKE_FAILED"},
|
||||
status=500
|
||||
)
|
||||
@@ -0,0 +1,634 @@
|
||||
"""
|
||||
Blockchain operations routes for TON integration with async wallet management.
|
||||
Provides secure transaction handling, balance queries, and smart contract interactions.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Dict, List, Optional, Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sanic import Blueprint, Request, response
|
||||
from sanic.response import JSONResponse
|
||||
from sqlalchemy import select, update, and_
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session, get_cache_manager
|
||||
from app.core.logging import get_logger
|
||||
from app.core.models.user import User
|
||||
from app.api.middleware import require_auth, validate_request, rate_limit
|
||||
from app.core.validation import BlockchainTransactionSchema
|
||||
from app.core.background.ton_service import TONService
|
||||
|
||||
# Initialize blueprint
|
||||
blockchain_bp = Blueprint("blockchain", url_prefix="/api/v1/blockchain")
|
||||
logger = get_logger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
@blockchain_bp.route("/wallet/balance", methods=["GET"])
|
||||
@rate_limit(limit=100, window=3600) # 100 balance checks per hour
|
||||
@require_auth(permissions=["blockchain.read"])
|
||||
async def get_wallet_balance(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Get user wallet balance with caching for performance.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: Wallet balance information
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
cache_manager = get_cache_manager()
|
||||
|
||||
# Try cache first
|
||||
balance_key = f"wallet_balance:{user_id}"
|
||||
cached_balance = await cache_manager.get(balance_key)
|
||||
|
||||
if cached_balance:
|
||||
return response.json({
|
||||
"balance": cached_balance,
|
||||
"cached": True,
|
||||
"updated_at": cached_balance.get("updated_at")
|
||||
})
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Get user wallet address
|
||||
user_stmt = select(User).where(User.id == user_id)
|
||||
user_result = await session.execute(user_stmt)
|
||||
user = user_result.scalar_one_or_none()
|
||||
|
||||
if not user or not user.wallet_address:
|
||||
return response.json(
|
||||
{"error": "Wallet not configured", "code": "WALLET_NOT_CONFIGURED"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Get balance from TON service
|
||||
ton_service = TONService()
|
||||
balance_data = await ton_service.get_wallet_balance(user.wallet_address)
|
||||
|
||||
if balance_data.get("error"):
|
||||
return response.json(
|
||||
{"error": balance_data["error"], "code": "BALANCE_FETCH_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
# Cache balance for 5 minutes
|
||||
balance_response = {
|
||||
"address": user.wallet_address,
|
||||
"balance_nanotons": balance_data["balance"],
|
||||
"balance_tons": str(Decimal(balance_data["balance"]) / Decimal("1000000000")),
|
||||
"last_transaction_lt": balance_data.get("last_transaction_lt"),
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
await cache_manager.set(balance_key, balance_response, ttl=300)
|
||||
|
||||
await logger.ainfo(
|
||||
"Wallet balance retrieved",
|
||||
user_id=str(user_id),
|
||||
address=user.wallet_address,
|
||||
balance=balance_data["balance"]
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"balance": balance_response,
|
||||
"cached": False
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get wallet balance",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to get balance", "code": "BALANCE_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@blockchain_bp.route("/wallet/transactions", methods=["GET"])
|
||||
@rate_limit(limit=50, window=3600) # 50 transaction history requests per hour
|
||||
@require_auth(permissions=["blockchain.read"])
|
||||
async def get_wallet_transactions(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Get wallet transaction history with pagination.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: Transaction history
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
# Parse query parameters
|
||||
limit = min(int(request.args.get("limit", 20)), 100) # Max 100 transactions
|
||||
offset = max(int(request.args.get("offset", 0)), 0)
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Get user wallet address
|
||||
user_stmt = select(User).where(User.id == user_id)
|
||||
user_result = await session.execute(user_stmt)
|
||||
user = user_result.scalar_one_or_none()
|
||||
|
||||
if not user or not user.wallet_address:
|
||||
return response.json(
|
||||
{"error": "Wallet not configured", "code": "WALLET_NOT_CONFIGURED"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Check cache for recent transactions
|
||||
cache_manager = get_cache_manager()
|
||||
cache_key = f"wallet_transactions:{user_id}:{limit}:{offset}"
|
||||
cached_transactions = await cache_manager.get(cache_key)
|
||||
|
||||
if cached_transactions:
|
||||
return response.json({
|
||||
"transactions": cached_transactions,
|
||||
"cached": True
|
||||
})
|
||||
|
||||
# Get transactions from TON service
|
||||
ton_service = TONService()
|
||||
transactions_data = await ton_service.get_wallet_transactions(
|
||||
user.wallet_address,
|
||||
limit=limit,
|
||||
offset=offset
|
||||
)
|
||||
|
||||
if transactions_data.get("error"):
|
||||
return response.json(
|
||||
{"error": transactions_data["error"], "code": "TRANSACTIONS_FETCH_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
# Process and format transactions
|
||||
formatted_transactions = []
|
||||
for tx in transactions_data.get("transactions", []):
|
||||
formatted_tx = {
|
||||
"hash": tx.get("hash"),
|
||||
"lt": tx.get("lt"),
|
||||
"timestamp": tx.get("utime"),
|
||||
"value": tx.get("value", "0"),
|
||||
"value_tons": str(Decimal(tx.get("value", "0")) / Decimal("1000000000")),
|
||||
"fee": tx.get("fee", "0"),
|
||||
"source": tx.get("in_msg", {}).get("source"),
|
||||
"destination": tx.get("out_msgs", [{}])[0].get("destination"),
|
||||
"message": tx.get("in_msg", {}).get("message", ""),
|
||||
"type": "incoming" if tx.get("in_msg") else "outgoing",
|
||||
"status": "success" if tx.get("success") else "failed"
|
||||
}
|
||||
formatted_transactions.append(formatted_tx)
|
||||
|
||||
# Cache for 2 minutes
|
||||
await cache_manager.set(cache_key, formatted_transactions, ttl=120)
|
||||
|
||||
return response.json({
|
||||
"transactions": formatted_transactions,
|
||||
"total": len(formatted_transactions),
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"cached": False
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get wallet transactions",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to get transactions", "code": "TRANSACTIONS_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@blockchain_bp.route("/transaction/send", methods=["POST"])
|
||||
@rate_limit(limit=10, window=3600) # 10 transactions per hour
|
||||
@require_auth(permissions=["blockchain.write"])
|
||||
@validate_request(BlockchainTransactionSchema)
|
||||
async def send_transaction(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Send TON transaction with comprehensive validation and monitoring.
|
||||
|
||||
Args:
|
||||
request: Sanic request with transaction data
|
||||
|
||||
Returns:
|
||||
JSONResponse: Transaction submission result
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
data = request.json
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Get user with wallet
|
||||
user_stmt = select(User).where(User.id == user_id)
|
||||
user_result = await session.execute(user_stmt)
|
||||
user = user_result.scalar_one_or_none()
|
||||
|
||||
if not user or not user.wallet_address or not user.wallet_private_key:
|
||||
return response.json(
|
||||
{"error": "Wallet not properly configured", "code": "WALLET_INCOMPLETE"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Validate transaction limits
|
||||
amount_nanotons = data.get("amount", 0)
|
||||
max_transaction = settings.MAX_TRANSACTION_AMOUNT * 1000000000 # Convert to nanotons
|
||||
|
||||
if amount_nanotons > max_transaction:
|
||||
return response.json(
|
||||
{"error": f"Amount exceeds maximum allowed ({settings.MAX_TRANSACTION_AMOUNT} TON)",
|
||||
"code": "AMOUNT_EXCEEDED"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Check daily transaction limit
|
||||
cache_manager = get_cache_manager()
|
||||
daily_limit_key = f"daily_transactions:{user_id}:{datetime.utcnow().date()}"
|
||||
daily_amount = await cache_manager.get(daily_limit_key, default=0)
|
||||
|
||||
if daily_amount + amount_nanotons > settings.DAILY_TRANSACTION_LIMIT * 1000000000:
|
||||
return response.json(
|
||||
{"error": "Daily transaction limit exceeded", "code": "DAILY_LIMIT_EXCEEDED"},
|
||||
status=429
|
||||
)
|
||||
|
||||
# Prepare transaction
|
||||
transaction_data = {
|
||||
"transaction_type": data["transaction_type"],
|
||||
"recipient_address": data.get("recipient_address"),
|
||||
"amount": amount_nanotons,
|
||||
"message": data.get("message", ""),
|
||||
"sender_address": user.wallet_address
|
||||
}
|
||||
|
||||
# Send transaction via TON service
|
||||
ton_service = TONService()
|
||||
tx_result = await ton_service.send_transaction(
|
||||
private_key=user.wallet_private_key,
|
||||
**transaction_data
|
||||
)
|
||||
|
||||
if tx_result.get("error"):
|
||||
await logger.awarning(
|
||||
"Transaction failed",
|
||||
user_id=str(user_id),
|
||||
error=tx_result["error"],
|
||||
**transaction_data
|
||||
)
|
||||
return response.json(
|
||||
{"error": tx_result["error"], "code": "TRANSACTION_FAILED"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Update daily limit counter
|
||||
await cache_manager.increment(daily_limit_key, amount_nanotons, ttl=86400)
|
||||
|
||||
# Store transaction record
|
||||
from app.core.models.blockchain import BlockchainTransaction
|
||||
async with get_async_session() as session:
|
||||
tx_record = BlockchainTransaction(
|
||||
id=uuid4(),
|
||||
user_id=user_id,
|
||||
transaction_hash=tx_result["hash"],
|
||||
transaction_type=data["transaction_type"],
|
||||
amount=amount_nanotons,
|
||||
recipient_address=data.get("recipient_address"),
|
||||
sender_address=user.wallet_address,
|
||||
message=data.get("message", ""),
|
||||
status="pending",
|
||||
network_fee=tx_result.get("fee", 0),
|
||||
block_hash=tx_result.get("block_hash"),
|
||||
logical_time=tx_result.get("lt")
|
||||
)
|
||||
session.add(tx_record)
|
||||
await session.commit()
|
||||
|
||||
# Clear balance cache
|
||||
balance_key = f"wallet_balance:{user_id}"
|
||||
await cache_manager.delete(balance_key)
|
||||
|
||||
await logger.ainfo(
|
||||
"Transaction sent successfully",
|
||||
user_id=str(user_id),
|
||||
transaction_hash=tx_result["hash"],
|
||||
amount=amount_nanotons,
|
||||
recipient=data.get("recipient_address")
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"message": "Transaction sent successfully",
|
||||
"transaction": {
|
||||
"hash": tx_result["hash"],
|
||||
"amount": amount_nanotons,
|
||||
"amount_tons": str(Decimal(amount_nanotons) / Decimal("1000000000")),
|
||||
"recipient": data.get("recipient_address"),
|
||||
"fee": tx_result.get("fee", 0),
|
||||
"status": "pending",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
}, status=201)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to send transaction",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to send transaction", "code": "SEND_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@blockchain_bp.route("/transaction/<tx_hash>/status", methods=["GET"])
|
||||
@rate_limit(limit=100, window=3600) # 100 status checks per hour
|
||||
@require_auth(permissions=["blockchain.read"])
|
||||
async def get_transaction_status(request: Request, tx_hash: str) -> JSONResponse:
|
||||
"""
|
||||
Get transaction status and confirmation details.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
tx_hash: Transaction hash to check
|
||||
|
||||
Returns:
|
||||
JSONResponse: Transaction status information
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
# Check cache first
|
||||
cache_manager = get_cache_manager()
|
||||
status_key = f"tx_status:{tx_hash}"
|
||||
cached_status = await cache_manager.get(status_key)
|
||||
|
||||
if cached_status and cached_status.get("status") in ["confirmed", "failed"]:
|
||||
# Cache confirmed/failed transactions longer
|
||||
return response.json(cached_status)
|
||||
|
||||
# Get transaction from database
|
||||
async with get_async_session() as session:
|
||||
from app.core.models.blockchain import BlockchainTransaction
|
||||
|
||||
tx_stmt = select(BlockchainTransaction).where(
|
||||
and_(
|
||||
BlockchainTransaction.transaction_hash == tx_hash,
|
||||
BlockchainTransaction.user_id == user_id
|
||||
)
|
||||
)
|
||||
tx_result = await session.execute(tx_stmt)
|
||||
tx_record = tx_result.scalar_one_or_none()
|
||||
|
||||
if not tx_record:
|
||||
return response.json(
|
||||
{"error": "Transaction not found", "code": "TRANSACTION_NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
# Get current status from blockchain
|
||||
ton_service = TONService()
|
||||
status_data = await ton_service.get_transaction_status(tx_hash)
|
||||
|
||||
if status_data.get("error"):
|
||||
# Return database status if blockchain query fails
|
||||
tx_status = {
|
||||
"hash": tx_record.transaction_hash,
|
||||
"status": tx_record.status,
|
||||
"confirmations": 0,
|
||||
"amount": tx_record.amount,
|
||||
"created_at": tx_record.created_at.isoformat(),
|
||||
"blockchain_error": status_data["error"]
|
||||
}
|
||||
else:
|
||||
# Update status based on blockchain data
|
||||
new_status = "confirmed" if status_data.get("confirmed") else "pending"
|
||||
if status_data.get("failed"):
|
||||
new_status = "failed"
|
||||
|
||||
tx_status = {
|
||||
"hash": tx_record.transaction_hash,
|
||||
"status": new_status,
|
||||
"confirmations": status_data.get("confirmations", 0),
|
||||
"block_hash": status_data.get("block_hash"),
|
||||
"block_time": status_data.get("block_time"),
|
||||
"amount": tx_record.amount,
|
||||
"fee": status_data.get("fee", tx_record.network_fee),
|
||||
"created_at": tx_record.created_at.isoformat(),
|
||||
"confirmed_at": status_data.get("confirmed_at")
|
||||
}
|
||||
|
||||
# Update database record if status changed
|
||||
if tx_record.status != new_status:
|
||||
async with get_async_session() as session:
|
||||
update_stmt = (
|
||||
update(BlockchainTransaction)
|
||||
.where(BlockchainTransaction.id == tx_record.id)
|
||||
.values(
|
||||
status=new_status,
|
||||
confirmations=status_data.get("confirmations", 0),
|
||||
confirmed_at=datetime.fromisoformat(status_data["confirmed_at"])
|
||||
if status_data.get("confirmed_at") else None
|
||||
)
|
||||
)
|
||||
await session.execute(update_stmt)
|
||||
await session.commit()
|
||||
|
||||
# Cache status (longer for final states)
|
||||
cache_ttl = 300 if tx_status["status"] == "pending" else 3600 # 5 min vs 1 hour
|
||||
await cache_manager.set(status_key, tx_status, ttl=cache_ttl)
|
||||
|
||||
return response.json(tx_status)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get transaction status",
|
||||
user_id=str(request.ctx.user.id),
|
||||
tx_hash=tx_hash,
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to get transaction status", "code": "STATUS_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@blockchain_bp.route("/wallet/create", methods=["POST"])
|
||||
@rate_limit(limit=1, window=86400) # 1 wallet creation per day
|
||||
@require_auth(permissions=["blockchain.wallet.create"])
|
||||
async def create_wallet(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Create new TON wallet for user (one per user).
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: Wallet creation result
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Check if user already has a wallet
|
||||
user_stmt = select(User).where(User.id == user_id)
|
||||
user_result = await session.execute(user_stmt)
|
||||
user = user_result.scalar_one_or_none()
|
||||
|
||||
if not user:
|
||||
return response.json(
|
||||
{"error": "User not found", "code": "USER_NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
if user.wallet_address:
|
||||
return response.json(
|
||||
{"error": "Wallet already exists", "code": "WALLET_EXISTS"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Create wallet via TON service
|
||||
ton_service = TONService()
|
||||
wallet_data = await ton_service.create_wallet()
|
||||
|
||||
if wallet_data.get("error"):
|
||||
return response.json(
|
||||
{"error": wallet_data["error"], "code": "WALLET_CREATION_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
# Store wallet information (encrypt private key)
|
||||
from app.core.security import encrypt_data
|
||||
encrypted_private_key = encrypt_data(
|
||||
wallet_data["private_key"],
|
||||
context=f"wallet:{user_id}"
|
||||
)
|
||||
|
||||
user.wallet_address = wallet_data["address"]
|
||||
user.wallet_private_key = encrypted_private_key
|
||||
user.wallet_created_at = datetime.utcnow()
|
||||
|
||||
await session.commit()
|
||||
|
||||
await logger.ainfo(
|
||||
"Wallet created successfully",
|
||||
user_id=str(user_id),
|
||||
wallet_address=wallet_data["address"]
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"message": "Wallet created successfully",
|
||||
"wallet": {
|
||||
"address": wallet_data["address"],
|
||||
"created_at": datetime.utcnow().isoformat(),
|
||||
"balance": "0",
|
||||
"network": "TON"
|
||||
},
|
||||
"security_note": "Private key is encrypted and stored securely. Keep your account secure."
|
||||
}, status=201)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to create wallet",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to create wallet", "code": "WALLET_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@blockchain_bp.route("/stats", methods=["GET"])
|
||||
@rate_limit(limit=50, window=3600) # 50 stats requests per hour
|
||||
@require_auth(permissions=["blockchain.read"])
|
||||
async def get_blockchain_stats(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Get user blockchain activity statistics.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: Blockchain activity statistics
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
async with get_async_session() as session:
|
||||
from sqlalchemy import func
|
||||
from app.core.models.blockchain import BlockchainTransaction
|
||||
|
||||
# Get transaction statistics
|
||||
stats_stmt = select(
|
||||
func.count(BlockchainTransaction.id).label('total_transactions'),
|
||||
func.sum(BlockchainTransaction.amount).label('total_amount'),
|
||||
func.sum(BlockchainTransaction.network_fee).label('total_fees')
|
||||
).where(BlockchainTransaction.user_id == user_id)
|
||||
|
||||
stats_result = await session.execute(stats_stmt)
|
||||
stats = stats_result.first()
|
||||
|
||||
# Get transactions by type
|
||||
type_stats_stmt = select(
|
||||
BlockchainTransaction.transaction_type,
|
||||
func.count(BlockchainTransaction.id).label('count'),
|
||||
func.sum(BlockchainTransaction.amount).label('amount')
|
||||
).where(
|
||||
BlockchainTransaction.user_id == user_id
|
||||
).group_by(BlockchainTransaction.transaction_type)
|
||||
|
||||
type_result = await session.execute(type_stats_stmt)
|
||||
type_stats = {
|
||||
row.transaction_type: {
|
||||
'count': row.count,
|
||||
'total_amount': row.amount or 0
|
||||
}
|
||||
for row in type_result
|
||||
}
|
||||
|
||||
# Get recent activity (last 30 days)
|
||||
recent_date = datetime.utcnow() - timedelta(days=30)
|
||||
recent_stmt = select(
|
||||
func.count(BlockchainTransaction.id).label('recent_count'),
|
||||
func.sum(BlockchainTransaction.amount).label('recent_amount')
|
||||
).where(
|
||||
and_(
|
||||
BlockchainTransaction.user_id == user_id,
|
||||
BlockchainTransaction.created_at >= recent_date
|
||||
)
|
||||
)
|
||||
|
||||
recent_result = await session.execute(recent_stmt)
|
||||
recent_stats = recent_result.first()
|
||||
|
||||
blockchain_stats = {
|
||||
"total_transactions": stats.total_transactions or 0,
|
||||
"total_amount_nanotons": stats.total_amount or 0,
|
||||
"total_amount_tons": str(Decimal(stats.total_amount or 0) / Decimal("1000000000")),
|
||||
"total_fees_nanotons": stats.total_fees or 0,
|
||||
"total_fees_tons": str(Decimal(stats.total_fees or 0) / Decimal("1000000000")),
|
||||
"by_type": type_stats,
|
||||
"recent_activity": {
|
||||
"transactions_30d": recent_stats.recent_count or 0,
|
||||
"amount_30d_nanotons": recent_stats.recent_amount or 0,
|
||||
"amount_30d_tons": str(Decimal(recent_stats.recent_amount or 0) / Decimal("1000000000"))
|
||||
},
|
||||
"generated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
return response.json(blockchain_stats)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get blockchain stats",
|
||||
user_id=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to get blockchain statistics", "code": "STATS_FAILED"},
|
||||
status=500
|
||||
)
|
||||
@@ -0,0 +1,591 @@
|
||||
"""
|
||||
Enhanced content management routes with async operations and comprehensive validation.
|
||||
Provides secure upload, download, metadata management with Redis caching.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sanic import Blueprint, Request, response
|
||||
from sanic.response import JSONResponse, ResponseStream
|
||||
from sqlalchemy import select, update, delete, and_, or_
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session, get_cache_manager
|
||||
from app.core.logging import get_logger
|
||||
from app.core.models.content import Content, ContentMetadata, ContentAccess, License
|
||||
from app.core.models.user import User
|
||||
from app.api.middleware import require_auth, validate_request, rate_limit
|
||||
from app.core.validation import ContentSchema, ContentUpdateSchema, ContentSearchSchema
|
||||
from app.core.storage import StorageManager
|
||||
from app.core.security import encrypt_data, decrypt_data, generate_access_token
|
||||
|
||||
# Initialize blueprint
|
||||
content_bp = Blueprint("content", url_prefix="/api/v1/content")
|
||||
logger = get_logger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
@content_bp.route("/", methods=["POST"])
|
||||
@rate_limit(limit=50, window=3600) # 50 uploads per hour
|
||||
@require_auth(permissions=["content.create"])
|
||||
@validate_request(ContentSchema)
|
||||
async def create_content(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Create new content with metadata and security validation.
|
||||
|
||||
Args:
|
||||
request: Sanic request with validated content data
|
||||
|
||||
Returns:
|
||||
JSONResponse: Created content information with upload URLs
|
||||
"""
|
||||
try:
|
||||
data = request.json
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Check user upload quota
|
||||
quota_key = f"user:{user_id}:upload_quota"
|
||||
cache_manager = get_cache_manager()
|
||||
|
||||
current_quota = await cache_manager.get(quota_key, default=0)
|
||||
if current_quota >= settings.MAX_UPLOADS_PER_DAY:
|
||||
return response.json(
|
||||
{"error": "Upload quota exceeded", "code": "QUOTA_EXCEEDED"},
|
||||
status=429
|
||||
)
|
||||
|
||||
# Create content record
|
||||
content = Content(
|
||||
id=uuid4(),
|
||||
user_id=user_id,
|
||||
title=data["title"],
|
||||
description=data.get("description"),
|
||||
content_type=data["content_type"],
|
||||
file_size=data.get("file_size", 0),
|
||||
status="pending",
|
||||
visibility=data.get("visibility", "private"),
|
||||
tags=data.get("tags", []),
|
||||
license_id=data.get("license_id")
|
||||
)
|
||||
|
||||
session.add(content)
|
||||
|
||||
# Create metadata if provided
|
||||
if data.get("metadata"):
|
||||
metadata = ContentMetadata(
|
||||
content_id=content.id,
|
||||
metadata_type="custom",
|
||||
data=data["metadata"]
|
||||
)
|
||||
session.add(metadata)
|
||||
|
||||
await session.commit()
|
||||
await session.refresh(content)
|
||||
|
||||
# Update quota counter
|
||||
await cache_manager.increment(quota_key, ttl=86400) # 24 hours
|
||||
|
||||
# Generate upload URLs for chunked upload
|
||||
storage_manager = StorageManager()
|
||||
upload_info = await storage_manager.create_upload_session(
|
||||
content.id, data.get("file_size", 0)
|
||||
)
|
||||
|
||||
# Cache content for quick access
|
||||
content_cache_key = f"content:{content.id}"
|
||||
await cache_manager.set(
|
||||
content_cache_key,
|
||||
{
|
||||
"id": str(content.id),
|
||||
"title": content.title,
|
||||
"status": content.status,
|
||||
"user_id": str(content.user_id)
|
||||
},
|
||||
ttl=3600
|
||||
)
|
||||
|
||||
await logger.ainfo(
|
||||
"Content created successfully",
|
||||
content_id=str(content.id),
|
||||
user_id=str(user_id),
|
||||
title=content.title
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"content_id": str(content.id),
|
||||
"upload_session": upload_info,
|
||||
"status": content.status,
|
||||
"created_at": content.created_at.isoformat()
|
||||
}, status=201)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to create content",
|
||||
error=str(e),
|
||||
user_id=str(user_id)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to create content", "code": "CREATION_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@content_bp.route("/<content_id:uuid>", methods=["GET"])
|
||||
@rate_limit(limit=200, window=3600) # 200 requests per hour
|
||||
@require_auth(permissions=["content.read"])
|
||||
async def get_content(request: Request, content_id: UUID) -> JSONResponse:
|
||||
"""
|
||||
Retrieve content information with access control and caching.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
content_id: UUID of the content to retrieve
|
||||
|
||||
Returns:
|
||||
JSONResponse: Content information or error
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
cache_manager = get_cache_manager()
|
||||
|
||||
# Try cache first
|
||||
cache_key = f"content:{content_id}:full"
|
||||
cached_content = await cache_manager.get(cache_key)
|
||||
|
||||
if cached_content:
|
||||
# Check access permissions from cache
|
||||
if await _check_content_access(content_id, user_id, "read"):
|
||||
return response.json(cached_content)
|
||||
else:
|
||||
return response.json(
|
||||
{"error": "Access denied", "code": "ACCESS_DENIED"},
|
||||
status=403
|
||||
)
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Load content with relationships
|
||||
stmt = (
|
||||
select(Content)
|
||||
.options(
|
||||
selectinload(Content.metadata),
|
||||
selectinload(Content.access_controls),
|
||||
selectinload(Content.license)
|
||||
)
|
||||
.where(Content.id == content_id)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content:
|
||||
return response.json(
|
||||
{"error": "Content not found", "code": "NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
# Check access permissions
|
||||
if not await _check_content_access_db(session, content, user_id, "read"):
|
||||
return response.json(
|
||||
{"error": "Access denied", "code": "ACCESS_DENIED"},
|
||||
status=403
|
||||
)
|
||||
|
||||
# Prepare response data
|
||||
content_data = {
|
||||
"id": str(content.id),
|
||||
"title": content.title,
|
||||
"description": content.description,
|
||||
"content_type": content.content_type,
|
||||
"file_size": content.file_size,
|
||||
"status": content.status,
|
||||
"visibility": content.visibility,
|
||||
"tags": content.tags,
|
||||
"created_at": content.created_at.isoformat(),
|
||||
"updated_at": content.updated_at.isoformat(),
|
||||
"metadata": [
|
||||
{
|
||||
"type": m.metadata_type,
|
||||
"data": m.data
|
||||
} for m in content.metadata
|
||||
],
|
||||
"license": {
|
||||
"name": content.license.name,
|
||||
"description": content.license.description
|
||||
} if content.license else None
|
||||
}
|
||||
|
||||
# Cache the result
|
||||
await cache_manager.set(cache_key, content_data, ttl=1800) # 30 minutes
|
||||
|
||||
# Update access statistics
|
||||
await _update_access_stats(content_id, user_id, "view")
|
||||
|
||||
return response.json(content_data)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to retrieve content",
|
||||
content_id=str(content_id),
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to retrieve content", "code": "RETRIEVAL_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@content_bp.route("/<content_id:uuid>", methods=["PUT"])
|
||||
@rate_limit(limit=100, window=3600) # 100 updates per hour
|
||||
@require_auth(permissions=["content.update"])
|
||||
@validate_request(ContentUpdateSchema)
|
||||
async def update_content(request: Request, content_id: UUID) -> JSONResponse:
|
||||
"""
|
||||
Update content metadata and settings with validation.
|
||||
|
||||
Args:
|
||||
request: Sanic request with update data
|
||||
content_id: UUID of content to update
|
||||
|
||||
Returns:
|
||||
JSONResponse: Updated content information
|
||||
"""
|
||||
try:
|
||||
data = request.json
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Load existing content
|
||||
stmt = select(Content).where(Content.id == content_id)
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content:
|
||||
return response.json(
|
||||
{"error": "Content not found", "code": "NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
# Check update permissions
|
||||
if not await _check_content_access_db(session, content, user_id, "update"):
|
||||
return response.json(
|
||||
{"error": "Access denied", "code": "ACCESS_DENIED"},
|
||||
status=403
|
||||
)
|
||||
|
||||
# Update fields
|
||||
for field, value in data.items():
|
||||
if hasattr(content, field) and field not in ["id", "user_id", "created_at"]:
|
||||
setattr(content, field, value)
|
||||
|
||||
content.updated_at = datetime.utcnow()
|
||||
await session.commit()
|
||||
|
||||
# Invalidate caches
|
||||
cache_manager = get_cache_manager()
|
||||
await cache_manager.delete(f"content:{content_id}")
|
||||
await cache_manager.delete(f"content:{content_id}:full")
|
||||
|
||||
await logger.ainfo(
|
||||
"Content updated successfully",
|
||||
content_id=str(content_id),
|
||||
user_id=str(user_id),
|
||||
updated_fields=list(data.keys())
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"content_id": str(content_id),
|
||||
"status": "updated",
|
||||
"updated_at": content.updated_at.isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to update content",
|
||||
content_id=str(content_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to update content", "code": "UPDATE_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@content_bp.route("/search", methods=["POST"])
|
||||
@rate_limit(limit=100, window=3600) # 100 searches per hour
|
||||
@require_auth(permissions=["content.read"])
|
||||
@validate_request(ContentSearchSchema)
|
||||
async def search_content(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Search content with filters, pagination and caching.
|
||||
|
||||
Args:
|
||||
request: Sanic request with search parameters
|
||||
|
||||
Returns:
|
||||
JSONResponse: Search results with pagination
|
||||
"""
|
||||
try:
|
||||
data = request.json
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
# Build cache key from search parameters
|
||||
search_key = f"search:{hash(str(sorted(data.items())))}:{user_id}"
|
||||
cache_manager = get_cache_manager()
|
||||
|
||||
# Try cache first
|
||||
cached_results = await cache_manager.get(search_key)
|
||||
if cached_results:
|
||||
return response.json(cached_results)
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Build base query
|
||||
stmt = select(Content).where(
|
||||
or_(
|
||||
Content.visibility == "public",
|
||||
Content.user_id == user_id
|
||||
)
|
||||
)
|
||||
|
||||
# Apply filters
|
||||
if data.get("query"):
|
||||
query = f"%{data['query']}%"
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
Content.title.ilike(query),
|
||||
Content.description.ilike(query)
|
||||
)
|
||||
)
|
||||
|
||||
if data.get("content_type"):
|
||||
stmt = stmt.where(Content.content_type == data["content_type"])
|
||||
|
||||
if data.get("tags"):
|
||||
for tag in data["tags"]:
|
||||
stmt = stmt.where(Content.tags.contains([tag]))
|
||||
|
||||
if data.get("status"):
|
||||
stmt = stmt.where(Content.status == data["status"])
|
||||
|
||||
# Apply date filters
|
||||
if data.get("date_from"):
|
||||
stmt = stmt.where(Content.created_at >= datetime.fromisoformat(data["date_from"]))
|
||||
|
||||
if data.get("date_to"):
|
||||
stmt = stmt.where(Content.created_at <= datetime.fromisoformat(data["date_to"]))
|
||||
|
||||
# Apply pagination
|
||||
page = data.get("page", 1)
|
||||
per_page = min(data.get("per_page", 20), 100) # Max 100 items per page
|
||||
offset = (page - 1) * per_page
|
||||
|
||||
# Get total count
|
||||
from sqlalchemy import func
|
||||
count_stmt = select(func.count(Content.id)).select_from(stmt.subquery())
|
||||
total_result = await session.execute(count_stmt)
|
||||
total = total_result.scalar()
|
||||
|
||||
# Apply ordering and pagination
|
||||
if data.get("sort_by") == "created_at":
|
||||
stmt = stmt.order_by(Content.created_at.desc())
|
||||
elif data.get("sort_by") == "title":
|
||||
stmt = stmt.order_by(Content.title.asc())
|
||||
else:
|
||||
stmt = stmt.order_by(Content.updated_at.desc())
|
||||
|
||||
stmt = stmt.offset(offset).limit(per_page)
|
||||
|
||||
# Execute query
|
||||
result = await session.execute(stmt)
|
||||
content_list = result.scalars().all()
|
||||
|
||||
# Prepare response
|
||||
search_results = {
|
||||
"results": [
|
||||
{
|
||||
"id": str(content.id),
|
||||
"title": content.title,
|
||||
"description": content.description,
|
||||
"content_type": content.content_type,
|
||||
"file_size": content.file_size,
|
||||
"status": content.status,
|
||||
"visibility": content.visibility,
|
||||
"tags": content.tags,
|
||||
"created_at": content.created_at.isoformat()
|
||||
} for content in content_list
|
||||
],
|
||||
"pagination": {
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"total": total,
|
||||
"pages": (total + per_page - 1) // per_page
|
||||
}
|
||||
}
|
||||
|
||||
# Cache results for 5 minutes
|
||||
await cache_manager.set(search_key, search_results, ttl=300)
|
||||
|
||||
return response.json(search_results)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Search failed",
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Search failed", "code": "SEARCH_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@content_bp.route("/<content_id:uuid>/download", methods=["GET"])
|
||||
@rate_limit(limit=50, window=3600) # 50 downloads per hour
|
||||
@require_auth(permissions=["content.download"])
|
||||
async def download_content(request: Request, content_id: UUID) -> ResponseStream:
|
||||
"""
|
||||
Secure content download with access control and logging.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
content_id: UUID of content to download
|
||||
|
||||
Returns:
|
||||
ResponseStream: File stream or error response
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Load content
|
||||
stmt = select(Content).where(Content.id == content_id)
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content:
|
||||
return response.json(
|
||||
{"error": "Content not found", "code": "NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
# Check download permissions
|
||||
if not await _check_content_access_db(session, content, user_id, "download"):
|
||||
return response.json(
|
||||
{"error": "Access denied", "code": "ACCESS_DENIED"},
|
||||
status=403
|
||||
)
|
||||
|
||||
# Generate download token
|
||||
download_token = generate_access_token(
|
||||
{"content_id": str(content_id), "user_id": str(user_id)},
|
||||
expires_in=3600 # 1 hour
|
||||
)
|
||||
|
||||
# Log download activity
|
||||
await _update_access_stats(content_id, user_id, "download")
|
||||
|
||||
# Get storage manager and create download stream
|
||||
storage_manager = StorageManager()
|
||||
file_stream = await storage_manager.get_file_stream(content.file_path)
|
||||
|
||||
await logger.ainfo(
|
||||
"Content download initiated",
|
||||
content_id=str(content_id),
|
||||
user_id=str(user_id),
|
||||
filename=content.title
|
||||
)
|
||||
|
||||
return await response.stream(
|
||||
file_stream,
|
||||
headers={
|
||||
"Content-Type": content.content_type or "application/octet-stream",
|
||||
"Content-Disposition": f'attachment; filename="{content.title}"',
|
||||
"Content-Length": str(content.file_size),
|
||||
"X-Download-Token": download_token
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Download failed",
|
||||
content_id=str(content_id),
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Download failed", "code": "DOWNLOAD_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
async def _check_content_access(content_id: UUID, user_id: UUID, action: str) -> bool:
|
||||
"""Check user access to content from cache or database."""
|
||||
cache_manager = get_cache_manager()
|
||||
access_key = f"access:{content_id}:{user_id}:{action}"
|
||||
|
||||
cached_access = await cache_manager.get(access_key)
|
||||
if cached_access is not None:
|
||||
return cached_access
|
||||
|
||||
async with get_async_session() as session:
|
||||
stmt = select(Content).where(Content.id == content_id)
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content:
|
||||
return False
|
||||
|
||||
has_access = await _check_content_access_db(session, content, user_id, action)
|
||||
|
||||
# Cache result for 5 minutes
|
||||
await cache_manager.set(access_key, has_access, ttl=300)
|
||||
|
||||
return has_access
|
||||
|
||||
async def _check_content_access_db(session, content: Content, user_id: UUID, action: str) -> bool:
|
||||
"""Check user access to content in database."""
|
||||
# Content owner always has access
|
||||
if content.user_id == user_id:
|
||||
return True
|
||||
|
||||
# Public content allows read access
|
||||
if content.visibility == "public" and action in ["read", "view"]:
|
||||
return True
|
||||
|
||||
# Check explicit access controls
|
||||
stmt = (
|
||||
select(ContentAccess)
|
||||
.where(
|
||||
and_(
|
||||
ContentAccess.content_id == content.id,
|
||||
ContentAccess.user_id == user_id,
|
||||
ContentAccess.permission == action,
|
||||
ContentAccess.expires_at > datetime.utcnow()
|
||||
)
|
||||
)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
access_control = result.scalar_one_or_none()
|
||||
|
||||
return access_control is not None
|
||||
|
||||
async def _update_access_stats(content_id: UUID, user_id: UUID, action: str) -> None:
|
||||
"""Update content access statistics."""
|
||||
try:
|
||||
cache_manager = get_cache_manager()
|
||||
|
||||
# Update daily stats
|
||||
today = datetime.utcnow().date().isoformat()
|
||||
stats_key = f"stats:{content_id}:{action}:{today}"
|
||||
await cache_manager.increment(stats_key, ttl=86400)
|
||||
|
||||
# Update user activity
|
||||
user_activity_key = f"activity:{user_id}:{action}:{today}"
|
||||
await cache_manager.increment(user_activity_key, ttl=86400)
|
||||
|
||||
except Exception as e:
|
||||
await logger.awarning(
|
||||
"Failed to update access stats",
|
||||
content_id=str(content_id),
|
||||
user_id=str(user_id),
|
||||
action=action,
|
||||
error=str(e)
|
||||
)
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Health check and system status endpoints."""
|
||||
|
||||
import logging
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any
|
||||
|
||||
from sanic import Blueprint, Request, response
|
||||
from sanic.response import JSONResponse
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session
|
||||
from app.core.metrics import get_metrics, get_metrics_content_type, metrics_collector
|
||||
from app.core.background.indexer_service import indexer_service
|
||||
from app.core.background.convert_service import convert_service
|
||||
from app.core.background.ton_service import ton_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
health_bp = Blueprint("health", version=1)
|
||||
|
||||
|
||||
@health_bp.route("/health", methods=["GET"])
|
||||
async def health_check(request: Request) -> JSONResponse:
|
||||
"""Basic health check endpoint."""
|
||||
return response.json({
|
||||
"status": "healthy",
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"service": "my-uploader-bot",
|
||||
"version": "2.0.0"
|
||||
})
|
||||
|
||||
|
||||
@health_bp.route("/health/detailed", methods=["GET"])
|
||||
async def detailed_health_check(request: Request) -> JSONResponse:
|
||||
"""Detailed health check with component status."""
|
||||
health_status = {
|
||||
"status": "healthy",
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"service": "my-uploader-bot",
|
||||
"version": "2.0.0",
|
||||
"components": {}
|
||||
}
|
||||
|
||||
overall_healthy = True
|
||||
|
||||
# Database health
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
await session.execute("SELECT 1")
|
||||
health_status["components"]["database"] = {
|
||||
"status": "healthy",
|
||||
"message": "Database connection successful"
|
||||
}
|
||||
except Exception as e:
|
||||
health_status["components"]["database"] = {
|
||||
"status": "unhealthy",
|
||||
"message": f"Database error: {str(e)}"
|
||||
}
|
||||
overall_healthy = False
|
||||
|
||||
# Redis health
|
||||
try:
|
||||
import redis.asyncio as redis
|
||||
settings = get_settings()
|
||||
redis_client = redis.from_url(settings.redis_url)
|
||||
await redis_client.ping()
|
||||
await redis_client.close()
|
||||
|
||||
health_status["components"]["cache"] = {
|
||||
"status": "healthy",
|
||||
"message": "Redis connection successful"
|
||||
}
|
||||
except Exception as e:
|
||||
health_status["components"]["cache"] = {
|
||||
"status": "unhealthy",
|
||||
"message": f"Redis error: {str(e)}"
|
||||
}
|
||||
overall_healthy = False
|
||||
|
||||
# TON service health
|
||||
try:
|
||||
# Check if TON service is responsive
|
||||
test_result = await ton_service.ping()
|
||||
health_status["components"]["blockchain"] = {
|
||||
"status": "healthy" if test_result else "degraded",
|
||||
"message": "TON service available" if test_result else "TON service degraded"
|
||||
}
|
||||
if not test_result:
|
||||
overall_healthy = False
|
||||
except Exception as e:
|
||||
health_status["components"]["blockchain"] = {
|
||||
"status": "unhealthy",
|
||||
"message": f"TON service error: {str(e)}"
|
||||
}
|
||||
overall_healthy = False
|
||||
|
||||
# Background services health
|
||||
health_status["components"]["background_services"] = {
|
||||
"indexer": {
|
||||
"status": "healthy" if indexer_service.is_running else "stopped",
|
||||
"active_tasks": len([t for t in indexer_service.tasks if not t.done()])
|
||||
},
|
||||
"converter": {
|
||||
"status": "healthy" if convert_service.is_running else "stopped",
|
||||
"active_tasks": len([t for t in convert_service.tasks if not t.done()])
|
||||
}
|
||||
}
|
||||
|
||||
# Update overall status
|
||||
if not overall_healthy:
|
||||
health_status["status"] = "unhealthy"
|
||||
|
||||
status_code = 200 if overall_healthy else 503
|
||||
return response.json(health_status, status=status_code)
|
||||
|
||||
|
||||
@health_bp.route("/health/ready", methods=["GET"])
|
||||
async def readiness_check(request: Request) -> JSONResponse:
|
||||
"""Kubernetes readiness probe endpoint."""
|
||||
try:
|
||||
# Quick database check
|
||||
async with get_async_session() as session:
|
||||
await session.execute("SELECT 1")
|
||||
|
||||
return response.json({
|
||||
"status": "ready",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
except Exception as e:
|
||||
return response.json({
|
||||
"status": "not_ready",
|
||||
"error": str(e),
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}, status=503)
|
||||
|
||||
|
||||
@health_bp.route("/health/live", methods=["GET"])
|
||||
async def liveness_check(request: Request) -> JSONResponse:
|
||||
"""Kubernetes liveness probe endpoint."""
|
||||
return response.json({
|
||||
"status": "alive",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
|
||||
@health_bp.route("/metrics", methods=["GET"])
|
||||
async def prometheus_metrics(request: Request):
|
||||
"""Prometheus metrics endpoint."""
|
||||
try:
|
||||
metrics_data = await get_metrics()
|
||||
return response.raw(
|
||||
metrics_data,
|
||||
content_type=get_metrics_content_type()
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating metrics: {e}")
|
||||
return response.json({
|
||||
"error": "Failed to generate metrics"
|
||||
}, status=500)
|
||||
|
||||
|
||||
@health_bp.route("/stats", methods=["GET"])
|
||||
async def system_stats(request: Request) -> JSONResponse:
|
||||
"""System statistics endpoint."""
|
||||
try:
|
||||
stats = {
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"uptime": metrics_collector.start_time,
|
||||
"services": {}
|
||||
}
|
||||
|
||||
# Get indexer stats
|
||||
try:
|
||||
indexer_stats = await indexer_service.get_indexing_stats()
|
||||
stats["services"]["indexer"] = indexer_stats
|
||||
except Exception as e:
|
||||
stats["services"]["indexer"] = {"error": str(e)}
|
||||
|
||||
# Get converter stats
|
||||
try:
|
||||
converter_stats = await convert_service.get_processing_stats()
|
||||
stats["services"]["converter"] = converter_stats
|
||||
except Exception as e:
|
||||
stats["services"]["converter"] = {"error": str(e)}
|
||||
|
||||
return response.json(stats)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting system stats: {e}")
|
||||
return response.json({
|
||||
"error": "Failed to get system stats"
|
||||
}, status=500)
|
||||
|
||||
|
||||
@health_bp.route("/debug/info", methods=["GET"])
|
||||
async def debug_info(request: Request) -> JSONResponse:
|
||||
"""Debug information endpoint (development only)."""
|
||||
settings = get_settings()
|
||||
|
||||
if settings.environment != "development":
|
||||
return response.json({
|
||||
"error": "Debug endpoint only available in development"
|
||||
}, status=403)
|
||||
|
||||
debug_data = {
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"environment": settings.environment,
|
||||
"debug_mode": settings.debug,
|
||||
"database_url": settings.database_url.replace(
|
||||
settings.database_url.split('@')[0].split('//')[1],
|
||||
"***:***"
|
||||
) if '@' in settings.database_url else "***",
|
||||
"redis_url": settings.redis_url.replace(
|
||||
settings.redis_url.split('@')[0].split('//')[1],
|
||||
"***:***"
|
||||
) if '@' in settings.redis_url else "***",
|
||||
"storage_backend": settings.storage_backend,
|
||||
"ton_network": settings.ton_network,
|
||||
"active_tasks": {
|
||||
"indexer": len([t for t in indexer_service.tasks if not t.done()]),
|
||||
"converter": len([t for t in convert_service.tasks if not t.done()])
|
||||
}
|
||||
}
|
||||
|
||||
return response.json(debug_data)
|
||||
@@ -0,0 +1,379 @@
|
||||
"""MY Network Monitoring Interface - веб-интерфейс мониторинга сети в хакерском стиле."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Any
|
||||
from fastapi import APIRouter, Request, HTTPException
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Создать router для мониторинга
|
||||
router = APIRouter(prefix="/api/my/monitor", tags=["MY Network Monitoring"])
|
||||
|
||||
# Настроить шаблоны
|
||||
templates_dir = Path(__file__).parent.parent.parent / "templates"
|
||||
templates_dir.mkdir(exist_ok=True)
|
||||
templates = Jinja2Templates(directory=str(templates_dir))
|
||||
|
||||
|
||||
def get_node_service():
|
||||
"""Получить сервис ноды."""
|
||||
try:
|
||||
from app.core.my_network.node_service import get_node_service
|
||||
return get_node_service()
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting node service: {e}")
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/", response_class=HTMLResponse)
|
||||
async def monitoring_dashboard(request: Request):
|
||||
"""Главная страница мониторинга MY Network."""
|
||||
try:
|
||||
# Получить данные для дашборда
|
||||
node_service = get_node_service()
|
||||
|
||||
if not node_service:
|
||||
monitoring_data = {
|
||||
"status": "offline",
|
||||
"error": "MY Network service not available"
|
||||
}
|
||||
else:
|
||||
# Собрать данные со всех компонентов
|
||||
node_info = await node_service.get_node_info()
|
||||
peers_info = await node_service.get_peers_info()
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
monitoring_data = {
|
||||
"status": "online",
|
||||
"node_info": node_info,
|
||||
"peers_info": peers_info,
|
||||
"sync_status": sync_status,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
return templates.TemplateResponse("my_network_monitor.html", {
|
||||
"request": request,
|
||||
"monitoring_data": monitoring_data
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error rendering monitoring dashboard: {e}")
|
||||
|
||||
# Fallback HTML если шаблоны не работают
|
||||
return HTMLResponse(content=generate_fallback_html(str(e)))
|
||||
|
||||
|
||||
@router.get("/ascii")
|
||||
async def get_ascii_status():
|
||||
"""Получить ASCII статус сети."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
if not node_service:
|
||||
return {"ascii": generate_offline_ascii(), "status": "offline"}
|
||||
|
||||
# Получить данные
|
||||
node_info = await node_service.get_node_info()
|
||||
peers_info = await node_service.get_peers_info()
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
# Генерировать ASCII
|
||||
ascii_art = await generate_network_ascii(node_info, peers_info, sync_status)
|
||||
|
||||
return {
|
||||
"ascii": ascii_art,
|
||||
"status": "online",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating ASCII status: {e}")
|
||||
return {"ascii": generate_error_ascii(str(e)), "status": "error"}
|
||||
|
||||
|
||||
@router.get("/live")
|
||||
async def live_monitoring_data():
|
||||
"""Получить живые данные для мониторинга."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
if not node_service:
|
||||
raise HTTPException(status_code=503, detail="MY Network service unavailable")
|
||||
|
||||
# Получить свежие данные
|
||||
node_info = await node_service.get_node_info()
|
||||
peers_info = await node_service.get_peers_info()
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
# Статистика сети
|
||||
network_stats = {
|
||||
"connected_peers": peers_info["peer_count"],
|
||||
"active_syncs": sync_status["active_syncs"],
|
||||
"queue_size": sync_status["queue_size"],
|
||||
"uptime": node_info["uptime"],
|
||||
"status": node_info["status"]
|
||||
}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"node_info": node_info,
|
||||
"network_stats": network_stats,
|
||||
"peers": peers_info["peers"][:10], # Показать только первые 10 пиров
|
||||
"sync_status": sync_status
|
||||
},
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting live monitoring data: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
async def generate_network_ascii(node_info: Dict[str, Any], peers_info: Dict[str, Any], sync_status: Dict[str, Any]) -> str:
|
||||
"""Генерировать ASCII представление состояния сети."""
|
||||
|
||||
ascii_parts = []
|
||||
|
||||
# Заголовок
|
||||
ascii_parts.append("""
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ MY NETWORK v2.0 ║
|
||||
║ Distributed Content Protocol ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
""")
|
||||
|
||||
# Информация о ноде
|
||||
status_indicator = "🟢" if node_info.get("status") == "running" else "🔴"
|
||||
uptime_hours = int(node_info.get("uptime", 0) / 3600)
|
||||
|
||||
ascii_parts.append(f"""
|
||||
┌─ NODE STATUS ────────────────────────────────────────────────────────────────┐
|
||||
│ Node ID: {node_info.get('node_id', 'unknown')[:16]}... │
|
||||
│ Status: {status_indicator} {node_info.get('status', 'unknown').upper()} │
|
||||
│ Uptime: {uptime_hours}h {int((node_info.get('uptime', 0) % 3600) / 60)}m │
|
||||
│ Version: MY Network {node_info.get('version', '2.0')} │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
""")
|
||||
|
||||
# Информация о пирах
|
||||
peer_count = peers_info.get("peer_count", 0)
|
||||
peer_status = "🌐" if peer_count > 0 else "🏝️"
|
||||
|
||||
ascii_parts.append(f"""
|
||||
┌─ NETWORK STATUS ─────────────────────────────────────────────────────────────┐
|
||||
│ Connected Peers: {peer_status} {peer_count:>3} │
|
||||
│ Known Nodes: {len(peers_info.get('peers', [])):>3} │
|
||||
│ Network Health: {'CONNECTED' if peer_count > 0 else 'ISOLATED':>9} │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
""")
|
||||
|
||||
# Статус синхронизации
|
||||
sync_running = sync_status.get("is_running", False)
|
||||
active_syncs = sync_status.get("active_syncs", 0)
|
||||
queue_size = sync_status.get("queue_size", 0)
|
||||
|
||||
sync_indicator = "⚡" if sync_running else "⏸️"
|
||||
|
||||
ascii_parts.append(f"""
|
||||
┌─ SYNC STATUS ────────────────────────────────────────────────────────────────┐
|
||||
│ Sync Engine: {sync_indicator} {'RUNNING' if sync_running else 'STOPPED':>7} │
|
||||
│ Active Syncs: {active_syncs:>3} │
|
||||
│ Queue Size: {queue_size:>3} │
|
||||
│ Workers: {sync_status.get('workers_count', 0):>3} │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
""")
|
||||
|
||||
# Визуализация сети
|
||||
if peer_count > 0:
|
||||
ascii_parts.append(generate_network_topology(peers_info.get("peers", [])[:6]))
|
||||
|
||||
# Недавние события синхронизации
|
||||
recent_syncs = sync_status.get("recent_syncs", [])
|
||||
if recent_syncs:
|
||||
ascii_parts.append(generate_sync_history(recent_syncs[-5:]))
|
||||
|
||||
# Подвал
|
||||
current_time = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
ascii_parts.append(f"""
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ Last Updated: {current_time} ║
|
||||
║ MY Network Protocol - Decentralized Content Distribution System ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
""")
|
||||
|
||||
return "".join(ascii_parts)
|
||||
|
||||
|
||||
def generate_network_topology(peers: List[Dict[str, Any]]) -> str:
|
||||
"""Генерировать ASCII топологию сети."""
|
||||
|
||||
topology = ["""
|
||||
┌─ NETWORK TOPOLOGY ───────────────────────────────────────────────────────────┐
|
||||
│ │
|
||||
│ [THIS NODE] │
|
||||
│ │ │"""]
|
||||
|
||||
if len(peers) == 1:
|
||||
topology.append("│ │ │")
|
||||
topology.append(f"│ [{peers[0].get('node_id', 'unknown')[:8]}...] │")
|
||||
elif len(peers) <= 3:
|
||||
topology.append("│ ┌───────┼───────┐ │")
|
||||
for i, peer in enumerate(peers):
|
||||
spaces = " " if i == 0 else (" " if i == 1 else " ")
|
||||
topology.append(f"│{spaces}[{peer.get('node_id', 'unknown')[:8]}...] │")
|
||||
else:
|
||||
topology.append("│ ┌───────┬───────┼───────┬───────┐ │")
|
||||
topology.append("│ │ │ │ │ │ │")
|
||||
for i, peer in enumerate(peers[:5]):
|
||||
if i < 5:
|
||||
spaces = [" ", " ", " ", " ", " "][i]
|
||||
topology.append(f"│{spaces}[{peer.get('node_id', 'unknown')[:6]}] │")
|
||||
if len(peers) > 5:
|
||||
topology.append("│ ... │")
|
||||
|
||||
topology.append("│ │")
|
||||
topology.append("└──────────────────────────────────────────────────────────────────────────────┘")
|
||||
|
||||
return "\n".join(topology) + "\n"
|
||||
|
||||
|
||||
def generate_sync_history(recent_syncs: List[Dict[str, Any]]) -> str:
|
||||
"""Генерировать историю синхронизации."""
|
||||
|
||||
history = ["""
|
||||
┌─ RECENT SYNC ACTIVITY ───────────────────────────────────────────────────────┐"""]
|
||||
|
||||
if not recent_syncs:
|
||||
history.append("│ No recent sync activity │")
|
||||
else:
|
||||
for sync in recent_syncs:
|
||||
content_hash = sync.get("content_hash", "unknown")[:12]
|
||||
status = sync.get("status", "unknown")
|
||||
status_icon = {"completed": "✅", "failed": "❌", "partial": "⚠️"}.get(status, "❓")
|
||||
|
||||
history.append(f"│ {status_icon} {content_hash}... - {status.upper():>9} │")
|
||||
|
||||
history.append("└──────────────────────────────────────────────────────────────────────────────┘")
|
||||
|
||||
return "\n".join(history) + "\n"
|
||||
|
||||
|
||||
def generate_offline_ascii() -> str:
|
||||
"""Генерировать ASCII для офлайн состояния."""
|
||||
return """
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ MY NETWORK v2.0 ║
|
||||
║ Distributed Content Protocol ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
|
||||
┌─ SYSTEM STATUS ──────────────────────────────────────────────────────────────┐
|
||||
│ │
|
||||
│ 🔴 OFFLINE │
|
||||
│ │
|
||||
│ MY Network service is not available │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ Status: OFFLINE - Service not initialized ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
"""
|
||||
|
||||
|
||||
def generate_error_ascii(error_message: str) -> str:
|
||||
"""Генерировать ASCII для ошибки."""
|
||||
return f"""
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ MY NETWORK v2.0 ║
|
||||
║ Distributed Content Protocol ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
|
||||
┌─ ERROR STATE ────────────────────────────────────────────────────────────────┐
|
||||
│ │
|
||||
│ ❌ ERROR │
|
||||
│ │
|
||||
│ {error_message[:64]:^64} │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ Status: ERROR - Check system logs for details ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
"""
|
||||
|
||||
|
||||
def generate_fallback_html(error_message: str = "") -> str:
|
||||
"""Генерировать fallback HTML если шаблоны не работают."""
|
||||
return f'''
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>MY Network Monitor</title>
|
||||
<style>
|
||||
body {{
|
||||
background: #000;
|
||||
color: #0f0;
|
||||
font-family: 'Courier New', monospace;
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
overflow-x: auto;
|
||||
}}
|
||||
.container {{
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
}}
|
||||
.ascii-art {{
|
||||
white-space: pre;
|
||||
font-size: 12px;
|
||||
line-height: 1.2;
|
||||
}}
|
||||
.error {{
|
||||
color: #f00;
|
||||
text-align: center;
|
||||
padding: 20px;
|
||||
}}
|
||||
.refresh-btn {{
|
||||
background: #0f0;
|
||||
color: #000;
|
||||
border: none;
|
||||
padding: 10px 20px;
|
||||
font-family: inherit;
|
||||
cursor: pointer;
|
||||
margin: 20px 0;
|
||||
}}
|
||||
.refresh-btn:hover {{
|
||||
background: #fff;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="ascii-art">
|
||||
{generate_error_ascii(error_message) if error_message else generate_offline_ascii()}
|
||||
</div>
|
||||
|
||||
<button class="refresh-btn" onclick="location.reload()">REFRESH SYSTEM STATUS</button>
|
||||
|
||||
<div class="error">
|
||||
{f"Error: {error_message}" if error_message else "MY Network service not available"}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Автообновление каждые 30 секунд
|
||||
setTimeout(() => location.reload(), 30000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
@@ -0,0 +1,452 @@
|
||||
"""MY Network Monitoring Sanic Blueprint - веб-интерфейс мониторинга сети."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Any
|
||||
from pathlib import Path
|
||||
|
||||
from sanic import Blueprint, Request
|
||||
from sanic.response import json as json_response, html as html_response
|
||||
from sanic.exceptions import SanicException
|
||||
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Создать blueprint для мониторинга
|
||||
bp = Blueprint("my_monitoring", url_prefix="/api/my/monitor")
|
||||
|
||||
|
||||
def get_node_service():
|
||||
"""Получить сервис ноды."""
|
||||
try:
|
||||
from app.core.my_network.node_service import get_node_service
|
||||
return get_node_service()
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting node service: {e}")
|
||||
return None
|
||||
|
||||
|
||||
@bp.get("/")
|
||||
async def monitoring_dashboard(request: Request):
|
||||
"""Главная страница мониторинга MY Network."""
|
||||
try:
|
||||
# Получить данные для дашборда
|
||||
node_service = get_node_service()
|
||||
|
||||
if not node_service:
|
||||
monitoring_data = {
|
||||
"status": "offline",
|
||||
"error": "MY Network service not available"
|
||||
}
|
||||
else:
|
||||
# Собрать данные со всех компонентов
|
||||
node_info = await node_service.get_node_info()
|
||||
peers_info = await node_service.get_peers_info()
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
monitoring_data = {
|
||||
"status": "online",
|
||||
"node_info": node_info,
|
||||
"peers_info": peers_info,
|
||||
"sync_status": sync_status,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Попробовать использовать шаблон
|
||||
try:
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
# Настроить Jinja2
|
||||
templates_dir = Path(__file__).parent.parent.parent / "templates"
|
||||
if templates_dir.exists():
|
||||
env = Environment(loader=FileSystemLoader(str(templates_dir)))
|
||||
template = env.get_template("my_network_monitor.html")
|
||||
|
||||
html_content = template.render(monitoring_data=monitoring_data)
|
||||
return html_response(html_content)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Template rendering failed: {e}")
|
||||
|
||||
# Fallback HTML если шаблоны не работают
|
||||
return html_response(generate_fallback_html(monitoring_data))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error rendering monitoring dashboard: {e}")
|
||||
return html_response(generate_fallback_html({"status": "error", "error": str(e)}))
|
||||
|
||||
|
||||
@bp.get("/ascii")
|
||||
async def get_ascii_status(request: Request):
|
||||
"""Получить ASCII статус сети."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
if not node_service:
|
||||
return json_response({"ascii": generate_offline_ascii(), "status": "offline"})
|
||||
|
||||
# Получить данные
|
||||
node_info = await node_service.get_node_info()
|
||||
peers_info = await node_service.get_peers_info()
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
# Генерировать ASCII
|
||||
ascii_art = await generate_network_ascii(node_info, peers_info, sync_status)
|
||||
|
||||
return json_response({
|
||||
"ascii": ascii_art,
|
||||
"status": "online",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating ASCII status: {e}")
|
||||
return json_response({"ascii": generate_error_ascii(str(e)), "status": "error"})
|
||||
|
||||
|
||||
@bp.get("/live")
|
||||
async def live_monitoring_data(request: Request):
|
||||
"""Получить живые данные для мониторинга."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
if not node_service:
|
||||
return json_response(
|
||||
{"error": "MY Network service unavailable"},
|
||||
status=503
|
||||
)
|
||||
|
||||
# Получить свежие данные
|
||||
node_info = await node_service.get_node_info()
|
||||
peers_info = await node_service.get_peers_info()
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
# Статистика сети
|
||||
network_stats = {
|
||||
"connected_peers": peers_info["peer_count"],
|
||||
"active_syncs": sync_status["active_syncs"],
|
||||
"queue_size": sync_status["queue_size"],
|
||||
"uptime": node_info["uptime"],
|
||||
"status": node_info["status"]
|
||||
}
|
||||
|
||||
return json_response({
|
||||
"success": True,
|
||||
"data": {
|
||||
"node_info": node_info,
|
||||
"network_stats": network_stats,
|
||||
"peers": peers_info["peers"][:10], # Показать только первые 10 пиров
|
||||
"sync_status": sync_status
|
||||
},
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting live monitoring data: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
async def generate_network_ascii(node_info: Dict[str, Any], peers_info: Dict[str, Any], sync_status: Dict[str, Any]) -> str:
|
||||
"""Генерировать ASCII представление состояния сети."""
|
||||
|
||||
ascii_parts = []
|
||||
|
||||
# Заголовок
|
||||
ascii_parts.append("""
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ MY NETWORK v2.0 ║
|
||||
║ Distributed Content Protocol ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
""")
|
||||
|
||||
# Информация о ноде
|
||||
status_indicator = "🟢" if node_info.get("status") == "running" else "🔴"
|
||||
uptime_hours = int(node_info.get("uptime", 0) / 3600)
|
||||
|
||||
ascii_parts.append(f"""
|
||||
┌─ NODE STATUS ────────────────────────────────────────────────────────────────┐
|
||||
│ Node ID: {node_info.get('node_id', 'unknown')[:16]}... │
|
||||
│ Status: {status_indicator} {node_info.get('status', 'unknown').upper()} │
|
||||
│ Uptime: {uptime_hours}h {int((node_info.get('uptime', 0) % 3600) / 60)}m │
|
||||
│ Version: MY Network {node_info.get('version', '2.0')} │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
""")
|
||||
|
||||
# Информация о пирах
|
||||
peer_count = peers_info.get("peer_count", 0)
|
||||
peer_status = "🌐" if peer_count > 0 else "🏝️"
|
||||
|
||||
ascii_parts.append(f"""
|
||||
┌─ NETWORK STATUS ─────────────────────────────────────────────────────────────┐
|
||||
│ Connected Peers: {peer_status} {peer_count:>3} │
|
||||
│ Known Nodes: {len(peers_info.get('peers', [])):>3} │
|
||||
│ Network Health: {'CONNECTED' if peer_count > 0 else 'ISOLATED':>9} │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
""")
|
||||
|
||||
# Статус синхронизации
|
||||
sync_running = sync_status.get("is_running", False)
|
||||
active_syncs = sync_status.get("active_syncs", 0)
|
||||
queue_size = sync_status.get("queue_size", 0)
|
||||
|
||||
sync_indicator = "⚡" if sync_running else "⏸️"
|
||||
|
||||
ascii_parts.append(f"""
|
||||
┌─ SYNC STATUS ────────────────────────────────────────────────────────────────┐
|
||||
│ Sync Engine: {sync_indicator} {'RUNNING' if sync_running else 'STOPPED':>7} │
|
||||
│ Active Syncs: {active_syncs:>3} │
|
||||
│ Queue Size: {queue_size:>3} │
|
||||
│ Workers: {sync_status.get('workers_count', 0):>3} │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
""")
|
||||
|
||||
# Подвал
|
||||
current_time = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
ascii_parts.append(f"""
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ Last Updated: {current_time} ║
|
||||
║ MY Network Protocol - Decentralized Content Distribution System ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
""")
|
||||
|
||||
return "".join(ascii_parts)
|
||||
|
||||
|
||||
def generate_offline_ascii() -> str:
|
||||
"""Генерировать ASCII для офлайн состояния."""
|
||||
return """
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ MY NETWORK v2.0 ║
|
||||
║ Distributed Content Protocol ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
|
||||
┌─ SYSTEM STATUS ──────────────────────────────────────────────────────────────┐
|
||||
│ │
|
||||
│ 🔴 OFFLINE │
|
||||
│ │
|
||||
│ MY Network service is not available │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ Status: OFFLINE - Service not initialized ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
"""
|
||||
|
||||
|
||||
def generate_error_ascii(error_message: str) -> str:
|
||||
"""Генерировать ASCII для ошибки."""
|
||||
return f"""
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ MY NETWORK v2.0 ║
|
||||
║ Distributed Content Protocol ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
|
||||
┌─ ERROR STATE ────────────────────────────────────────────────────────────────┐
|
||||
│ │
|
||||
│ ❌ ERROR │
|
||||
│ │
|
||||
│ {error_message[:64]:^64} │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ Status: ERROR - Check system logs for details ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
"""
|
||||
|
||||
|
||||
def generate_fallback_html(monitoring_data: Dict[str, Any]) -> str:
|
||||
"""Генерировать fallback HTML если шаблоны не работают."""
|
||||
|
||||
status = monitoring_data.get("status", "unknown")
|
||||
error_message = monitoring_data.get("error", "")
|
||||
|
||||
# Генерировать информацию о статусе
|
||||
if status == "online":
|
||||
node_info = monitoring_data.get("node_info", {})
|
||||
peers_info = monitoring_data.get("peers_info", {})
|
||||
sync_status = monitoring_data.get("sync_status", {})
|
||||
|
||||
status_info = f"""
|
||||
<div class="status-section">
|
||||
<h3>Node Status</h3>
|
||||
<ul>
|
||||
<li>Node ID: {node_info.get('node_id', 'unknown')[:16]}...</li>
|
||||
<li>Status: {node_info.get('status', 'unknown').upper()}</li>
|
||||
<li>Uptime: {int(node_info.get('uptime', 0) / 3600)}h {int((node_info.get('uptime', 0) % 3600) / 60)}m</li>
|
||||
<li>Version: MY Network {node_info.get('version', '2.0')}</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="status-section">
|
||||
<h3>Network Status</h3>
|
||||
<ul>
|
||||
<li>Connected Peers: {peers_info.get('peer_count', 0)}</li>
|
||||
<li>Known Nodes: {len(peers_info.get('peers', []))}</li>
|
||||
<li>Network Health: {'CONNECTED' if peers_info.get('peer_count', 0) > 0 else 'ISOLATED'}</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="status-section">
|
||||
<h3>Sync Status</h3>
|
||||
<ul>
|
||||
<li>Sync Engine: {'RUNNING' if sync_status.get('is_running', False) else 'STOPPED'}</li>
|
||||
<li>Active Syncs: {sync_status.get('active_syncs', 0)}</li>
|
||||
<li>Queue Size: {sync_status.get('queue_size', 0)}</li>
|
||||
<li>Workers: {sync_status.get('workers_count', 0)}</li>
|
||||
</ul>
|
||||
</div>
|
||||
"""
|
||||
else:
|
||||
status_info = f"""
|
||||
<div class="error-section">
|
||||
<h3>Status: {status.upper()}</h3>
|
||||
<p>{error_message if error_message else 'MY Network service not available'}</p>
|
||||
</div>
|
||||
"""
|
||||
|
||||
return f'''
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>MY Network Monitor</title>
|
||||
<style>
|
||||
body {{
|
||||
background: linear-gradient(135deg, #0a0a0a 0%, #1a1a2e 50%, #16213e 100%);
|
||||
color: #00ff41;
|
||||
font-family: 'Courier New', monospace;
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
min-height: 100vh;
|
||||
}}
|
||||
|
||||
.container {{
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
}}
|
||||
|
||||
.header {{
|
||||
text-align: center;
|
||||
margin-bottom: 30px;
|
||||
padding: 20px;
|
||||
border: 2px solid #00ff41;
|
||||
border-radius: 10px;
|
||||
background: rgba(0, 255, 65, 0.05);
|
||||
}}
|
||||
|
||||
.header h1 {{
|
||||
font-size: 2.5em;
|
||||
text-shadow: 0 0 10px #00ff41;
|
||||
margin: 0;
|
||||
}}
|
||||
|
||||
.status-section {{
|
||||
background: rgba(0, 0, 0, 0.7);
|
||||
border: 1px solid #00ff41;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin: 20px 0;
|
||||
}}
|
||||
|
||||
.status-section h3 {{
|
||||
color: #00ff41;
|
||||
margin-bottom: 15px;
|
||||
text-transform: uppercase;
|
||||
border-bottom: 1px solid #00ff41;
|
||||
padding-bottom: 5px;
|
||||
}}
|
||||
|
||||
.status-section ul {{
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
}}
|
||||
|
||||
.status-section li {{
|
||||
margin: 10px 0;
|
||||
padding: 5px 0;
|
||||
border-bottom: 1px dotted #333;
|
||||
}}
|
||||
|
||||
.error-section {{
|
||||
background: rgba(255, 0, 0, 0.1);
|
||||
border: 1px solid #ff0000;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin: 20px 0;
|
||||
text-align: center;
|
||||
}}
|
||||
|
||||
.error-section h3 {{
|
||||
color: #ff0000;
|
||||
margin-bottom: 15px;
|
||||
}}
|
||||
|
||||
.controls {{
|
||||
text-align: center;
|
||||
margin: 30px 0;
|
||||
}}
|
||||
|
||||
.btn {{
|
||||
background: linear-gradient(45deg, #00ff41, #00cc33);
|
||||
color: #000;
|
||||
border: none;
|
||||
padding: 12px 24px;
|
||||
font-family: inherit;
|
||||
font-weight: bold;
|
||||
cursor: pointer;
|
||||
border-radius: 5px;
|
||||
text-transform: uppercase;
|
||||
margin: 0 10px;
|
||||
text-decoration: none;
|
||||
display: inline-block;
|
||||
}}
|
||||
|
||||
.btn:hover {{
|
||||
background: linear-gradient(45deg, #00cc33, #00ff41);
|
||||
}}
|
||||
|
||||
.footer {{
|
||||
text-align: center;
|
||||
margin-top: 40px;
|
||||
padding: 20px;
|
||||
border-top: 1px solid #00ff41;
|
||||
color: #888;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<h1>MY NETWORK MONITOR</h1>
|
||||
<p>Distributed Content Protocol v2.0</p>
|
||||
<p>Last Update: {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')}</p>
|
||||
</div>
|
||||
|
||||
{status_info}
|
||||
|
||||
<div class="controls">
|
||||
<a href="/api/my/monitor/" class="btn">🔄 REFRESH</a>
|
||||
<a href="/api/my/monitor/ascii" class="btn">📊 ASCII VIEW</a>
|
||||
<a href="/api/my/node/info" class="btn">ℹ️ NODE INFO</a>
|
||||
<a href="/api/my/health" class="btn">❤️ HEALTH</a>
|
||||
</div>
|
||||
|
||||
<div class="footer">
|
||||
<p>MY Network Protocol - Decentralized Content Distribution System</p>
|
||||
<p>Real-time monitoring dashboard</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Автообновление каждые 30 секунд
|
||||
setTimeout(() => location.reload(), 30000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
@@ -0,0 +1,655 @@
|
||||
"""MY Network API Routes - эндпоинты для работы с распределенной сетью."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any
|
||||
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Query
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
from sqlalchemy import select, and_, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database_compatible import get_async_session
|
||||
from app.core.models.content_compatible import Content, ContentMetadata
|
||||
from app.core.security import get_current_user_optional
|
||||
from app.core.cache import cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Создать router для MY Network API
|
||||
router = APIRouter(prefix="/api/my", tags=["MY Network"])
|
||||
|
||||
|
||||
def get_node_service():
|
||||
"""Получить сервис ноды."""
|
||||
try:
|
||||
from app.core.my_network.node_service import get_node_service
|
||||
return get_node_service()
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting node service: {e}")
|
||||
raise HTTPException(status_code=503, detail="MY Network service unavailable")
|
||||
|
||||
|
||||
@router.get("/node/info")
|
||||
async def get_node_info():
|
||||
"""Получить информацию о текущей ноде."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
if not node_service:
|
||||
raise HTTPException(status_code=503, detail="Node service not available")
|
||||
|
||||
node_info = await node_service.get_node_info()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": node_info,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting node info: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/node/peers")
|
||||
async def get_node_peers():
|
||||
"""Получить список подключенных пиров."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
peers_info = await node_service.get_peers_info()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"connected_peers": peers_info["connected_peers"],
|
||||
"peer_count": peers_info["peer_count"],
|
||||
"peers": peers_info["peers"]
|
||||
},
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting peers: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/node/peers/connect")
|
||||
async def connect_to_peer(peer_data: Dict[str, Any]):
|
||||
"""Подключиться к новому пиру."""
|
||||
try:
|
||||
peer_address = peer_data.get("address")
|
||||
if not peer_address:
|
||||
raise HTTPException(status_code=400, detail="Peer address is required")
|
||||
|
||||
node_service = get_node_service()
|
||||
success = await node_service.peer_manager.connect_to_peer(peer_address)
|
||||
|
||||
if success:
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Successfully connected to peer: {peer_address}",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="Failed to connect to peer")
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error connecting to peer: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.delete("/node/peers/{peer_id}")
|
||||
async def disconnect_peer(peer_id: str):
|
||||
"""Отключиться от пира."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
success = await node_service.peer_manager.disconnect_peer(peer_id)
|
||||
|
||||
if success:
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Successfully disconnected from peer: {peer_id}",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
else:
|
||||
raise HTTPException(status_code=404, detail="Peer not found or already disconnected")
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error disconnecting peer: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/content/list")
|
||||
async def get_content_list(
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
session: AsyncSession = Depends(get_async_session)
|
||||
):
|
||||
"""Получить список доступного контента."""
|
||||
try:
|
||||
# Кэшировать результат на 5 минут
|
||||
cache_key = f"my_network:content_list:{limit}:{offset}"
|
||||
cached_result = await cache.get(cache_key)
|
||||
|
||||
if cached_result:
|
||||
return json.loads(cached_result)
|
||||
|
||||
# Получить контент из БД
|
||||
stmt = (
|
||||
select(Content, ContentMetadata)
|
||||
.outerjoin(ContentMetadata, Content.id == ContentMetadata.content_id)
|
||||
.where(Content.is_active == True)
|
||||
.order_by(Content.created_at.desc())
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
content_items = []
|
||||
|
||||
for content, metadata in result:
|
||||
content_data = {
|
||||
"hash": content.sha256_hash or content.md5_hash,
|
||||
"filename": content.filename,
|
||||
"original_filename": content.original_filename,
|
||||
"file_size": content.file_size,
|
||||
"file_type": content.file_type,
|
||||
"mime_type": content.mime_type,
|
||||
"created_at": content.created_at.isoformat(),
|
||||
"encrypted": getattr(content, 'encrypted', False),
|
||||
"metadata": metadata.to_dict() if metadata else {}
|
||||
}
|
||||
content_items.append(content_data)
|
||||
|
||||
# Получить общее количество
|
||||
count_stmt = select(func.count(Content.id)).where(Content.is_active == True)
|
||||
count_result = await session.execute(count_stmt)
|
||||
total_count = count_result.scalar()
|
||||
|
||||
response_data = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"content": content_items,
|
||||
"total": total_count,
|
||||
"limit": limit,
|
||||
"offset": offset
|
||||
},
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Кэшировать результат
|
||||
await cache.set(cache_key, json.dumps(response_data), expire=300)
|
||||
|
||||
return response_data
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting content list: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/content/{content_hash}/exists")
|
||||
async def check_content_exists(
|
||||
content_hash: str,
|
||||
session: AsyncSession = Depends(get_async_session)
|
||||
):
|
||||
"""Проверить существование контента по хешу."""
|
||||
try:
|
||||
# Кэшировать результат на 30 минут
|
||||
cache_key = f"my_network:content_exists:{content_hash}"
|
||||
cached_result = await cache.get(cache_key)
|
||||
|
||||
if cached_result is not None:
|
||||
return {"exists": cached_result == "true", "hash": content_hash}
|
||||
|
||||
# Проверить в БД
|
||||
stmt = select(Content.id).where(
|
||||
and_(
|
||||
Content.is_active == True,
|
||||
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
|
||||
)
|
||||
)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
exists = result.scalar_one_or_none() is not None
|
||||
|
||||
# Кэшировать результат
|
||||
await cache.set(cache_key, "true" if exists else "false", expire=1800)
|
||||
|
||||
return {
|
||||
"exists": exists,
|
||||
"hash": content_hash,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking content existence: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/content/{content_hash}/metadata")
|
||||
async def get_content_metadata(
|
||||
content_hash: str,
|
||||
session: AsyncSession = Depends(get_async_session)
|
||||
):
|
||||
"""Получить метаданные контента."""
|
||||
try:
|
||||
# Кэшировать результат на 10 минут
|
||||
cache_key = f"my_network:content_metadata:{content_hash}"
|
||||
cached_result = await cache.get(cache_key)
|
||||
|
||||
if cached_result:
|
||||
return json.loads(cached_result)
|
||||
|
||||
# Найти контент в БД
|
||||
stmt = (
|
||||
select(Content, ContentMetadata)
|
||||
.outerjoin(ContentMetadata, Content.id == ContentMetadata.content_id)
|
||||
.where(
|
||||
and_(
|
||||
Content.is_active == True,
|
||||
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
content_data = result.first()
|
||||
|
||||
if not content_data:
|
||||
raise HTTPException(status_code=404, detail="Content not found")
|
||||
|
||||
content, metadata = content_data
|
||||
|
||||
response_data = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"hash": content_hash,
|
||||
"filename": content.filename,
|
||||
"original_filename": content.original_filename,
|
||||
"file_size": content.file_size,
|
||||
"file_type": content.file_type,
|
||||
"mime_type": content.mime_type,
|
||||
"created_at": content.created_at.isoformat(),
|
||||
"updated_at": content.updated_at.isoformat() if content.updated_at else None,
|
||||
"encrypted": getattr(content, 'encrypted', False),
|
||||
"processing_status": getattr(content, 'processing_status', 'completed'),
|
||||
"metadata": metadata.to_dict() if metadata else {}
|
||||
},
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Кэшировать результат
|
||||
await cache.set(cache_key, json.dumps(response_data), expire=600)
|
||||
|
||||
return response_data
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting content metadata: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/content/{content_hash}/download")
|
||||
async def download_content(
|
||||
content_hash: str,
|
||||
session: AsyncSession = Depends(get_async_session)
|
||||
):
|
||||
"""Скачать контент по хешу."""
|
||||
try:
|
||||
# Найти контент в БД
|
||||
stmt = select(Content).where(
|
||||
and_(
|
||||
Content.is_active == True,
|
||||
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
|
||||
)
|
||||
)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content:
|
||||
raise HTTPException(status_code=404, detail="Content not found")
|
||||
|
||||
# Проверить существование файла
|
||||
file_path = Path(content.file_path)
|
||||
if not file_path.exists():
|
||||
raise HTTPException(status_code=404, detail="File not found on disk")
|
||||
|
||||
# Вернуть файл
|
||||
return FileResponse(
|
||||
path=str(file_path),
|
||||
filename=content.original_filename or content.filename,
|
||||
media_type=content.mime_type or "application/octet-stream"
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading content: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/content/{content_hash}/upload")
|
||||
async def upload_content(
|
||||
content_hash: str,
|
||||
file: UploadFile = File(...),
|
||||
session: AsyncSession = Depends(get_async_session)
|
||||
):
|
||||
"""Загрузить контент в ноду."""
|
||||
try:
|
||||
# Проверить, не существует ли уже контент
|
||||
exists_stmt = select(Content.id).where(
|
||||
and_(
|
||||
Content.is_active == True,
|
||||
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
|
||||
)
|
||||
)
|
||||
|
||||
exists_result = await session.execute(exists_stmt)
|
||||
if exists_result.scalar_one_or_none():
|
||||
return {
|
||||
"success": True,
|
||||
"message": "Content already exists",
|
||||
"hash": content_hash
|
||||
}
|
||||
|
||||
# Создать директорию для хранения
|
||||
storage_path = Path("./storage/my-network/received")
|
||||
storage_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Сохранить файл
|
||||
file_path = storage_path / f"{content_hash}_{file.filename}"
|
||||
|
||||
with open(file_path, "wb") as buffer:
|
||||
content_data = await file.read()
|
||||
buffer.write(content_data)
|
||||
|
||||
# Вычислить хеши для проверки
|
||||
import hashlib
|
||||
md5_hash = hashlib.md5(content_data).hexdigest()
|
||||
sha256_hash = hashlib.sha256(content_data).hexdigest()
|
||||
|
||||
# Проверить соответствие хеша
|
||||
if content_hash not in [md5_hash, sha256_hash]:
|
||||
file_path.unlink() # Удалить файл
|
||||
raise HTTPException(status_code=400, detail="Content hash mismatch")
|
||||
|
||||
# Сохранить в БД
|
||||
new_content = Content(
|
||||
filename=file.filename,
|
||||
original_filename=file.filename,
|
||||
file_path=str(file_path),
|
||||
file_size=len(content_data),
|
||||
file_type=file.filename.split('.')[-1] if '.' in file.filename else 'unknown',
|
||||
mime_type=file.content_type or "application/octet-stream",
|
||||
md5_hash=md5_hash,
|
||||
sha256_hash=sha256_hash,
|
||||
is_active=True,
|
||||
processing_status="completed"
|
||||
)
|
||||
|
||||
session.add(new_content)
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"Successfully uploaded content {content_hash}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": "Content uploaded successfully",
|
||||
"hash": content_hash,
|
||||
"content_id": new_content.id,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error uploading content: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/content/replicate")
|
||||
async def replicate_content(replication_request: Dict[str, Any]):
|
||||
"""Принять запрос на репликацию контента."""
|
||||
try:
|
||||
content_hash = replication_request.get("content_hash")
|
||||
metadata = replication_request.get("metadata", {})
|
||||
source_node = replication_request.get("source_node")
|
||||
|
||||
if not content_hash:
|
||||
raise HTTPException(status_code=400, detail="Content hash is required")
|
||||
|
||||
# Проверить, нужна ли репликация
|
||||
async with get_async_session() as session:
|
||||
exists_stmt = select(Content.id).where(
|
||||
and_(
|
||||
Content.is_active == True,
|
||||
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
|
||||
)
|
||||
)
|
||||
|
||||
exists_result = await session.execute(exists_stmt)
|
||||
if exists_result.scalar_one_or_none():
|
||||
return {
|
||||
"success": True,
|
||||
"message": "Content already exists, replication not needed",
|
||||
"hash": content_hash
|
||||
}
|
||||
|
||||
# Подготовить для репликации
|
||||
logger.info(f"Accepting replication request for {content_hash} from {source_node}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": "Replication request accepted",
|
||||
"hash": content_hash,
|
||||
"ready_for_upload": True,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing replication request: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/sync/status")
|
||||
async def get_sync_status():
|
||||
"""Получить статус синхронизации."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": sync_status,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting sync status: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/sync/start")
|
||||
async def start_network_sync():
|
||||
"""Запустить синхронизацию с сетью."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
sync_result = await node_service.sync_manager.sync_with_network()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": sync_result,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting network sync: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/sync/content/{content_hash}")
|
||||
async def get_content_sync_status(content_hash: str):
|
||||
"""Получить статус синхронизации конкретного контента."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
sync_status = await node_service.sync_manager.get_content_sync_status(content_hash)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": sync_status,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting content sync status: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/content/{content_hash}/replicate")
|
||||
async def replicate_content_to_nodes(
|
||||
content_hash: str,
|
||||
replication_config: Dict[str, Any]
|
||||
):
|
||||
"""Реплицировать контент на указанные ноды."""
|
||||
try:
|
||||
target_nodes = replication_config.get("target_nodes", [])
|
||||
if not target_nodes:
|
||||
raise HTTPException(status_code=400, detail="Target nodes are required")
|
||||
|
||||
node_service = get_node_service()
|
||||
replication_result = await node_service.sync_manager.replicate_content_to_nodes(
|
||||
content_hash,
|
||||
target_nodes
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": replication_result,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error replicating content: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/network/stats")
|
||||
async def get_network_stats():
|
||||
"""Получить статистику сети."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
# Получить информацию о ноде и пирах
|
||||
node_info = await node_service.get_node_info()
|
||||
peers_info = await node_service.get_peers_info()
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
# Статистика контента
|
||||
async with get_async_session() as session:
|
||||
# Общее количество контента
|
||||
content_count_stmt = select(func.count(Content.id)).where(Content.is_active == True)
|
||||
content_count_result = await session.execute(content_count_stmt)
|
||||
total_content = content_count_result.scalar()
|
||||
|
||||
# Размер контента
|
||||
size_stmt = select(func.sum(Content.file_size)).where(Content.is_active == True)
|
||||
size_result = await session.execute(size_stmt)
|
||||
total_size = size_result.scalar() or 0
|
||||
|
||||
# Контент по типам
|
||||
type_stmt = select(Content.file_type, func.count(Content.id)).where(Content.is_active == True).group_by(Content.file_type)
|
||||
type_result = await session.execute(type_stmt)
|
||||
content_by_type = {row[0]: row[1] for row in type_result}
|
||||
|
||||
network_stats = {
|
||||
"node_info": {
|
||||
"node_id": node_info["node_id"],
|
||||
"uptime": node_info["uptime"],
|
||||
"version": node_info["version"],
|
||||
"status": node_info["status"]
|
||||
},
|
||||
"network": {
|
||||
"connected_peers": peers_info["peer_count"],
|
||||
"known_peers": len(peers_info["peers"]),
|
||||
"network_health": "good" if peers_info["peer_count"] > 0 else "isolated"
|
||||
},
|
||||
"content": {
|
||||
"total_items": total_content,
|
||||
"total_size_bytes": total_size,
|
||||
"total_size_mb": round(total_size / (1024 * 1024), 2),
|
||||
"content_by_type": content_by_type
|
||||
},
|
||||
"sync": {
|
||||
"active_syncs": sync_status["active_syncs"],
|
||||
"queue_size": sync_status["queue_size"],
|
||||
"is_running": sync_status["is_running"]
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": network_stats,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting network stats: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def health_check():
|
||||
"""Проверка здоровья MY Network ноды."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
# Базовая проверка сервисов
|
||||
health_status = {
|
||||
"status": "healthy",
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"services": {
|
||||
"node_service": node_service is not None,
|
||||
"peer_manager": hasattr(node_service, 'peer_manager') if node_service else False,
|
||||
"sync_manager": hasattr(node_service, 'sync_manager') if node_service else False,
|
||||
"database": True # Если дошли до этой точки, БД работает
|
||||
}
|
||||
}
|
||||
|
||||
# Проверить подключение к пирам
|
||||
if node_service:
|
||||
peers_info = await node_service.get_peers_info()
|
||||
health_status["network"] = {
|
||||
"connected_peers": peers_info["peer_count"],
|
||||
"status": "connected" if peers_info["peer_count"] > 0 else "isolated"
|
||||
}
|
||||
|
||||
# Определить общий статус
|
||||
if not all(health_status["services"].values()):
|
||||
health_status["status"] = "unhealthy"
|
||||
elif node_service and peers_info["peer_count"] == 0:
|
||||
health_status["status"] = "isolated"
|
||||
|
||||
return health_status
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Health check failed: {e}")
|
||||
return {
|
||||
"status": "unhealthy",
|
||||
"error": str(e),
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
"""MY Network Sanic Blueprint - маршруты для работы с распределенной сетью."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
from sanic import Blueprint, Request
|
||||
from sanic.response import json as json_response, file as file_response
|
||||
from sanic.exceptions import SanicException
|
||||
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Создать blueprint для MY Network API
|
||||
bp = Blueprint("my_network", url_prefix="/api/my")
|
||||
|
||||
|
||||
def get_node_service():
|
||||
"""Получить сервис ноды."""
|
||||
try:
|
||||
from app.core.my_network.node_service import get_node_service
|
||||
return get_node_service()
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting node service: {e}")
|
||||
return None
|
||||
|
||||
|
||||
@bp.get("/node/info")
|
||||
async def get_node_info(request: Request):
|
||||
"""Получить информацию о текущей ноде."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
if not node_service:
|
||||
return json_response(
|
||||
{"error": "MY Network service not available"},
|
||||
status=503
|
||||
)
|
||||
|
||||
node_info = await node_service.get_node_info()
|
||||
|
||||
return json_response({
|
||||
"success": True,
|
||||
"data": node_info,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting node info: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
@bp.get("/node/peers")
|
||||
async def get_node_peers(request: Request):
|
||||
"""Получить список подключенных пиров."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
if not node_service:
|
||||
return json_response(
|
||||
{"error": "MY Network service not available"},
|
||||
status=503
|
||||
)
|
||||
|
||||
peers_info = await node_service.get_peers_info()
|
||||
|
||||
return json_response({
|
||||
"success": True,
|
||||
"data": {
|
||||
"connected_peers": peers_info["connected_peers"],
|
||||
"peer_count": peers_info["peer_count"],
|
||||
"peers": peers_info["peers"]
|
||||
},
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting peers: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
@bp.post("/node/peers/connect")
|
||||
async def connect_to_peer(request: Request):
|
||||
"""Подключиться к новому пиру."""
|
||||
try:
|
||||
peer_data = request.json
|
||||
peer_address = peer_data.get("address")
|
||||
|
||||
if not peer_address:
|
||||
return json_response({"error": "Peer address is required"}, status=400)
|
||||
|
||||
node_service = get_node_service()
|
||||
if not node_service:
|
||||
return json_response(
|
||||
{"error": "MY Network service not available"},
|
||||
status=503
|
||||
)
|
||||
|
||||
success = await node_service.peer_manager.connect_to_peer(peer_address)
|
||||
|
||||
if success:
|
||||
return json_response({
|
||||
"success": True,
|
||||
"message": f"Successfully connected to peer: {peer_address}",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
else:
|
||||
return json_response({"error": "Failed to connect to peer"}, status=400)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error connecting to peer: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
@bp.delete("/node/peers/<peer_id>")
|
||||
async def disconnect_peer(request: Request, peer_id: str):
|
||||
"""Отключиться от пира."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
if not node_service:
|
||||
return json_response(
|
||||
{"error": "MY Network service not available"},
|
||||
status=503
|
||||
)
|
||||
|
||||
success = await node_service.peer_manager.disconnect_peer(peer_id)
|
||||
|
||||
if success:
|
||||
return json_response({
|
||||
"success": True,
|
||||
"message": f"Successfully disconnected from peer: {peer_id}",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
else:
|
||||
return json_response(
|
||||
{"error": "Peer not found or already disconnected"},
|
||||
status=404
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error disconnecting peer: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
@bp.get("/content/list")
|
||||
async def get_content_list(request: Request):
|
||||
"""Получить список доступного контента."""
|
||||
try:
|
||||
# Получить параметры запроса
|
||||
limit = min(int(request.args.get("limit", 100)), 1000)
|
||||
offset = max(int(request.args.get("offset", 0)), 0)
|
||||
|
||||
# Кэшировать результат на 5 минут
|
||||
from app.core.cache import cache
|
||||
cache_key = f"my_network:content_list:{limit}:{offset}"
|
||||
cached_result = await cache.get(cache_key)
|
||||
|
||||
if cached_result:
|
||||
return json_response(json.loads(cached_result))
|
||||
|
||||
# Получить контент из БД
|
||||
from app.core.database_compatible import get_async_session
|
||||
from app.core.models.content_compatible import Content, ContentMetadata
|
||||
from sqlalchemy import select, func
|
||||
|
||||
async with get_async_session() as session:
|
||||
stmt = (
|
||||
select(Content, ContentMetadata)
|
||||
.outerjoin(ContentMetadata, Content.id == ContentMetadata.content_id)
|
||||
.where(Content.is_active == True)
|
||||
.order_by(Content.created_at.desc())
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
content_items = []
|
||||
|
||||
for content, metadata in result:
|
||||
content_data = {
|
||||
"hash": content.sha256_hash or content.md5_hash,
|
||||
"filename": content.filename,
|
||||
"original_filename": content.original_filename,
|
||||
"file_size": content.file_size,
|
||||
"file_type": content.file_type,
|
||||
"mime_type": content.mime_type,
|
||||
"created_at": content.created_at.isoformat(),
|
||||
"encrypted": getattr(content, 'encrypted', False),
|
||||
"metadata": metadata.to_dict() if metadata else {}
|
||||
}
|
||||
content_items.append(content_data)
|
||||
|
||||
# Получить общее количество
|
||||
count_stmt = select(func.count(Content.id)).where(Content.is_active == True)
|
||||
count_result = await session.execute(count_stmt)
|
||||
total_count = count_result.scalar()
|
||||
|
||||
response_data = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"content": content_items,
|
||||
"total": total_count,
|
||||
"limit": limit,
|
||||
"offset": offset
|
||||
},
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Кэшировать результат
|
||||
await cache.set(cache_key, json.dumps(response_data), expire=300)
|
||||
|
||||
return json_response(response_data)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting content list: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
@bp.get("/content/<content_hash>/exists")
|
||||
async def check_content_exists(request: Request, content_hash: str):
|
||||
"""Проверить существование контента по хешу."""
|
||||
try:
|
||||
# Кэшировать результат на 30 минут
|
||||
from app.core.cache import cache
|
||||
cache_key = f"my_network:content_exists:{content_hash}"
|
||||
cached_result = await cache.get(cache_key)
|
||||
|
||||
if cached_result is not None:
|
||||
return json_response({"exists": cached_result == "true", "hash": content_hash})
|
||||
|
||||
# Проверить в БД
|
||||
from app.core.database_compatible import get_async_session
|
||||
from app.core.models.content_compatible import Content
|
||||
from sqlalchemy import select, and_
|
||||
|
||||
async with get_async_session() as session:
|
||||
stmt = select(Content.id).where(
|
||||
and_(
|
||||
Content.is_active == True,
|
||||
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
|
||||
)
|
||||
)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
exists = result.scalar_one_or_none() is not None
|
||||
|
||||
# Кэшировать результат
|
||||
await cache.set(cache_key, "true" if exists else "false", expire=1800)
|
||||
|
||||
return json_response({
|
||||
"exists": exists,
|
||||
"hash": content_hash,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking content existence: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
@bp.get("/sync/status")
|
||||
async def get_sync_status(request: Request):
|
||||
"""Получить статус синхронизации."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
if not node_service:
|
||||
return json_response(
|
||||
{"error": "MY Network service not available"},
|
||||
status=503
|
||||
)
|
||||
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
return json_response({
|
||||
"success": True,
|
||||
"data": sync_status,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting sync status: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
@bp.post("/sync/start")
|
||||
async def start_network_sync(request: Request):
|
||||
"""Запустить синхронизацию с сетью."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
if not node_service:
|
||||
return json_response(
|
||||
{"error": "MY Network service not available"},
|
||||
status=503
|
||||
)
|
||||
|
||||
sync_result = await node_service.sync_manager.sync_with_network()
|
||||
|
||||
return json_response({
|
||||
"success": True,
|
||||
"data": sync_result,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting network sync: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
@bp.get("/network/stats")
|
||||
async def get_network_stats(request: Request):
|
||||
"""Получить статистику сети."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
if not node_service:
|
||||
return json_response(
|
||||
{"error": "MY Network service not available"},
|
||||
status=503
|
||||
)
|
||||
|
||||
# Получить информацию о ноде и пирах
|
||||
node_info = await node_service.get_node_info()
|
||||
peers_info = await node_service.get_peers_info()
|
||||
sync_status = await node_service.sync_manager.get_sync_status()
|
||||
|
||||
# Статистика контента
|
||||
from app.core.database_compatible import get_async_session
|
||||
from app.core.models.content_compatible import Content
|
||||
from sqlalchemy import select, func
|
||||
|
||||
async with get_async_session() as session:
|
||||
# Общее количество контента
|
||||
content_count_stmt = select(func.count(Content.id)).where(Content.is_active == True)
|
||||
content_count_result = await session.execute(content_count_stmt)
|
||||
total_content = content_count_result.scalar()
|
||||
|
||||
# Размер контента
|
||||
size_stmt = select(func.sum(Content.file_size)).where(Content.is_active == True)
|
||||
size_result = await session.execute(size_stmt)
|
||||
total_size = size_result.scalar() or 0
|
||||
|
||||
# Контент по типам
|
||||
type_stmt = select(Content.file_type, func.count(Content.id)).where(Content.is_active == True).group_by(Content.file_type)
|
||||
type_result = await session.execute(type_stmt)
|
||||
content_by_type = {row[0]: row[1] for row in type_result}
|
||||
|
||||
network_stats = {
|
||||
"node_info": {
|
||||
"node_id": node_info["node_id"],
|
||||
"uptime": node_info["uptime"],
|
||||
"version": node_info["version"],
|
||||
"status": node_info["status"]
|
||||
},
|
||||
"network": {
|
||||
"connected_peers": peers_info["peer_count"],
|
||||
"known_peers": len(peers_info["peers"]),
|
||||
"network_health": "good" if peers_info["peer_count"] > 0 else "isolated"
|
||||
},
|
||||
"content": {
|
||||
"total_items": total_content,
|
||||
"total_size_bytes": total_size,
|
||||
"total_size_mb": round(total_size / (1024 * 1024), 2),
|
||||
"content_by_type": content_by_type
|
||||
},
|
||||
"sync": {
|
||||
"active_syncs": sync_status["active_syncs"],
|
||||
"queue_size": sync_status["queue_size"],
|
||||
"is_running": sync_status["is_running"]
|
||||
}
|
||||
}
|
||||
|
||||
return json_response({
|
||||
"success": True,
|
||||
"data": network_stats,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting network stats: {e}")
|
||||
return json_response({"error": str(e)}, status=500)
|
||||
|
||||
|
||||
@bp.get("/health")
|
||||
async def health_check(request: Request):
|
||||
"""Проверка здоровья MY Network ноды."""
|
||||
try:
|
||||
node_service = get_node_service()
|
||||
|
||||
# Базовая проверка сервисов
|
||||
health_status = {
|
||||
"status": "healthy",
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"services": {
|
||||
"node_service": node_service is not None,
|
||||
"peer_manager": hasattr(node_service, 'peer_manager') if node_service else False,
|
||||
"sync_manager": hasattr(node_service, 'sync_manager') if node_service else False,
|
||||
"database": True # Если дошли до этой точки, БД работает
|
||||
}
|
||||
}
|
||||
|
||||
# Проверить подключение к пирам
|
||||
if node_service:
|
||||
peers_info = await node_service.get_peers_info()
|
||||
health_status["network"] = {
|
||||
"connected_peers": peers_info["peer_count"],
|
||||
"status": "connected" if peers_info["peer_count"] > 0 else "isolated"
|
||||
}
|
||||
|
||||
# Определить общий статус
|
||||
if not all(health_status["services"].values()):
|
||||
health_status["status"] = "unhealthy"
|
||||
elif node_service and peers_info["peer_count"] == 0:
|
||||
health_status["status"] = "isolated"
|
||||
|
||||
return json_response(health_status)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Health check failed: {e}")
|
||||
return json_response({
|
||||
"status": "unhealthy",
|
||||
"error": str(e),
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}, status=500)
|
||||
@@ -0,0 +1,708 @@
|
||||
"""
|
||||
Storage management routes with chunked uploads, download handling, and file operations.
|
||||
Provides secure file operations with progress tracking and comprehensive validation.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any
|
||||
from uuid import UUID
|
||||
|
||||
from sanic import Blueprint, Request, response
|
||||
from sanic.response import JSONResponse, ResponseStream
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session, get_cache_manager
|
||||
from app.core.logging import get_logger
|
||||
from app.core.storage import StorageManager
|
||||
from app.core.security import validate_file_signature, generate_secure_filename
|
||||
from app.api.middleware import require_auth, validate_request, rate_limit
|
||||
from app.core.validation import StorageUploadSchema, ChunkUploadSchema
|
||||
|
||||
# Initialize blueprint
|
||||
storage_bp = Blueprint("storage", url_prefix="/api/v1/storage")
|
||||
logger = get_logger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
@storage_bp.route("/upload", methods=["POST"])
|
||||
@rate_limit(limit=10, window=3600) # 10 upload sessions per hour
|
||||
@require_auth(permissions=["storage.upload"])
|
||||
@validate_request(StorageUploadSchema)
|
||||
async def initiate_upload(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Initiate chunked file upload session with security validation.
|
||||
|
||||
Args:
|
||||
request: Sanic request with upload parameters
|
||||
|
||||
Returns:
|
||||
JSONResponse: Upload session information
|
||||
"""
|
||||
try:
|
||||
data = request.json
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
# Validate file size against user quota
|
||||
cache_manager = get_cache_manager()
|
||||
quota_key = f"user:{user_id}:storage_quota"
|
||||
current_usage = await cache_manager.get(quota_key, default=0)
|
||||
|
||||
if current_usage + data["file_size"] > settings.MAX_STORAGE_PER_USER:
|
||||
return response.json(
|
||||
{"error": "Storage quota exceeded", "code": "QUOTA_EXCEEDED"},
|
||||
status=429
|
||||
)
|
||||
|
||||
# Generate secure filename
|
||||
secure_filename = generate_secure_filename(data["filename"], user_id)
|
||||
|
||||
# Validate content type
|
||||
allowed_types = {
|
||||
'image/jpeg', 'image/png', 'image/gif', 'image/webp',
|
||||
'video/mp4', 'video/webm', 'video/avi',
|
||||
'audio/mpeg', 'audio/wav', 'audio/flac', 'audio/ogg',
|
||||
'application/pdf', 'text/plain', 'application/json',
|
||||
'application/zip', 'application/x-rar'
|
||||
}
|
||||
|
||||
if data["content_type"] not in allowed_types:
|
||||
return response.json(
|
||||
{"error": "File type not allowed", "code": "TYPE_NOT_ALLOWED"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Create content record first
|
||||
async with get_async_session() as session:
|
||||
from app.core.models.content import Content
|
||||
|
||||
content = Content(
|
||||
user_id=user_id,
|
||||
title=secure_filename,
|
||||
content_type=data["content_type"],
|
||||
file_size=data["file_size"],
|
||||
status="uploading",
|
||||
visibility="private"
|
||||
)
|
||||
|
||||
session.add(content)
|
||||
await session.commit()
|
||||
await session.refresh(content)
|
||||
|
||||
# Create upload session
|
||||
storage_manager = StorageManager()
|
||||
upload_session = await storage_manager.create_upload_session(
|
||||
content.id,
|
||||
data["file_size"]
|
||||
)
|
||||
|
||||
# Update user quota
|
||||
await cache_manager.increment(quota_key, data["file_size"], ttl=86400)
|
||||
|
||||
await logger.ainfo(
|
||||
"Upload session initiated",
|
||||
user_id=str(user_id),
|
||||
content_id=str(content.id),
|
||||
filename=secure_filename,
|
||||
file_size=data["file_size"]
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"upload_session": upload_session,
|
||||
"content_id": str(content.id),
|
||||
"secure_filename": secure_filename,
|
||||
"status": "ready_for_upload"
|
||||
}, status=201)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to initiate upload",
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to initiate upload", "code": "UPLOAD_INIT_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@storage_bp.route("/upload/<upload_id:uuid>/chunk", methods=["POST"])
|
||||
@rate_limit(limit=1000, window=3600) # 1000 chunks per hour
|
||||
@require_auth(permissions=["storage.upload"])
|
||||
async def upload_chunk(request: Request, upload_id: UUID) -> JSONResponse:
|
||||
"""
|
||||
Upload individual file chunk with validation and progress tracking.
|
||||
|
||||
Args:
|
||||
request: Sanic request with chunk data
|
||||
upload_id: Upload session UUID
|
||||
|
||||
Returns:
|
||||
JSONResponse: Chunk upload status
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
# Get chunk data from form
|
||||
if 'chunk' not in request.files:
|
||||
return response.json(
|
||||
{"error": "No chunk data provided", "code": "NO_CHUNK_DATA"},
|
||||
status=400
|
||||
)
|
||||
|
||||
chunk_file = request.files['chunk'][0]
|
||||
chunk_data = chunk_file.body
|
||||
|
||||
# Get chunk metadata
|
||||
chunk_index = int(request.form.get('chunk_index', 0))
|
||||
chunk_hash = request.form.get('chunk_hash', '')
|
||||
is_final = request.form.get('is_final', 'false').lower() == 'true'
|
||||
|
||||
if not chunk_hash:
|
||||
return response.json(
|
||||
{"error": "Chunk hash required", "code": "HASH_REQUIRED"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Validate chunk size
|
||||
if len(chunk_data) > settings.MAX_CHUNK_SIZE:
|
||||
return response.json(
|
||||
{"error": "Chunk too large", "code": "CHUNK_TOO_LARGE"},
|
||||
status=400
|
||||
)
|
||||
|
||||
# Upload chunk
|
||||
storage_manager = StorageManager()
|
||||
result = await storage_manager.upload_chunk(
|
||||
upload_id,
|
||||
chunk_index,
|
||||
chunk_data,
|
||||
chunk_hash
|
||||
)
|
||||
|
||||
# Check if upload is complete
|
||||
if is_final or result["uploaded_chunks"] == result["total_chunks"]:
|
||||
# Finalize upload
|
||||
finalize_result = await storage_manager.finalize_upload(upload_id)
|
||||
result.update(finalize_result)
|
||||
|
||||
await logger.ainfo(
|
||||
"Upload completed",
|
||||
upload_id=str(upload_id),
|
||||
user_id=str(user_id),
|
||||
content_id=finalize_result.get("content_id")
|
||||
)
|
||||
|
||||
return response.json(result)
|
||||
|
||||
except ValueError as e:
|
||||
await logger.awarning(
|
||||
"Chunk upload validation failed",
|
||||
upload_id=str(upload_id),
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": str(e), "code": "VALIDATION_FAILED"},
|
||||
status=400
|
||||
)
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Chunk upload failed",
|
||||
upload_id=str(upload_id),
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Chunk upload failed", "code": "CHUNK_UPLOAD_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@storage_bp.route("/upload/<upload_id:uuid>/status", methods=["GET"])
|
||||
@rate_limit(limit=100, window=3600) # 100 status checks per hour
|
||||
@require_auth(permissions=["storage.upload"])
|
||||
async def get_upload_status(request: Request, upload_id: UUID) -> JSONResponse:
|
||||
"""
|
||||
Get upload session status and progress.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
upload_id: Upload session UUID
|
||||
|
||||
Returns:
|
||||
JSONResponse: Upload progress information
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
storage_manager = StorageManager()
|
||||
|
||||
# Get session data
|
||||
session_data = await storage_manager._get_upload_session(upload_id)
|
||||
|
||||
if not session_data:
|
||||
return response.json(
|
||||
{"error": "Upload session not found", "code": "SESSION_NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
# Verify user ownership
|
||||
async with get_async_session() as session:
|
||||
from app.core.models.content import Content
|
||||
|
||||
stmt = select(Content).where(
|
||||
Content.id == UUID(session_data["content_id"])
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content or content.user_id != user_id:
|
||||
return response.json(
|
||||
{"error": "Access denied", "code": "ACCESS_DENIED"},
|
||||
status=403
|
||||
)
|
||||
|
||||
# Calculate progress
|
||||
uploaded_chunks = len(session_data.get("uploaded_chunks", []))
|
||||
total_chunks = session_data["total_chunks"]
|
||||
progress_percent = (uploaded_chunks / total_chunks * 100) if total_chunks > 0 else 0
|
||||
|
||||
return response.json({
|
||||
"upload_id": str(upload_id),
|
||||
"status": session_data["status"],
|
||||
"progress": {
|
||||
"uploaded_chunks": uploaded_chunks,
|
||||
"total_chunks": total_chunks,
|
||||
"percent": round(progress_percent, 2)
|
||||
},
|
||||
"created_at": session_data["created_at"],
|
||||
"expires_at": session_data["expires_at"]
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get upload status",
|
||||
upload_id=str(upload_id),
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to get upload status", "code": "STATUS_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@storage_bp.route("/upload/<upload_id:uuid>", methods=["DELETE"])
|
||||
@rate_limit(limit=50, window=3600) # 50 cancellations per hour
|
||||
@require_auth(permissions=["storage.upload"])
|
||||
async def cancel_upload(request: Request, upload_id: UUID) -> JSONResponse:
|
||||
"""
|
||||
Cancel upload session and clean up temporary files.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
upload_id: Upload session UUID
|
||||
|
||||
Returns:
|
||||
JSONResponse: Cancellation status
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
storage_manager = StorageManager()
|
||||
|
||||
# Get session data
|
||||
session_data = await storage_manager._get_upload_session(upload_id)
|
||||
|
||||
if not session_data:
|
||||
return response.json(
|
||||
{"error": "Upload session not found", "code": "SESSION_NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
# Verify user ownership
|
||||
content_id = UUID(session_data["content_id"])
|
||||
async with get_async_session() as session:
|
||||
from app.core.models.content import Content
|
||||
|
||||
stmt = select(Content).where(Content.id == content_id)
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content or content.user_id != user_id:
|
||||
return response.json(
|
||||
{"error": "Access denied", "code": "ACCESS_DENIED"},
|
||||
status=403
|
||||
)
|
||||
|
||||
# Delete content record
|
||||
await session.delete(content)
|
||||
await session.commit()
|
||||
|
||||
# Clean up chunks and session
|
||||
cache_manager = get_cache_manager()
|
||||
session_key = f"upload_session:{upload_id}"
|
||||
await cache_manager.delete(session_key)
|
||||
|
||||
# Clean up chunks from storage
|
||||
for chunk_index in session_data.get("uploaded_chunks", []):
|
||||
chunk_id = f"{upload_id}_{chunk_index:06d}"
|
||||
await storage_manager.backend.delete_chunk(chunk_id)
|
||||
|
||||
# Update user quota
|
||||
quota_key = f"user:{user_id}:storage_quota"
|
||||
await cache_manager.decrement(quota_key, session_data.get("total_size", 0))
|
||||
|
||||
await logger.ainfo(
|
||||
"Upload cancelled",
|
||||
upload_id=str(upload_id),
|
||||
user_id=str(user_id),
|
||||
content_id=str(content_id)
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"status": "cancelled",
|
||||
"upload_id": str(upload_id)
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to cancel upload",
|
||||
upload_id=str(upload_id),
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to cancel upload", "code": "CANCEL_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@storage_bp.route("/files/<content_id:uuid>", methods=["DELETE"])
|
||||
@rate_limit(limit=50, window=3600) # 50 deletions per hour
|
||||
@require_auth(permissions=["storage.delete"])
|
||||
async def delete_file(request: Request, content_id: UUID) -> JSONResponse:
|
||||
"""
|
||||
Delete content file and cleanup storage.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
content_id: Content UUID to delete
|
||||
|
||||
Returns:
|
||||
JSONResponse: Deletion status
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
async with get_async_session() as session:
|
||||
from app.core.models.content import Content
|
||||
|
||||
# Get content
|
||||
stmt = select(Content).where(Content.id == content_id)
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content:
|
||||
return response.json(
|
||||
{"error": "Content not found", "code": "NOT_FOUND"},
|
||||
status=404
|
||||
)
|
||||
|
||||
# Check permissions
|
||||
if content.user_id != user_id and not request.ctx.user.is_admin:
|
||||
return response.json(
|
||||
{"error": "Access denied", "code": "ACCESS_DENIED"},
|
||||
status=403
|
||||
)
|
||||
|
||||
# Delete files
|
||||
storage_manager = StorageManager()
|
||||
deletion_success = await storage_manager.delete_content_files(content_id)
|
||||
|
||||
if not deletion_success:
|
||||
await logger.awarning(
|
||||
"File deletion partially failed",
|
||||
content_id=str(content_id),
|
||||
user_id=str(user_id)
|
||||
)
|
||||
|
||||
# Update user quota
|
||||
cache_manager = get_cache_manager()
|
||||
quota_key = f"user:{user_id}:storage_quota"
|
||||
await cache_manager.decrement(quota_key, content.file_size or 0)
|
||||
|
||||
# Clear caches
|
||||
await cache_manager.delete(f"content:{content_id}")
|
||||
await cache_manager.delete(f"content:{content_id}:full")
|
||||
|
||||
await logger.ainfo(
|
||||
"Content deleted",
|
||||
content_id=str(content_id),
|
||||
user_id=str(user_id),
|
||||
file_size=content.file_size
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"status": "deleted",
|
||||
"content_id": str(content_id)
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to delete content",
|
||||
content_id=str(content_id),
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to delete content", "code": "DELETE_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@storage_bp.route("/quota", methods=["GET"])
|
||||
@rate_limit(limit=100, window=3600) # 100 quota checks per hour
|
||||
@require_auth(permissions=["storage.read"])
|
||||
async def get_storage_quota(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Get user storage quota and usage information.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: Quota information
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
# Get current usage from cache
|
||||
cache_manager = get_cache_manager()
|
||||
quota_key = f"user:{user_id}:storage_quota"
|
||||
current_usage = await cache_manager.get(quota_key, default=0)
|
||||
|
||||
# Calculate accurate usage from database
|
||||
async with get_async_session() as session:
|
||||
from sqlalchemy import func
|
||||
from app.core.models.content import Content
|
||||
|
||||
stmt = select(
|
||||
func.count(Content.id).label('file_count'),
|
||||
func.sum(Content.file_size).label('total_size')
|
||||
).where(
|
||||
Content.user_id == user_id,
|
||||
Content.status == 'completed'
|
||||
)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
stats = result.first()
|
||||
|
||||
accurate_usage = stats.total_size or 0
|
||||
file_count = stats.file_count or 0
|
||||
|
||||
# Update cache with accurate value
|
||||
if abs(current_usage - accurate_usage) > 1024: # Update if difference > 1KB
|
||||
await cache_manager.set(quota_key, accurate_usage, ttl=86400)
|
||||
current_usage = accurate_usage
|
||||
|
||||
# Calculate quota information
|
||||
max_quota = settings.MAX_STORAGE_PER_USER
|
||||
usage_percent = (current_usage / max_quota * 100) if max_quota > 0 else 0
|
||||
|
||||
return response.json({
|
||||
"quota": {
|
||||
"used_bytes": current_usage,
|
||||
"max_bytes": max_quota,
|
||||
"available_bytes": max(0, max_quota - current_usage),
|
||||
"usage_percent": round(usage_percent, 2)
|
||||
},
|
||||
"files": {
|
||||
"count": file_count,
|
||||
"max_files": settings.MAX_FILES_PER_USER
|
||||
},
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get storage quota",
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to get quota information", "code": "QUOTA_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@storage_bp.route("/stats", methods=["GET"])
|
||||
@rate_limit(limit=50, window=3600) # 50 stats requests per hour
|
||||
@require_auth(permissions=["storage.read"])
|
||||
async def get_storage_stats(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Get detailed storage statistics for user.
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: Detailed storage statistics
|
||||
"""
|
||||
try:
|
||||
user_id = request.ctx.user.id
|
||||
|
||||
async with get_async_session() as session:
|
||||
from sqlalchemy import func
|
||||
from app.core.models.content import Content
|
||||
|
||||
# Get statistics by content type
|
||||
type_stmt = select(
|
||||
Content.content_type,
|
||||
func.count(Content.id).label('count'),
|
||||
func.sum(Content.file_size).label('size'),
|
||||
func.avg(Content.file_size).label('avg_size')
|
||||
).where(
|
||||
Content.user_id == user_id,
|
||||
Content.status == 'completed'
|
||||
).group_by(Content.content_type)
|
||||
|
||||
type_result = await session.execute(type_stmt)
|
||||
type_stats = {
|
||||
row.content_type: {
|
||||
'count': row.count,
|
||||
'total_size': row.size or 0,
|
||||
'average_size': row.avg_size or 0
|
||||
}
|
||||
for row in type_result
|
||||
}
|
||||
|
||||
# Get upload statistics by month
|
||||
monthly_stmt = select(
|
||||
func.date_trunc('month', Content.created_at).label('month'),
|
||||
func.count(Content.id).label('uploads'),
|
||||
func.sum(Content.file_size).label('size')
|
||||
).where(
|
||||
Content.user_id == user_id,
|
||||
Content.status == 'completed',
|
||||
Content.created_at >= datetime.utcnow().replace(day=1) - timedelta(days=365)
|
||||
).group_by(func.date_trunc('month', Content.created_at))
|
||||
|
||||
monthly_result = await session.execute(monthly_stmt)
|
||||
monthly_stats = [
|
||||
{
|
||||
'month': row.month.isoformat(),
|
||||
'uploads': row.uploads,
|
||||
'size': row.size or 0
|
||||
}
|
||||
for row in monthly_result
|
||||
]
|
||||
|
||||
return response.json({
|
||||
"by_type": type_stats,
|
||||
"monthly": monthly_stats,
|
||||
"generated_at": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get storage stats",
|
||||
user_id=str(user_id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Failed to get storage statistics", "code": "STATS_FAILED"},
|
||||
status=500
|
||||
)
|
||||
|
||||
@storage_bp.route("/cleanup", methods=["POST"])
|
||||
@rate_limit(limit=5, window=3600) # 5 cleanup operations per hour
|
||||
@require_auth(permissions=["storage.admin"])
|
||||
async def cleanup_orphaned_files(request: Request) -> JSONResponse:
|
||||
"""
|
||||
Clean up orphaned files and incomplete uploads (admin only).
|
||||
|
||||
Args:
|
||||
request: Sanic request object
|
||||
|
||||
Returns:
|
||||
JSONResponse: Cleanup results
|
||||
"""
|
||||
try:
|
||||
if not request.ctx.user.is_admin:
|
||||
return response.json(
|
||||
{"error": "Admin access required", "code": "ADMIN_REQUIRED"},
|
||||
status=403
|
||||
)
|
||||
|
||||
storage_manager = StorageManager()
|
||||
cache_manager = get_cache_manager()
|
||||
|
||||
cleanup_stats = {
|
||||
"orphaned_chunks": 0,
|
||||
"expired_sessions": 0,
|
||||
"failed_uploads": 0,
|
||||
"freed_space": 0
|
||||
}
|
||||
|
||||
# Clean up expired upload sessions
|
||||
async with get_async_session() as session:
|
||||
from app.core.models.storage import ContentUploadSession
|
||||
from app.core.models.content import Content
|
||||
|
||||
# Get expired sessions
|
||||
expired_sessions_stmt = select(ContentUploadSession).where(
|
||||
ContentUploadSession.expires_at < datetime.utcnow()
|
||||
)
|
||||
expired_result = await session.execute(expired_sessions_stmt)
|
||||
expired_sessions = expired_result.scalars().all()
|
||||
|
||||
for upload_session in expired_sessions:
|
||||
# Clean up chunks
|
||||
session_key = f"upload_session:{upload_session.id}"
|
||||
session_data = await cache_manager.get(session_key)
|
||||
|
||||
if session_data:
|
||||
for chunk_index in session_data.get("uploaded_chunks", []):
|
||||
chunk_id = f"{upload_session.id}_{chunk_index:06d}"
|
||||
if await storage_manager.backend.delete_chunk(chunk_id):
|
||||
cleanup_stats["orphaned_chunks"] += 1
|
||||
|
||||
# Delete session
|
||||
await session.delete(upload_session)
|
||||
await cache_manager.delete(session_key)
|
||||
cleanup_stats["expired_sessions"] += 1
|
||||
|
||||
# Clean up failed uploads (older than 24 hours)
|
||||
failed_uploads_stmt = select(Content).where(
|
||||
Content.status.in_(['uploading', 'processing', 'failed']),
|
||||
Content.created_at < datetime.utcnow() - timedelta(hours=24)
|
||||
)
|
||||
failed_result = await session.execute(failed_uploads_stmt)
|
||||
failed_uploads = failed_result.scalars().all()
|
||||
|
||||
for content in failed_uploads:
|
||||
if content.file_path:
|
||||
if await storage_manager.backend.delete_file(content.file_path):
|
||||
cleanup_stats["freed_space"] += content.file_size or 0
|
||||
|
||||
await session.delete(content)
|
||||
cleanup_stats["failed_uploads"] += 1
|
||||
|
||||
await session.commit()
|
||||
|
||||
await logger.ainfo(
|
||||
"Storage cleanup completed",
|
||||
**cleanup_stats,
|
||||
admin_user=str(request.ctx.user.id)
|
||||
)
|
||||
|
||||
return response.json({
|
||||
"status": "cleanup_completed",
|
||||
"results": cleanup_stats,
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Storage cleanup failed",
|
||||
admin_user=str(request.ctx.user.id),
|
||||
error=str(e)
|
||||
)
|
||||
return response.json(
|
||||
{"error": "Cleanup operation failed", "code": "CLEANUP_FAILED"},
|
||||
status=500
|
||||
)
|
||||
@@ -1,267 +1,602 @@
|
||||
"""Media conversion service for processing uploaded files."""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
import os
|
||||
import uuid
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import magic # python-magic for MIME detection
|
||||
from base58 import b58decode, b58encode
|
||||
from sqlalchemy import and_, or_
|
||||
from app.core.models.node_storage import StoredContent
|
||||
from app.core.models._telegram import Wrapped_CBotChat
|
||||
from app.core._utils.send_status import send_status
|
||||
from app.core.logger import make_log
|
||||
from app.core.models.user import User
|
||||
from app.core.models import WalletConnection
|
||||
from app.core.storage import db_session
|
||||
from app.core._config import UPLOADS_DIR
|
||||
from app.core.content.content_id import ContentId
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Set, Any, Tuple
|
||||
|
||||
import aiofiles
|
||||
import redis.asyncio as redis
|
||||
from PIL import Image, ImageOps
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session
|
||||
from app.core.models.content import Content, FileUpload
|
||||
from app.core.storage import storage_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def convert_loop(memory):
|
||||
with db_session() as session:
|
||||
# Query for unprocessed encrypted content
|
||||
unprocessed_encrypted_content = session.query(StoredContent).filter(
|
||||
and_(
|
||||
StoredContent.type == "onchain/content",
|
||||
or_(
|
||||
StoredContent.btfs_cid == None,
|
||||
StoredContent.ipfs_cid == None,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if not unprocessed_encrypted_content:
|
||||
make_log("ConvertProcess", "No content to convert", level="debug")
|
||||
return
|
||||
class ConvertService:
|
||||
"""Service for converting and processing uploaded media files."""
|
||||
|
||||
# Достаем расшифрованный файл
|
||||
decrypted_content = session.query(StoredContent).filter(
|
||||
StoredContent.id == unprocessed_encrypted_content.decrypted_content_id
|
||||
).first()
|
||||
if not decrypted_content:
|
||||
make_log("ConvertProcess", "Decrypted content not found", level="error")
|
||||
return
|
||||
|
||||
# Определяем путь и расширение входного файла
|
||||
input_file_path = f"/Storage/storedContent/{decrypted_content.hash}"
|
||||
input_ext = (unprocessed_encrypted_content.filename.split('.')[-1]
|
||||
if '.' in unprocessed_encrypted_content.filename else "mp4")
|
||||
|
||||
# ==== Новая логика: определение MIME-тип через python-magic ====
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
self.redis_client: Optional[redis.Redis] = None
|
||||
self.is_running = False
|
||||
self.tasks: Set[asyncio.Task] = set()
|
||||
|
||||
# Conversion configuration
|
||||
self.batch_size = 10
|
||||
self.process_interval = 5 # seconds
|
||||
self.max_retries = 3
|
||||
self.temp_dir = Path(tempfile.gettempdir()) / "uploader_convert"
|
||||
self.temp_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Supported formats
|
||||
self.image_formats = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff'}
|
||||
self.video_formats = {'.mp4', '.avi', '.mov', '.wmv', '.flv', '.webm', '.mkv'}
|
||||
self.audio_formats = {'.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac'}
|
||||
self.document_formats = {'.pdf', '.doc', '.docx', '.txt', '.rtf'}
|
||||
|
||||
# Image processing settings
|
||||
self.thumbnail_sizes = [(150, 150), (300, 300), (800, 600)]
|
||||
self.image_quality = 85
|
||||
self.max_image_size = (2048, 2048)
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Start the conversion service."""
|
||||
try:
|
||||
mime_type = magic.from_file(input_file_path.replace("/Storage/storedContent", "/app/data"), mime=True)
|
||||
except Exception as e:
|
||||
make_log("ConvertProcess", f"magic probe failed: {e}", level="warning")
|
||||
mime_type = ""
|
||||
|
||||
if mime_type.startswith("video/"):
|
||||
content_kind = "video"
|
||||
elif mime_type.startswith("audio/"):
|
||||
content_kind = "audio"
|
||||
else:
|
||||
content_kind = "other"
|
||||
|
||||
make_log("ConvertProcess", f"Detected content_kind={content_kind}, mime={mime_type}", level="info")
|
||||
|
||||
# Для прочих типов сохраняем raw копию и выходим
|
||||
if content_kind == "other":
|
||||
make_log("ConvertProcess", f"Content {unprocessed_encrypted_content.id} processed. Not audio/video, copy just", level="info")
|
||||
unprocessed_encrypted_content.btfs_cid = ContentId(
|
||||
version=2, content_hash=b58decode(decrypted_content.hash)
|
||||
).serialize_v2()
|
||||
unprocessed_encrypted_content.ipfs_cid = ContentId(
|
||||
version=2, content_hash=b58decode(decrypted_content.hash)
|
||||
).serialize_v2()
|
||||
unprocessed_encrypted_content.meta = {
|
||||
**unprocessed_encrypted_content.meta,
|
||||
'converted_content': {
|
||||
option_name: decrypted_content.hash for option_name in ['high', 'low', 'low_preview']
|
||||
}
|
||||
}
|
||||
session.commit()
|
||||
return
|
||||
|
||||
# ==== Конвертация для видео или аудио: оригинальная логика ====
|
||||
# Static preview interval in seconds
|
||||
preview_interval = [0, 30]
|
||||
if unprocessed_encrypted_content.onchain_index in [2]:
|
||||
preview_interval = [0, 60]
|
||||
|
||||
make_log(
|
||||
"ConvertProcess",
|
||||
f"Processing content {unprocessed_encrypted_content.id} as {content_kind} with preview interval {preview_interval}",
|
||||
level="info"
|
||||
)
|
||||
|
||||
# Выбираем опции конвертации для видео и аудио
|
||||
if content_kind == "video":
|
||||
REQUIRED_CONVERT_OPTIONS = ['high', 'low', 'low_preview']
|
||||
else:
|
||||
REQUIRED_CONVERT_OPTIONS = ['high', 'low'] # no preview for audio
|
||||
|
||||
converted_content = {}
|
||||
logs_dir = "/Storage/logs/converter"
|
||||
|
||||
for option in REQUIRED_CONVERT_OPTIONS:
|
||||
# Set quality parameter and trim option (only for preview)
|
||||
if option == "low_preview":
|
||||
quality = "low"
|
||||
trim_value = f"{preview_interval[0]}-{preview_interval[1]}"
|
||||
else:
|
||||
quality = option
|
||||
trim_value = None
|
||||
|
||||
# Generate a unique output directory for docker container
|
||||
output_uuid = str(uuid.uuid4())
|
||||
output_dir = f"/Storage/storedContent/converter-output/{output_uuid}"
|
||||
|
||||
# Build the docker command
|
||||
cmd = [
|
||||
"docker", "run", "--rm",
|
||||
"-v", f"{input_file_path}:/app/input",
|
||||
"-v", f"{output_dir}:/app/output",
|
||||
"-v", f"{logs_dir}:/app/logs",
|
||||
"media_converter",
|
||||
"--ext", input_ext,
|
||||
"--quality", quality
|
||||
logger.info("Starting media conversion service")
|
||||
|
||||
# Initialize Redis connection
|
||||
self.redis_client = redis.from_url(
|
||||
self.settings.redis_url,
|
||||
encoding="utf-8",
|
||||
decode_responses=True,
|
||||
socket_keepalive=True,
|
||||
socket_keepalive_options={},
|
||||
health_check_interval=30,
|
||||
)
|
||||
|
||||
# Test Redis connection
|
||||
await self.redis_client.ping()
|
||||
logger.info("Redis connection established for converter")
|
||||
|
||||
# Start conversion tasks
|
||||
self.is_running = True
|
||||
|
||||
# Create conversion tasks
|
||||
tasks = [
|
||||
asyncio.create_task(self._process_pending_files_loop()),
|
||||
asyncio.create_task(self._cleanup_temp_files_loop()),
|
||||
asyncio.create_task(self._retry_failed_conversions_loop()),
|
||||
]
|
||||
if trim_value:
|
||||
cmd.extend(["--trim", trim_value])
|
||||
if content_kind == "audio":
|
||||
cmd.append("--audio-only") # audio-only flag
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
if process.returncode != 0:
|
||||
make_log("ConvertProcess", f"Docker conversion failed for option {option}: {stderr.decode()}", level="error")
|
||||
return
|
||||
|
||||
# List files in output dir
|
||||
|
||||
self.tasks.update(tasks)
|
||||
|
||||
# Wait for all tasks
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting conversion service: {e}")
|
||||
await self.stop()
|
||||
raise
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the conversion service."""
|
||||
logger.info("Stopping media conversion service")
|
||||
self.is_running = False
|
||||
|
||||
# Cancel all tasks
|
||||
for task in self.tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
# Wait for tasks to complete
|
||||
if self.tasks:
|
||||
await asyncio.gather(*self.tasks, return_exceptions=True)
|
||||
|
||||
# Close Redis connection
|
||||
if self.redis_client:
|
||||
await self.redis_client.close()
|
||||
|
||||
# Cleanup temp directory
|
||||
await self._cleanup_temp_directory()
|
||||
|
||||
logger.info("Conversion service stopped")
|
||||
|
||||
async def _process_pending_files_loop(self) -> None:
|
||||
"""Main loop for processing pending file conversions."""
|
||||
logger.info("Starting file conversion loop")
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
files = os.listdir(output_dir.replace("/Storage/storedContent", "/app/data"))
|
||||
await self._process_pending_files()
|
||||
await asyncio.sleep(self.process_interval)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
make_log("ConvertProcess", f"Error reading output directory {output_dir}: {e}", level="error")
|
||||
return
|
||||
|
||||
media_files = [f for f in files if f != "output.json"]
|
||||
if len(media_files) != 1:
|
||||
make_log("ConvertProcess", f"Expected one media file, found {len(media_files)} for option {option}", level="error")
|
||||
return
|
||||
|
||||
output_file = os.path.join(
|
||||
output_dir.replace("/Storage/storedContent", "/app/data"),
|
||||
media_files[0]
|
||||
)
|
||||
|
||||
# Compute SHA256 hash of the output file
|
||||
hash_process = await asyncio.create_subprocess_exec(
|
||||
"sha256sum", output_file,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
hash_stdout, hash_stderr = await hash_process.communicate()
|
||||
if hash_process.returncode != 0:
|
||||
make_log("ConvertProcess", f"Error computing sha256sum for option {option}: {hash_stderr.decode()}", level="error")
|
||||
return
|
||||
file_hash = hash_stdout.decode().split()[0]
|
||||
file_hash = b58encode(bytes.fromhex(file_hash)).decode()
|
||||
|
||||
# Save new StoredContent if not exists
|
||||
if not session.query(StoredContent).filter(
|
||||
StoredContent.hash == file_hash
|
||||
).first():
|
||||
new_content = StoredContent(
|
||||
type="local/content_bin",
|
||||
hash=file_hash,
|
||||
user_id=unprocessed_encrypted_content.user_id,
|
||||
filename=media_files[0],
|
||||
meta={'encrypted_file_hash': unprocessed_encrypted_content.hash},
|
||||
created=datetime.now(),
|
||||
logger.error(f"Error in file conversion loop: {e}")
|
||||
await asyncio.sleep(self.process_interval)
|
||||
|
||||
async def _process_pending_files(self) -> None:
|
||||
"""Process pending file conversions."""
|
||||
async with get_async_session() as session:
|
||||
try:
|
||||
# Get pending uploads
|
||||
result = await session.execute(
|
||||
select(FileUpload)
|
||||
.where(
|
||||
FileUpload.status == "uploaded",
|
||||
FileUpload.processed == False
|
||||
)
|
||||
.limit(self.batch_size)
|
||||
)
|
||||
session.add(new_content)
|
||||
session.commit()
|
||||
|
||||
save_path = os.path.join(UPLOADS_DIR, file_hash)
|
||||
try:
|
||||
os.remove(save_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
try:
|
||||
shutil.move(output_file, save_path)
|
||||
uploads = result.scalars().all()
|
||||
|
||||
if not uploads:
|
||||
return
|
||||
|
||||
logger.info(f"Processing {len(uploads)} pending files")
|
||||
|
||||
# Process each upload
|
||||
for upload in uploads:
|
||||
await self._process_single_file(session, upload)
|
||||
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
make_log("ConvertProcess", f"Error moving output file {output_file} to {save_path}: {e}", level="error")
|
||||
return
|
||||
|
||||
converted_content[option] = file_hash
|
||||
|
||||
# Process output.json for ffprobe_meta
|
||||
output_json_path = os.path.join(
|
||||
output_dir.replace("/Storage/storedContent", "/app/data"),
|
||||
"output.json"
|
||||
)
|
||||
if os.path.exists(output_json_path) and unprocessed_encrypted_content.meta.get('ffprobe_meta') is None:
|
||||
try:
|
||||
with open(output_json_path, "r") as f:
|
||||
ffprobe_meta = json.load(f)
|
||||
unprocessed_encrypted_content.meta = {
|
||||
**unprocessed_encrypted_content.meta,
|
||||
'ffprobe_meta': ffprobe_meta
|
||||
}
|
||||
except Exception as e:
|
||||
make_log("ConvertProcess", f"Error handling output.json for option {option}: {e}", level="error")
|
||||
|
||||
# Cleanup output directory
|
||||
try:
|
||||
shutil.rmtree(output_dir.replace("/Storage/storedContent", "/app/data"))
|
||||
except Exception as e:
|
||||
make_log("ConvertProcess", f"Error removing output dir {output_dir}: {e}", level="warning")
|
||||
|
||||
# Finalize original record
|
||||
make_log("ConvertProcess", f"Content {unprocessed_encrypted_content.id} processed. Converted content: {converted_content}", level="info")
|
||||
unprocessed_encrypted_content.btfs_cid = ContentId(
|
||||
version=2, content_hash=b58decode(converted_content['high' if content_kind=='video' else 'low'])
|
||||
).serialize_v2()
|
||||
unprocessed_encrypted_content.ipfs_cid = ContentId(
|
||||
version=2, content_hash=b58decode(converted_content['low'])
|
||||
).serialize_v2()
|
||||
unprocessed_encrypted_content.meta = {
|
||||
**unprocessed_encrypted_content.meta,
|
||||
'converted_content': converted_content
|
||||
}
|
||||
session.commit()
|
||||
|
||||
# Notify user if needed
|
||||
if not unprocessed_encrypted_content.meta.get('upload_notify_msg_id'):
|
||||
wallet_owner_connection = session.query(WalletConnection).filter(
|
||||
WalletConnection.wallet_address == unprocessed_encrypted_content.owner_address
|
||||
).order_by(WalletConnection.id.desc()).first()
|
||||
if wallet_owner_connection:
|
||||
wallet_owner_user = wallet_owner_connection.user
|
||||
bot = Wrapped_CBotChat(
|
||||
memory._client_telegram_bot,
|
||||
chat_id=wallet_owner_user.telegram_id,
|
||||
user=wallet_owner_user,
|
||||
db_session=session
|
||||
)
|
||||
unprocessed_encrypted_content.meta['upload_notify_msg_id'] = await bot.send_content(session, unprocessed_encrypted_content)
|
||||
session.commit()
|
||||
|
||||
|
||||
async def main_fn(memory):
|
||||
make_log("ConvertProcess", "Service started", level="info")
|
||||
seqno = 0
|
||||
while True:
|
||||
logger.error(f"Error processing pending files: {e}")
|
||||
await session.rollback()
|
||||
|
||||
async def _process_single_file(self, session: AsyncSession, upload: FileUpload) -> None:
|
||||
"""Process a single file upload."""
|
||||
try:
|
||||
make_log("ConvertProcess", "Service running", level="debug")
|
||||
await convert_loop(memory)
|
||||
await asyncio.sleep(5)
|
||||
await send_status("convert_service", f"working (seqno={seqno})")
|
||||
seqno += 1
|
||||
except BaseException as e:
|
||||
make_log("ConvertProcess", f"Error: {e}", level="error")
|
||||
await asyncio.sleep(3)
|
||||
logger.info(f"Processing file: {upload.filename}")
|
||||
|
||||
# Mark as processing
|
||||
upload.status = "processing"
|
||||
upload.processing_started_at = datetime.utcnow()
|
||||
await session.commit()
|
||||
|
||||
# Get file extension
|
||||
file_ext = Path(upload.filename).suffix.lower()
|
||||
|
||||
# Process based on file type
|
||||
if file_ext in self.image_formats:
|
||||
await self._process_image(session, upload)
|
||||
elif file_ext in self.video_formats:
|
||||
await self._process_video(session, upload)
|
||||
elif file_ext in self.audio_formats:
|
||||
await self._process_audio(session, upload)
|
||||
elif file_ext in self.document_formats:
|
||||
await self._process_document(session, upload)
|
||||
else:
|
||||
# Just mark as processed for unsupported formats
|
||||
upload.status = "completed"
|
||||
upload.processed = True
|
||||
upload.processing_completed_at = datetime.utcnow()
|
||||
|
||||
# Cache processing result
|
||||
cache_key = f"processed:{upload.id}"
|
||||
processing_info = {
|
||||
"status": upload.status,
|
||||
"processed_at": datetime.utcnow().isoformat(),
|
||||
"metadata": upload.metadata or {}
|
||||
}
|
||||
await self.redis_client.setex(cache_key, 3600, json.dumps(processing_info))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing file {upload.filename}: {e}")
|
||||
|
||||
# Mark as failed
|
||||
upload.status = "failed"
|
||||
upload.error_message = str(e)
|
||||
upload.retry_count = (upload.retry_count or 0) + 1
|
||||
|
||||
if upload.retry_count >= self.max_retries:
|
||||
upload.processed = True # Stop retrying
|
||||
|
||||
async def _process_image(self, session: AsyncSession, upload: FileUpload) -> None:
|
||||
"""Process an image file."""
|
||||
try:
|
||||
# Download original file
|
||||
original_path = await self._download_file(upload)
|
||||
|
||||
if not original_path:
|
||||
raise Exception("Failed to download original file")
|
||||
|
||||
# Open image
|
||||
with Image.open(original_path) as img:
|
||||
# Extract metadata
|
||||
metadata = {
|
||||
"format": img.format,
|
||||
"mode": img.mode,
|
||||
"size": img.size,
|
||||
"has_transparency": img.mode in ('RGBA', 'LA') or 'transparency' in img.info
|
||||
}
|
||||
|
||||
# Fix orientation
|
||||
img = ImageOps.exif_transpose(img)
|
||||
|
||||
# Resize if too large
|
||||
if img.size[0] > self.max_image_size[0] or img.size[1] > self.max_image_size[1]:
|
||||
img.thumbnail(self.max_image_size, Image.Resampling.LANCZOS)
|
||||
metadata["resized"] = True
|
||||
|
||||
# Save optimized version
|
||||
optimized_path = self.temp_dir / f"optimized_{upload.id}.jpg"
|
||||
|
||||
# Convert to RGB if necessary
|
||||
if img.mode in ('RGBA', 'LA'):
|
||||
background = Image.new('RGB', img.size, (255, 255, 255))
|
||||
if img.mode == 'LA':
|
||||
img = img.convert('RGBA')
|
||||
background.paste(img, mask=img.split()[-1])
|
||||
img = background
|
||||
elif img.mode != 'RGB':
|
||||
img = img.convert('RGB')
|
||||
|
||||
img.save(
|
||||
optimized_path,
|
||||
'JPEG',
|
||||
quality=self.image_quality,
|
||||
optimize=True
|
||||
)
|
||||
|
||||
# Upload optimized version
|
||||
optimized_url = await storage_manager.upload_file(
|
||||
str(optimized_path),
|
||||
f"optimized/{upload.id}/image.jpg"
|
||||
)
|
||||
|
||||
# Generate thumbnails
|
||||
thumbnails = {}
|
||||
for size in self.thumbnail_sizes:
|
||||
thumbnail_path = await self._create_thumbnail(original_path, size)
|
||||
if thumbnail_path:
|
||||
thumb_url = await storage_manager.upload_file(
|
||||
str(thumbnail_path),
|
||||
f"thumbnails/{upload.id}/{size[0]}x{size[1]}.jpg"
|
||||
)
|
||||
thumbnails[f"{size[0]}x{size[1]}"] = thumb_url
|
||||
thumbnail_path.unlink() # Cleanup
|
||||
|
||||
# Update upload record
|
||||
upload.metadata = {
|
||||
**metadata,
|
||||
"thumbnails": thumbnails,
|
||||
"optimized_url": optimized_url
|
||||
}
|
||||
upload.status = "completed"
|
||||
upload.processed = True
|
||||
upload.processing_completed_at = datetime.utcnow()
|
||||
|
||||
# Cleanup temp files
|
||||
original_path.unlink()
|
||||
optimized_path.unlink()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing image {upload.filename}: {e}")
|
||||
raise
|
||||
|
||||
async def _process_video(self, session: AsyncSession, upload: FileUpload) -> None:
|
||||
"""Process a video file."""
|
||||
try:
|
||||
# For video processing, we would typically use ffmpeg
|
||||
# This is a simplified version that just extracts basic info
|
||||
|
||||
original_path = await self._download_file(upload)
|
||||
if not original_path:
|
||||
raise Exception("Failed to download original file")
|
||||
|
||||
# Basic video metadata (would use ffprobe in real implementation)
|
||||
metadata = {
|
||||
"type": "video",
|
||||
"file_size": original_path.stat().st_size,
|
||||
"processing_note": "Video processing requires ffmpeg implementation"
|
||||
}
|
||||
|
||||
# Generate video thumbnail (simplified)
|
||||
thumbnail_path = await self._create_video_thumbnail(original_path)
|
||||
if thumbnail_path:
|
||||
thumb_url = await storage_manager.upload_file(
|
||||
str(thumbnail_path),
|
||||
f"thumbnails/{upload.id}/video_thumb.jpg"
|
||||
)
|
||||
metadata["thumbnail"] = thumb_url
|
||||
thumbnail_path.unlink()
|
||||
|
||||
# Update upload record
|
||||
upload.metadata = metadata
|
||||
upload.status = "completed"
|
||||
upload.processed = True
|
||||
upload.processing_completed_at = datetime.utcnow()
|
||||
|
||||
# Cleanup
|
||||
original_path.unlink()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing video {upload.filename}: {e}")
|
||||
raise
|
||||
|
||||
async def _process_audio(self, session: AsyncSession, upload: FileUpload) -> None:
|
||||
"""Process an audio file."""
|
||||
try:
|
||||
original_path = await self._download_file(upload)
|
||||
if not original_path:
|
||||
raise Exception("Failed to download original file")
|
||||
|
||||
# Basic audio metadata
|
||||
metadata = {
|
||||
"type": "audio",
|
||||
"file_size": original_path.stat().st_size,
|
||||
"processing_note": "Audio processing requires additional libraries"
|
||||
}
|
||||
|
||||
# Update upload record
|
||||
upload.metadata = metadata
|
||||
upload.status = "completed"
|
||||
upload.processed = True
|
||||
upload.processing_completed_at = datetime.utcnow()
|
||||
|
||||
# Cleanup
|
||||
original_path.unlink()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing audio {upload.filename}: {e}")
|
||||
raise
|
||||
|
||||
async def _process_document(self, session: AsyncSession, upload: FileUpload) -> None:
|
||||
"""Process a document file."""
|
||||
try:
|
||||
original_path = await self._download_file(upload)
|
||||
if not original_path:
|
||||
raise Exception("Failed to download original file")
|
||||
|
||||
# Basic document metadata
|
||||
metadata = {
|
||||
"type": "document",
|
||||
"file_size": original_path.stat().st_size,
|
||||
"pages": 1, # Would extract actual page count for PDFs
|
||||
"processing_note": "Document processing requires additional libraries"
|
||||
}
|
||||
|
||||
# Update upload record
|
||||
upload.metadata = metadata
|
||||
upload.status = "completed"
|
||||
upload.processed = True
|
||||
upload.processing_completed_at = datetime.utcnow()
|
||||
|
||||
# Cleanup
|
||||
original_path.unlink()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing document {upload.filename}: {e}")
|
||||
raise
|
||||
|
||||
async def _download_file(self, upload: FileUpload) -> Optional[Path]:
|
||||
"""Download a file for processing."""
|
||||
try:
|
||||
if not upload.file_path:
|
||||
return None
|
||||
|
||||
# Create temp file path
|
||||
temp_path = self.temp_dir / f"original_{upload.id}_{upload.filename}"
|
||||
|
||||
# Download file from storage
|
||||
file_data = await storage_manager.get_file(upload.file_path)
|
||||
if not file_data:
|
||||
return None
|
||||
|
||||
# Write to temp file
|
||||
async with aiofiles.open(temp_path, 'wb') as f:
|
||||
await f.write(file_data)
|
||||
|
||||
return temp_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading file {upload.filename}: {e}")
|
||||
return None
|
||||
|
||||
async def _create_thumbnail(self, image_path: Path, size: Tuple[int, int]) -> Optional[Path]:
|
||||
"""Create a thumbnail from an image."""
|
||||
try:
|
||||
thumbnail_path = self.temp_dir / f"thumb_{size[0]}x{size[1]}_{image_path.name}"
|
||||
|
||||
with Image.open(image_path) as img:
|
||||
# Fix orientation
|
||||
img = ImageOps.exif_transpose(img)
|
||||
|
||||
# Create thumbnail
|
||||
img.thumbnail(size, Image.Resampling.LANCZOS)
|
||||
|
||||
# Convert to RGB if necessary
|
||||
if img.mode in ('RGBA', 'LA'):
|
||||
background = Image.new('RGB', img.size, (255, 255, 255))
|
||||
if img.mode == 'LA':
|
||||
img = img.convert('RGBA')
|
||||
background.paste(img, mask=img.split()[-1])
|
||||
img = background
|
||||
elif img.mode != 'RGB':
|
||||
img = img.convert('RGB')
|
||||
|
||||
# Save thumbnail
|
||||
img.save(
|
||||
thumbnail_path,
|
||||
'JPEG',
|
||||
quality=self.image_quality,
|
||||
optimize=True
|
||||
)
|
||||
|
||||
return thumbnail_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating thumbnail: {e}")
|
||||
return None
|
||||
|
||||
async def _create_video_thumbnail(self, video_path: Path) -> Optional[Path]:
|
||||
"""Create a thumbnail from a video file."""
|
||||
try:
|
||||
# This would require ffmpeg to extract a frame from the video
|
||||
# For now, return a placeholder
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating video thumbnail: {e}")
|
||||
return None
|
||||
|
||||
async def _cleanup_temp_files_loop(self) -> None:
|
||||
"""Loop for cleaning up temporary files."""
|
||||
logger.info("Starting temp file cleanup loop")
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
await self._cleanup_old_temp_files()
|
||||
await asyncio.sleep(3600) # Run every hour
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error in temp cleanup loop: {e}")
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
async def _cleanup_old_temp_files(self) -> None:
|
||||
"""Clean up old temporary files."""
|
||||
try:
|
||||
current_time = datetime.now().timestamp()
|
||||
|
||||
for file_path in self.temp_dir.glob("*"):
|
||||
if file_path.is_file():
|
||||
# Remove files older than 1 hour
|
||||
if current_time - file_path.stat().st_mtime > 3600:
|
||||
file_path.unlink()
|
||||
logger.debug(f"Removed old temp file: {file_path}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning up temp files: {e}")
|
||||
|
||||
async def _cleanup_temp_directory(self) -> None:
|
||||
"""Clean up the entire temp directory."""
|
||||
try:
|
||||
for file_path in self.temp_dir.glob("*"):
|
||||
if file_path.is_file():
|
||||
file_path.unlink()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning up temp directory: {e}")
|
||||
|
||||
async def _retry_failed_conversions_loop(self) -> None:
|
||||
"""Loop for retrying failed conversions."""
|
||||
logger.info("Starting retry loop for failed conversions")
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
await self._retry_failed_conversions()
|
||||
await asyncio.sleep(1800) # Run every 30 minutes
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error in retry loop: {e}")
|
||||
await asyncio.sleep(1800)
|
||||
|
||||
async def _retry_failed_conversions(self) -> None:
|
||||
"""Retry failed conversions that haven't exceeded max retries."""
|
||||
async with get_async_session() as session:
|
||||
try:
|
||||
# Get failed uploads that can be retried
|
||||
result = await session.execute(
|
||||
select(FileUpload)
|
||||
.where(
|
||||
FileUpload.status == "failed",
|
||||
FileUpload.processed == False,
|
||||
(FileUpload.retry_count < self.max_retries) | (FileUpload.retry_count.is_(None))
|
||||
)
|
||||
.limit(5) # Smaller batch for retries
|
||||
)
|
||||
uploads = result.scalars().all()
|
||||
|
||||
for upload in uploads:
|
||||
logger.info(f"Retrying failed conversion for: {upload.filename}")
|
||||
|
||||
# Reset status
|
||||
upload.status = "uploaded"
|
||||
upload.error_message = None
|
||||
|
||||
# Process the file
|
||||
await self._process_single_file(session, upload)
|
||||
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error retrying failed conversions: {e}")
|
||||
await session.rollback()
|
||||
|
||||
async def queue_file_for_processing(self, upload_id: str) -> bool:
|
||||
"""Queue a file for processing."""
|
||||
try:
|
||||
# Add to processing queue
|
||||
queue_key = "conversion_queue"
|
||||
await self.redis_client.lpush(queue_key, upload_id)
|
||||
|
||||
logger.info(f"Queued file {upload_id} for processing")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error queuing file for processing: {e}")
|
||||
return False
|
||||
|
||||
async def get_processing_stats(self) -> Dict[str, Any]:
|
||||
"""Get processing statistics."""
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
# Get upload stats by status
|
||||
status_result = await session.execute(
|
||||
select(FileUpload.status, asyncio.func.count())
|
||||
.group_by(FileUpload.status)
|
||||
)
|
||||
status_stats = dict(status_result.fetchall())
|
||||
|
||||
# Get processing stats
|
||||
processed_result = await session.execute(
|
||||
select(asyncio.func.count())
|
||||
.select_from(FileUpload)
|
||||
.where(FileUpload.processed == True)
|
||||
)
|
||||
processed_count = processed_result.scalar()
|
||||
|
||||
# Get failed stats
|
||||
failed_result = await session.execute(
|
||||
select(asyncio.func.count())
|
||||
.select_from(FileUpload)
|
||||
.where(FileUpload.status == "failed")
|
||||
)
|
||||
failed_count = failed_result.scalar()
|
||||
|
||||
return {
|
||||
"status_stats": status_stats,
|
||||
"processed_count": processed_count,
|
||||
"failed_count": failed_count,
|
||||
"is_running": self.is_running,
|
||||
"active_tasks": len([t for t in self.tasks if not t.done()]),
|
||||
"temp_files": len(list(self.temp_dir.glob("*"))),
|
||||
"last_update": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting processing stats: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
# Global converter instance
|
||||
convert_service = ConvertService()
|
||||
@@ -1,313 +1,500 @@
|
||||
"""Blockchain indexer service for monitoring transactions and events."""
|
||||
|
||||
import asyncio
|
||||
from base64 import b64decode
|
||||
from datetime import datetime
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Set, Any
|
||||
|
||||
from base58 import b58encode
|
||||
from sqlalchemy import String, and_, desc, cast
|
||||
from tonsdk.boc import Cell
|
||||
from tonsdk.utils import Address
|
||||
from app.core._config import CLIENT_TELEGRAM_BOT_USERNAME
|
||||
from app.core._blockchain.ton.platform import platform
|
||||
from app.core._blockchain.ton.toncenter import toncenter
|
||||
from app.core._utils.send_status import send_status
|
||||
from app.core.logger import make_log
|
||||
from app.core.models import UserContent, KnownTelegramMessage, ServiceConfig
|
||||
from app.core.models.node_storage import StoredContent
|
||||
from app.core._utils.resolve_content import resolve_content
|
||||
from app.core.models.wallet_connection import WalletConnection
|
||||
from app.core._keyboards import get_inline_keyboard
|
||||
from app.core.models._telegram import Wrapped_CBotChat
|
||||
from app.core.storage import db_session
|
||||
import os
|
||||
import traceback
|
||||
import redis.asyncio as redis
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session
|
||||
from app.core.models.blockchain import Transaction, Wallet, BlockchainNFT, BlockchainTokenBalance
|
||||
from app.core.background.ton_service import TONService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def indexer_loop(memory, platform_found: bool, seqno: int) -> [bool, int]:
|
||||
if not platform_found:
|
||||
platform_state = await toncenter.get_account(platform.address.to_string(1, 1, 1))
|
||||
if not platform_state.get('code'):
|
||||
make_log("TON", "Platform contract is not deployed, skipping loop", level="info")
|
||||
await send_status("indexer", "not working: platform is not deployed")
|
||||
return False, seqno
|
||||
else:
|
||||
platform_found = True
|
||||
class IndexerService:
|
||||
"""Service for indexing blockchain transactions and events."""
|
||||
|
||||
make_log("Indexer", "Service running", level="debug")
|
||||
with db_session() as session:
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
self.ton_service = TONService()
|
||||
self.redis_client: Optional[redis.Redis] = None
|
||||
self.is_running = False
|
||||
self.tasks: Set[asyncio.Task] = set()
|
||||
|
||||
# Indexing configuration
|
||||
self.batch_size = 100
|
||||
self.index_interval = 30 # seconds
|
||||
self.confirmation_blocks = 12
|
||||
self.max_retries = 3
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Start the indexer service."""
|
||||
try:
|
||||
result = await toncenter.run_get_method('EQD8TJ8xEWB1SpnRE4d89YO3jl0W0EiBnNS4IBaHaUmdfizE', 'get_pool_data')
|
||||
assert result['exit_code'] == 0, f"Error in get-method: {result}"
|
||||
assert result['stack'][0][0] == 'num', f"get first element is not num"
|
||||
assert result['stack'][1][0] == 'num', f"get second element is not num"
|
||||
usdt_per_ton = (int(result['stack'][0][1], 16) * 1e3) / int(result['stack'][1][1], 16)
|
||||
ton_per_star = 0.014 / usdt_per_ton
|
||||
ServiceConfig(session).set('live_tonPerStar', [ton_per_star, datetime.utcnow().timestamp()])
|
||||
make_log("TON_Daemon", f"TON per STAR price: {ton_per_star}", level="DEBUG")
|
||||
except BaseException as e:
|
||||
make_log("TON_Daemon", f"Error while saving TON per STAR price: {e}" + '\n' + traceback.format_exc(), level="ERROR")
|
||||
|
||||
new_licenses = session.query(UserContent).filter(
|
||||
and_(
|
||||
~UserContent.meta.contains({'notification_sent': True}),
|
||||
UserContent.type == 'nft/listen'
|
||||
logger.info("Starting blockchain indexer service")
|
||||
|
||||
# Initialize Redis connection
|
||||
self.redis_client = redis.from_url(
|
||||
self.settings.redis_url,
|
||||
encoding="utf-8",
|
||||
decode_responses=True,
|
||||
socket_keepalive=True,
|
||||
socket_keepalive_options={},
|
||||
health_check_interval=30,
|
||||
)
|
||||
).all()
|
||||
for new_license in new_licenses:
|
||||
licensed_content = session.query(StoredContent).filter(
|
||||
StoredContent.id == new_license.content_id
|
||||
).first()
|
||||
if not licensed_content:
|
||||
make_log("Indexer", f"Licensed content not found: {new_license.content_id}", level="error")
|
||||
|
||||
content_metadata = licensed_content.metadata_json(session)
|
||||
assert content_metadata, "No content metadata found"
|
||||
|
||||
if not (licensed_content.owner_address == new_license.owner_address):
|
||||
try:
|
||||
user = new_license.user
|
||||
if user.telegram_id and licensed_content:
|
||||
await (Wrapped_CBotChat(memory._client_telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)).send_content(
|
||||
session, licensed_content
|
||||
)
|
||||
|
||||
wallet_owner_connection = session.query(WalletConnection).filter_by(
|
||||
wallet_address=licensed_content.owner_address,
|
||||
invalidated=False
|
||||
).order_by(desc(WalletConnection.id)).first()
|
||||
wallet_owner_user = wallet_owner_connection.user
|
||||
if wallet_owner_user.telegram_id:
|
||||
wallet_owner_bot = Wrapped_CBotChat(memory._telegram_bot, chat_id=wallet_owner_user.telegram_id, user=wallet_owner_user, db_session=session)
|
||||
await wallet_owner_bot.send_message(
|
||||
user.translated('p_licenseWasBought').format(
|
||||
username=user.front_format(),
|
||||
nft_address=f'"https://tonviewer.com/{new_license.onchain_address}"',
|
||||
content_title=content_metadata.get('name', 'Unknown'),
|
||||
),
|
||||
message_type='notification',
|
||||
)
|
||||
except BaseException as e:
|
||||
make_log("IndexerSendNewLicense", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
|
||||
new_license.meta = {**new_license.meta, 'notification_sent': True}
|
||||
session.commit()
|
||||
|
||||
content_without_cid = session.query(StoredContent).filter(
|
||||
StoredContent.content_id == None
|
||||
)
|
||||
for target_content in content_without_cid:
|
||||
target_cid = target_content.cid.serialize_v2()
|
||||
make_log("Indexer", f"Content without CID: {target_content.hash}, setting CID: {target_cid}", level="debug")
|
||||
target_content.content_id = target_cid
|
||||
|
||||
session.commit()
|
||||
|
||||
last_known_index_ = session.query(StoredContent).filter(
|
||||
StoredContent.onchain_index != None
|
||||
).order_by(StoredContent.onchain_index.desc()).first()
|
||||
last_known_index = last_known_index_.onchain_index if last_known_index_ else 0
|
||||
last_known_index = max(last_known_index, 0)
|
||||
make_log("Indexer", f"Last known index: {last_known_index}", level="debug")
|
||||
if last_known_index_:
|
||||
next_item_index = last_known_index + 1
|
||||
else:
|
||||
next_item_index = 0
|
||||
|
||||
resolve_item_result = await toncenter.run_get_method(platform.address.to_string(1, 1, 1), 'get_nft_address_by_index', [['num', next_item_index]])
|
||||
make_log("Indexer", f"Resolve item result: {resolve_item_result}", level="debug")
|
||||
if resolve_item_result.get('exit_code', -1) != 0:
|
||||
make_log("Indexer", f"Resolve item error: {resolve_item_result}", level="error")
|
||||
return platform_found, seqno
|
||||
|
||||
item_address_cell_b64 = resolve_item_result['stack'][0][1]["bytes"]
|
||||
item_address_slice = Cell.one_from_boc(b64decode(item_address_cell_b64)).begin_parse()
|
||||
item_address = item_address_slice.read_msg_addr()
|
||||
make_log("Indexer", f"Item address: {item_address.to_string(1, 1, 1)}", level="debug")
|
||||
|
||||
item_get_data_result = await toncenter.run_get_method(item_address.to_string(1, 1, 1), 'indexator_data')
|
||||
if item_get_data_result.get('exit_code', -1) != 0:
|
||||
make_log("Indexer", f"Get item data error (maybe not deployed): {item_get_data_result}", level="debug")
|
||||
return platform_found, seqno
|
||||
|
||||
assert item_get_data_result['stack'][0][0] == 'num', "Item type is not a number"
|
||||
assert int(item_get_data_result['stack'][0][1], 16) == 1, "Item is not COP NFT"
|
||||
item_returned_address = Cell.one_from_boc(b64decode(item_get_data_result['stack'][1][1]['bytes'])).begin_parse().read_msg_addr()
|
||||
assert (
|
||||
item_returned_address.to_string(1, 1, 1) == item_address.to_string(1, 1, 1)
|
||||
), "Item address mismatch"
|
||||
|
||||
assert item_get_data_result['stack'][2][0] == 'num', "Item index is not a number"
|
||||
item_index = int(item_get_data_result['stack'][2][1], 16)
|
||||
assert item_index == next_item_index, "Item index mismatch"
|
||||
|
||||
item_platform_address = Cell.one_from_boc(b64decode(item_get_data_result['stack'][3][1]['bytes'])).begin_parse().read_msg_addr()
|
||||
assert item_platform_address.to_string(1, 1, 1) == Address(platform.address.to_string(1, 1, 1)).to_string(1, 1, 1), "Item platform address mismatch"
|
||||
|
||||
assert item_get_data_result['stack'][4][0] == 'num', "Item license type is not a number"
|
||||
item_license_type = int(item_get_data_result['stack'][4][1], 16)
|
||||
assert item_license_type == 0, "Item license type is not 0"
|
||||
|
||||
item_owner_address = Cell.one_from_boc(b64decode(item_get_data_result['stack'][5][1]["bytes"])).begin_parse().read_msg_addr()
|
||||
item_values = Cell.one_from_boc(b64decode(item_get_data_result['stack'][6][1]['bytes']))
|
||||
item_derivates = Cell.one_from_boc(b64decode(item_get_data_result['stack'][7][1]['bytes']))
|
||||
item_platform_variables = Cell.one_from_boc(b64decode(item_get_data_result['stack'][8][1]['bytes']))
|
||||
item_distribution = Cell.one_from_boc(b64decode(item_get_data_result['stack'][9][1]['bytes']))
|
||||
|
||||
item_distribution_slice = item_distribution.begin_parse()
|
||||
item_prices_slice = item_distribution_slice.refs[0].begin_parse()
|
||||
item_listen_license_price = item_prices_slice.read_coins()
|
||||
item_use_license_price = item_prices_slice.read_coins()
|
||||
item_resale_license_price = item_prices_slice.read_coins()
|
||||
|
||||
item_values_slice = item_values.begin_parse()
|
||||
item_content_hash_int = item_values_slice.read_uint(256)
|
||||
item_content_hash = item_content_hash_int.to_bytes(32, 'big')
|
||||
# item_content_hash_str = b58encode(item_content_hash).decode()
|
||||
item_metadata = item_values_slice.refs[0]
|
||||
item_content = item_values_slice.refs[1]
|
||||
item_metadata_str = item_metadata.bits.array.decode()
|
||||
item_content_cid_str = item_content.refs[0].bits.array.decode()
|
||||
item_content_cover_cid_str = item_content.refs[1].bits.array.decode()
|
||||
item_content_metadata_cid_str = item_content.refs[2].bits.array.decode()
|
||||
|
||||
item_content_cid, err = resolve_content(item_content_cid_str)
|
||||
item_content_hash = item_content_cid.content_hash
|
||||
item_content_hash_str = item_content_cid.content_hash_b58
|
||||
|
||||
item_metadata_packed = {
|
||||
'license_type': item_license_type,
|
||||
'item_address': item_address.to_string(1, 1, 1),
|
||||
'content_cid': item_content_cid_str,
|
||||
'cover_cid': item_content_cover_cid_str,
|
||||
'metadata_cid': item_content_metadata_cid_str,
|
||||
'derivates': b58encode(item_derivates.to_boc(False)).decode(),
|
||||
'platform_variables': b58encode(item_platform_variables.to_boc(False)).decode(),
|
||||
'license': {
|
||||
'listen': {
|
||||
'price': str(item_listen_license_price)
|
||||
},
|
||||
'use': {
|
||||
'price': str(item_use_license_price)
|
||||
},
|
||||
'resale': {
|
||||
'price': str(item_resale_license_price)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
user_wallet_connection = None
|
||||
if item_owner_address:
|
||||
user_wallet_connection = session.query(WalletConnection).filter(
|
||||
WalletConnection.wallet_address == item_owner_address.to_string(1, 1, 1)
|
||||
).first()
|
||||
|
||||
encrypted_stored_content = session.query(StoredContent).filter(
|
||||
StoredContent.hash == item_content_hash_str,
|
||||
# StoredContent.type.like("local%")
|
||||
).first()
|
||||
if encrypted_stored_content:
|
||||
is_duplicate = encrypted_stored_content.type.startswith("onchain") \
|
||||
and encrypted_stored_content.onchain_index != item_index
|
||||
if not is_duplicate:
|
||||
if encrypted_stored_content.type.startswith('local'):
|
||||
encrypted_stored_content.type = "onchain/content" + ("_unknown" if (encrypted_stored_content.key_id is None) else "")
|
||||
encrypted_stored_content.onchain_index = item_index
|
||||
encrypted_stored_content.owner_address = item_owner_address.to_string(1, 1, 1)
|
||||
user = None
|
||||
if user_wallet_connection:
|
||||
encrypted_stored_content.user_id = user_wallet_connection.user_id
|
||||
user = user_wallet_connection.user
|
||||
|
||||
if user:
|
||||
user_uploader_wrapper = Wrapped_CBotChat(memory._telegram_bot, chat_id=user.telegram_id, user=user, db_session=session)
|
||||
await user_uploader_wrapper.send_message(
|
||||
user.translated('p_contentWasIndexed').format(
|
||||
item_address=item_address.to_string(1, 1, 1),
|
||||
item_index=item_index,
|
||||
),
|
||||
message_type='notification',
|
||||
reply_markup=get_inline_keyboard([
|
||||
[{
|
||||
'text': user.translated('viewTrackAsClient_button'),
|
||||
'url': f"https://t.me/{CLIENT_TELEGRAM_BOT_USERNAME}?start=C{encrypted_stored_content.cid.serialize_v2()}"
|
||||
}],
|
||||
])
|
||||
)
|
||||
|
||||
try:
|
||||
for hint_message in session.query(KnownTelegramMessage).filter(
|
||||
and_(
|
||||
KnownTelegramMessage.chat_id == user.telegram_id,
|
||||
KnownTelegramMessage.type == 'hint',
|
||||
cast(KnownTelegramMessage.meta['encrypted_content_hash'], String) == encrypted_stored_content.hash,
|
||||
KnownTelegramMessage.deleted == False
|
||||
)
|
||||
).all():
|
||||
await user_uploader_wrapper.delete_message(hint_message.message_id)
|
||||
except BaseException as e:
|
||||
make_log("Indexer", f"Error while deleting hint messages: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
elif encrypted_stored_content.type.startswith('onchain') and encrypted_stored_content.onchain_index == item_index:
|
||||
encrypted_stored_content.type = "onchain/content" + ("_unknown" if (encrypted_stored_content.key_id is None) else "")
|
||||
encrypted_stored_content.owner_address = item_owner_address.to_string(1, 1, 1)
|
||||
if user_wallet_connection:
|
||||
encrypted_stored_content.user_id = user_wallet_connection.user_id
|
||||
else:
|
||||
make_log("Indexer", f"[CRITICAL] Item already indexed and ERRORED!: {item_content_hash_str}", level="error")
|
||||
return platform_found, seqno
|
||||
|
||||
encrypted_stored_content.updated = datetime.now()
|
||||
encrypted_stored_content.meta = {
|
||||
**encrypted_stored_content.meta,
|
||||
**item_metadata_packed
|
||||
}
|
||||
|
||||
session.commit()
|
||||
return platform_found, seqno
|
||||
else:
|
||||
item_metadata_packed['copied_from'] = encrypted_stored_content.id
|
||||
item_metadata_packed['copied_from_cid'] = encrypted_stored_content.cid.serialize_v2()
|
||||
item_content_hash_str = f"{b58encode(bytes(16) + os.urandom(30)).decode()}" # check this for vulnerability
|
||||
|
||||
onchain_stored_content = StoredContent(
|
||||
type="onchain/content_unknown",
|
||||
hash=item_content_hash_str,
|
||||
onchain_index=item_index,
|
||||
owner_address=item_owner_address.to_string(1, 1, 1) if item_owner_address else None,
|
||||
meta=item_metadata_packed,
|
||||
filename="UNKNOWN_ENCRYPTED_CONTENT",
|
||||
user_id=user_wallet_connection.user_id if user_wallet_connection else None,
|
||||
created=datetime.now(),
|
||||
encrypted=True,
|
||||
decrypted_content_id=None,
|
||||
key_id=None,
|
||||
updated=datetime.now()
|
||||
)
|
||||
session.add(onchain_stored_content)
|
||||
session.commit()
|
||||
make_log("Indexer", f"Item indexed: {item_content_hash_str}", level="info")
|
||||
last_known_index += 1
|
||||
|
||||
return platform_found, seqno
|
||||
|
||||
|
||||
async def main_fn(memory, ):
|
||||
make_log("Indexer", "Service started", level="info")
|
||||
platform_found = False
|
||||
seqno = 0
|
||||
while True:
|
||||
|
||||
# Test Redis connection
|
||||
await self.redis_client.ping()
|
||||
logger.info("Redis connection established for indexer")
|
||||
|
||||
# Start indexing tasks
|
||||
self.is_running = True
|
||||
|
||||
# Create indexing tasks
|
||||
tasks = [
|
||||
asyncio.create_task(self._index_transactions_loop()),
|
||||
asyncio.create_task(self._index_wallets_loop()),
|
||||
asyncio.create_task(self._index_nfts_loop()),
|
||||
asyncio.create_task(self._index_token_balances_loop()),
|
||||
asyncio.create_task(self._cleanup_cache_loop()),
|
||||
]
|
||||
|
||||
self.tasks.update(tasks)
|
||||
|
||||
# Wait for all tasks
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting indexer service: {e}")
|
||||
await self.stop()
|
||||
raise
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the indexer service."""
|
||||
logger.info("Stopping blockchain indexer service")
|
||||
self.is_running = False
|
||||
|
||||
# Cancel all tasks
|
||||
for task in self.tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
# Wait for tasks to complete
|
||||
if self.tasks:
|
||||
await asyncio.gather(*self.tasks, return_exceptions=True)
|
||||
|
||||
# Close Redis connection
|
||||
if self.redis_client:
|
||||
await self.redis_client.close()
|
||||
|
||||
logger.info("Indexer service stopped")
|
||||
|
||||
async def _index_transactions_loop(self) -> None:
|
||||
"""Main loop for indexing transactions."""
|
||||
logger.info("Starting transaction indexing loop")
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
await self._index_pending_transactions()
|
||||
await self._update_transaction_confirmations()
|
||||
await asyncio.sleep(self.index_interval)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error in transaction indexing loop: {e}")
|
||||
await asyncio.sleep(self.index_interval)
|
||||
|
||||
async def _index_wallets_loop(self) -> None:
|
||||
"""Loop for updating wallet information."""
|
||||
logger.info("Starting wallet indexing loop")
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
await self._update_wallet_balances()
|
||||
await asyncio.sleep(self.index_interval * 2) # Less frequent
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error in wallet indexing loop: {e}")
|
||||
await asyncio.sleep(self.index_interval * 2)
|
||||
|
||||
async def _index_nfts_loop(self) -> None:
|
||||
"""Loop for indexing NFT collections and transfers."""
|
||||
logger.info("Starting NFT indexing loop")
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
await self._index_nft_collections()
|
||||
await self._index_nft_transfers()
|
||||
await asyncio.sleep(self.index_interval * 4) # Even less frequent
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error in NFT indexing loop: {e}")
|
||||
await asyncio.sleep(self.index_interval * 4)
|
||||
|
||||
async def _index_token_balances_loop(self) -> None:
|
||||
"""Loop for updating token balances."""
|
||||
logger.info("Starting token balance indexing loop")
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
await self._update_token_balances()
|
||||
await asyncio.sleep(self.index_interval * 3)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error in token balance indexing loop: {e}")
|
||||
await asyncio.sleep(self.index_interval * 3)
|
||||
|
||||
async def _cleanup_cache_loop(self) -> None:
|
||||
"""Loop for cleaning up old cache entries."""
|
||||
logger.info("Starting cache cleanup loop")
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
await self._cleanup_old_cache_entries()
|
||||
await asyncio.sleep(3600) # Run every hour
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error in cache cleanup loop: {e}")
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
async def _index_pending_transactions(self) -> None:
|
||||
"""Index pending transactions from the database."""
|
||||
async with get_async_session() as session:
|
||||
try:
|
||||
# Get pending transactions
|
||||
result = await session.execute(
|
||||
select(Transaction)
|
||||
.where(Transaction.status == "pending")
|
||||
.limit(self.batch_size)
|
||||
)
|
||||
transactions = result.scalars().all()
|
||||
|
||||
if not transactions:
|
||||
return
|
||||
|
||||
logger.info(f"Indexing {len(transactions)} pending transactions")
|
||||
|
||||
# Process each transaction
|
||||
for transaction in transactions:
|
||||
await self._process_transaction(session, transaction)
|
||||
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error indexing pending transactions: {e}")
|
||||
await session.rollback()
|
||||
|
||||
async def _process_transaction(self, session: AsyncSession, transaction: Transaction) -> None:
|
||||
"""Process a single transaction."""
|
||||
try:
|
||||
platform_found, seqno = await indexer_loop(memory, platform_found, seqno)
|
||||
except BaseException as e:
|
||||
make_log("Indexer", f"Error: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
|
||||
if platform_found:
|
||||
await send_status("indexer", f"working (seqno={seqno})")
|
||||
|
||||
await asyncio.sleep(5)
|
||||
seqno += 1
|
||||
# Check transaction status on blockchain
|
||||
if transaction.tx_hash:
|
||||
tx_info = await self.ton_service.get_transaction_info(transaction.tx_hash)
|
||||
|
||||
if tx_info:
|
||||
# Update transaction with blockchain data
|
||||
transaction.status = tx_info.get("status", "pending")
|
||||
transaction.block_number = tx_info.get("block_number")
|
||||
transaction.gas_used = tx_info.get("gas_used")
|
||||
transaction.gas_price = tx_info.get("gas_price")
|
||||
transaction.confirmations = tx_info.get("confirmations", 0)
|
||||
transaction.updated_at = datetime.utcnow()
|
||||
|
||||
# Cache transaction info
|
||||
cache_key = f"tx:{transaction.tx_hash}"
|
||||
await self.redis_client.setex(
|
||||
cache_key,
|
||||
3600, # 1 hour
|
||||
json.dumps(tx_info)
|
||||
)
|
||||
|
||||
logger.debug(f"Updated transaction {transaction.tx_hash}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing transaction {transaction.id}: {e}")
|
||||
|
||||
async def _update_transaction_confirmations(self) -> None:
|
||||
"""Update confirmation counts for recent transactions."""
|
||||
async with get_async_session() as session:
|
||||
try:
|
||||
# Get recent confirmed transactions
|
||||
cutoff_time = datetime.utcnow() - timedelta(hours=24)
|
||||
result = await session.execute(
|
||||
select(Transaction)
|
||||
.where(
|
||||
Transaction.status == "confirmed",
|
||||
Transaction.confirmations < self.confirmation_blocks,
|
||||
Transaction.updated_at > cutoff_time
|
||||
)
|
||||
.limit(self.batch_size)
|
||||
)
|
||||
transactions = result.scalars().all()
|
||||
|
||||
for transaction in transactions:
|
||||
if transaction.tx_hash:
|
||||
try:
|
||||
confirmations = await self.ton_service.get_transaction_confirmations(
|
||||
transaction.tx_hash
|
||||
)
|
||||
|
||||
if confirmations != transaction.confirmations:
|
||||
transaction.confirmations = confirmations
|
||||
transaction.updated_at = datetime.utcnow()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating confirmations for {transaction.tx_hash}: {e}")
|
||||
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating transaction confirmations: {e}")
|
||||
await session.rollback()
|
||||
|
||||
async def _update_wallet_balances(self) -> None:
|
||||
"""Update wallet balances from the blockchain."""
|
||||
async with get_async_session() as session:
|
||||
try:
|
||||
# Get active wallets
|
||||
result = await session.execute(
|
||||
select(Wallet)
|
||||
.where(Wallet.is_active == True)
|
||||
.limit(self.batch_size)
|
||||
)
|
||||
wallets = result.scalars().all()
|
||||
|
||||
for wallet in wallets:
|
||||
try:
|
||||
# Get current balance
|
||||
balance = await self.ton_service.get_wallet_balance(wallet.address)
|
||||
|
||||
if balance != wallet.balance:
|
||||
wallet.balance = balance
|
||||
wallet.updated_at = datetime.utcnow()
|
||||
|
||||
# Cache balance
|
||||
cache_key = f"balance:{wallet.address}"
|
||||
await self.redis_client.setex(cache_key, 300, str(balance)) # 5 minutes
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating balance for wallet {wallet.address}: {e}")
|
||||
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating wallet balances: {e}")
|
||||
await session.rollback()
|
||||
|
||||
async def _index_nft_collections(self) -> None:
|
||||
"""Index NFT collections and metadata."""
|
||||
async with get_async_session() as session:
|
||||
try:
|
||||
# Get wallets to check for NFTs
|
||||
result = await session.execute(
|
||||
select(Wallet)
|
||||
.where(Wallet.is_active == True)
|
||||
.limit(self.batch_size // 4) # Smaller batch for NFTs
|
||||
)
|
||||
wallets = result.scalars().all()
|
||||
|
||||
for wallet in wallets:
|
||||
try:
|
||||
# Get NFTs for this wallet
|
||||
nfts = await self.ton_service.get_wallet_nfts(wallet.address)
|
||||
|
||||
for nft_data in nfts:
|
||||
await self._process_nft(session, wallet, nft_data)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error indexing NFTs for wallet {wallet.address}: {e}")
|
||||
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error indexing NFT collections: {e}")
|
||||
await session.rollback()
|
||||
|
||||
async def _process_nft(self, session: AsyncSession, wallet: Wallet, nft_data: Dict[str, Any]) -> None:
|
||||
"""Process a single NFT."""
|
||||
try:
|
||||
# Check if NFT exists
|
||||
result = await session.execute(
|
||||
select(BlockchainNFT)
|
||||
.where(
|
||||
BlockchainNFT.token_id == nft_data["token_id"],
|
||||
BlockchainNFT.collection_address == nft_data["collection_address"]
|
||||
)
|
||||
)
|
||||
existing_nft = result.scalar_one_or_none()
|
||||
|
||||
if existing_nft:
|
||||
# Update existing NFT
|
||||
existing_nft.owner_address = wallet.address
|
||||
existing_nft.metadata = nft_data.get("metadata", {})
|
||||
existing_nft.updated_at = datetime.utcnow()
|
||||
else:
|
||||
# Create new NFT
|
||||
new_nft = BlockchainNFT(
|
||||
wallet_id=wallet.id,
|
||||
token_id=nft_data["token_id"],
|
||||
collection_address=nft_data["collection_address"],
|
||||
owner_address=wallet.address,
|
||||
token_uri=nft_data.get("token_uri"),
|
||||
metadata=nft_data.get("metadata", {}),
|
||||
created_at=datetime.utcnow()
|
||||
)
|
||||
session.add(new_nft)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing NFT {nft_data.get('token_id')}: {e}")
|
||||
|
||||
async def _index_nft_transfers(self) -> None:
|
||||
"""Index NFT transfers."""
|
||||
# This would involve checking recent blocks for NFT transfer events
|
||||
# Implementation depends on the specific blockchain's event system
|
||||
pass
|
||||
|
||||
async def _update_token_balances(self) -> None:
|
||||
"""Update token balances for wallets."""
|
||||
async with get_async_session() as session:
|
||||
try:
|
||||
# Get wallets with token balances to update
|
||||
result = await session.execute(
|
||||
select(Wallet)
|
||||
.where(Wallet.is_active == True)
|
||||
.limit(self.batch_size // 2)
|
||||
)
|
||||
wallets = result.scalars().all()
|
||||
|
||||
for wallet in wallets:
|
||||
try:
|
||||
# Get token balances
|
||||
token_balances = await self.ton_service.get_wallet_token_balances(wallet.address)
|
||||
|
||||
for token_data in token_balances:
|
||||
await self._update_token_balance(session, wallet, token_data)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating token balances for {wallet.address}: {e}")
|
||||
|
||||
await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating token balances: {e}")
|
||||
await session.rollback()
|
||||
|
||||
async def _update_token_balance(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
wallet: Wallet,
|
||||
token_data: Dict[str, Any]
|
||||
) -> None:
|
||||
"""Update a single token balance."""
|
||||
try:
|
||||
# Check if balance record exists
|
||||
result = await session.execute(
|
||||
select(BlockchainTokenBalance)
|
||||
.where(
|
||||
BlockchainTokenBalance.wallet_id == wallet.id,
|
||||
BlockchainTokenBalance.token_address == token_data["token_address"]
|
||||
)
|
||||
)
|
||||
existing_balance = result.scalar_one_or_none()
|
||||
|
||||
if existing_balance:
|
||||
# Update existing balance
|
||||
existing_balance.balance = token_data["balance"]
|
||||
existing_balance.decimals = token_data.get("decimals", 18)
|
||||
existing_balance.updated_at = datetime.utcnow()
|
||||
else:
|
||||
# Create new balance record
|
||||
new_balance = BlockchainTokenBalance(
|
||||
wallet_id=wallet.id,
|
||||
token_address=token_data["token_address"],
|
||||
token_name=token_data.get("name"),
|
||||
token_symbol=token_data.get("symbol"),
|
||||
balance=token_data["balance"],
|
||||
decimals=token_data.get("decimals", 18),
|
||||
created_at=datetime.utcnow()
|
||||
)
|
||||
session.add(new_balance)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating token balance: {e}")
|
||||
|
||||
async def _cleanup_old_cache_entries(self) -> None:
|
||||
"""Clean up old cache entries."""
|
||||
try:
|
||||
# Get all keys with our prefixes
|
||||
patterns = ["tx:*", "balance:*", "nft:*", "token:*"]
|
||||
|
||||
for pattern in patterns:
|
||||
keys = await self.redis_client.keys(pattern)
|
||||
|
||||
# Check TTL and remove expired keys
|
||||
for key in keys:
|
||||
ttl = await self.redis_client.ttl(key)
|
||||
if ttl == -1: # No expiration set
|
||||
await self.redis_client.expire(key, 3600) # Set 1 hour expiration
|
||||
|
||||
logger.debug("Cache cleanup completed")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during cache cleanup: {e}")
|
||||
|
||||
async def get_indexing_stats(self) -> Dict[str, Any]:
|
||||
"""Get indexing statistics."""
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
# Get transaction stats
|
||||
tx_result = await session.execute(
|
||||
select(Transaction.status, asyncio.func.count())
|
||||
.group_by(Transaction.status)
|
||||
)
|
||||
tx_stats = dict(tx_result.fetchall())
|
||||
|
||||
# Get wallet stats
|
||||
wallet_result = await session.execute(
|
||||
select(asyncio.func.count())
|
||||
.select_from(Wallet)
|
||||
.where(Wallet.is_active == True)
|
||||
)
|
||||
active_wallets = wallet_result.scalar()
|
||||
|
||||
# Get NFT stats
|
||||
nft_result = await session.execute(
|
||||
select(asyncio.func.count())
|
||||
.select_from(BlockchainNFT)
|
||||
)
|
||||
total_nfts = nft_result.scalar()
|
||||
|
||||
return {
|
||||
"transaction_stats": tx_stats,
|
||||
"active_wallets": active_wallets,
|
||||
"total_nfts": total_nfts,
|
||||
"is_running": self.is_running,
|
||||
"active_tasks": len([t for t in self.tasks if not t.done()]),
|
||||
"last_update": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting indexing stats: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# loop = asyncio.get_event_loop()
|
||||
# loop.run_until_complete(main())
|
||||
# loop.close()
|
||||
# Global indexer instance
|
||||
indexer_service = IndexerService()
|
||||
+644
-276
@@ -1,290 +1,658 @@
|
||||
"""
|
||||
TON Blockchain service for wallet operations, transaction management, and smart contract interactions.
|
||||
Provides async operations with connection pooling, caching, and comprehensive error handling.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from base64 import b64decode
|
||||
import os
|
||||
import traceback
|
||||
import httpx
|
||||
|
||||
from sqlalchemy import and_, func
|
||||
from tonsdk.boc import begin_cell, Cell
|
||||
from tonsdk.contract.wallet import Wallets
|
||||
from tonsdk.utils import HighloadQueryId
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Dict, List, Optional, Any, Tuple
|
||||
from uuid import UUID
|
||||
|
||||
from app.core._blockchain.ton.platform import platform
|
||||
from app.core._blockchain.ton.toncenter import toncenter
|
||||
from app.core.models.tasks import BlockchainTask
|
||||
from app.core._config import MY_FUND_ADDRESS
|
||||
from app.core._secrets import service_wallet
|
||||
from app.core._utils.send_status import send_status
|
||||
from app.core.storage import db_session
|
||||
from app.core.logger import make_log
|
||||
import httpx
|
||||
from sqlalchemy import select, update, and_
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session, get_cache_manager
|
||||
from app.core.logging import get_logger
|
||||
from app.core.security import decrypt_data, encrypt_data
|
||||
|
||||
async def get_sw_seqno():
|
||||
sw_seqno_result = await toncenter.run_get_method(service_wallet.address.to_string(1, 1, 1), 'seqno')
|
||||
if sw_seqno_result.get('exit_code', -1) != 0:
|
||||
sw_seqno_value = 0
|
||||
else:
|
||||
sw_seqno_value = int(sw_seqno_result.get('stack', [['num', '0x0']])[0][1], 16)
|
||||
logger = get_logger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
return sw_seqno_value
|
||||
|
||||
|
||||
async def main_fn(memory):
|
||||
make_log("TON", f"Service started, SW = {service_wallet.address.to_string(1, 1, 1)}", level="info")
|
||||
sw_seqno_value = await get_sw_seqno()
|
||||
make_log("TON", f"Service wallet run seqno method: {sw_seqno_value}", level="info")
|
||||
if sw_seqno_value == 0:
|
||||
make_log("TON", "Service wallet is not deployed, deploying...", level="info")
|
||||
await toncenter.send_boc(
|
||||
service_wallet.create_transfer_message(
|
||||
[{
|
||||
'address': service_wallet.address.to_string(1, 1, 1),
|
||||
'amount': 1,
|
||||
'send_mode': 1,
|
||||
'payload': begin_cell().store_uint(0, 32).store_bytes(b"Init MY Node").end_cell()
|
||||
}], 0
|
||||
)['message'].to_boc(False)
|
||||
)
|
||||
await asyncio.sleep(5)
|
||||
return await main_fn(memory)
|
||||
|
||||
if os.getenv("TON_BEGIN_COMMAND_WITHDRAW"):
|
||||
await toncenter.send_boc(
|
||||
service_wallet.create_transfer_message(
|
||||
[{
|
||||
'address': MY_FUND_ADDRESS,
|
||||
'amount': 1,
|
||||
'send_mode': 128,
|
||||
'payload': begin_cell().end_cell()
|
||||
}], sw_seqno_value
|
||||
)['message'].to_boc(False)
|
||||
)
|
||||
make_log("TON", "Withdraw command sent", level="info")
|
||||
await asyncio.sleep(10)
|
||||
return await main_fn(memory)
|
||||
|
||||
# TODO: не деплоить если указан master_address и мы проверили что аккаунт существует. Сейчас platform у каждой ноды будет разным
|
||||
platform_state = await toncenter.get_account(platform.address.to_string(1, 1, 1))
|
||||
if not platform_state.get('code'):
|
||||
make_log("TON", "Platform contract is not deployed, send deploy transaction..", level="info")
|
||||
await toncenter.send_boc(
|
||||
service_wallet.create_transfer_message(
|
||||
[{
|
||||
'address': platform.address.to_string(1, 1, 1),
|
||||
'amount': int(0.08 * 10 ** 9),
|
||||
'send_mode': 1,
|
||||
'payload': begin_cell().store_uint(0, 32).store_uint(0, 64).end_cell(),
|
||||
'state_init': platform.create_state_init()['state_init']
|
||||
}], sw_seqno_value
|
||||
)['message'].to_boc(False)
|
||||
)
|
||||
|
||||
await send_status("ton_daemon", "working: deploying platform")
|
||||
await asyncio.sleep(15)
|
||||
return await main_fn(memory)
|
||||
class TONService:
|
||||
"""
|
||||
Comprehensive TON blockchain service with async operations.
|
||||
Handles wallet management, transactions, and smart contract interactions.
|
||||
"""
|
||||
|
||||
highload_wallet = Wallets.ALL['hv3'](
|
||||
private_key=service_wallet.options['private_key'],
|
||||
public_key=service_wallet.options['public_key'],
|
||||
wc=0
|
||||
)
|
||||
make_log("TON", f"Highload wallet address: {highload_wallet.address.to_string(1, 1, 1)}", level="info")
|
||||
highload_state = await toncenter.get_account(highload_wallet.address.to_string(1, 1, 1))
|
||||
if int(highload_state.get('balance', '0')) / 1e9 < 0.05:
|
||||
make_log("TON", "Highload wallet balance is less than 0.05, send topup transaction..", level="info")
|
||||
await toncenter.send_boc(
|
||||
service_wallet.create_transfer_message(
|
||||
[{
|
||||
'address': highload_wallet.address.to_string(1, 1, 0),
|
||||
'amount': int(0.08 * 10 ** 9),
|
||||
'send_mode': 1,
|
||||
'payload': begin_cell().store_uint(0, 32).end_cell()
|
||||
}], sw_seqno_value
|
||||
)['message'].to_boc(False)
|
||||
def __init__(self):
|
||||
self.api_endpoint = settings.TON_API_ENDPOINT
|
||||
self.testnet = settings.TON_TESTNET
|
||||
self.api_key = settings.TON_API_KEY
|
||||
self.timeout = 30
|
||||
|
||||
# HTTP client for API requests
|
||||
self.client = httpx.AsyncClient(
|
||||
timeout=self.timeout,
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.api_key}" if self.api_key else None,
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
)
|
||||
await send_status("ton_daemon", "working: topup highload wallet")
|
||||
await asyncio.sleep(15)
|
||||
return await main_fn(memory)
|
||||
|
||||
self.cache_manager = get_cache_manager()
|
||||
|
||||
if not highload_state.get('code'):
|
||||
make_log("TON", "Highload wallet contract is not deployed, send deploy transaction..", level="info")
|
||||
created_at_ts = int(datetime.utcnow().timestamp()) - 60
|
||||
await toncenter.send_boc(
|
||||
highload_wallet.create_transfer_message(
|
||||
service_wallet.address.to_string(1, 1, 1),
|
||||
1, HighloadQueryId.from_seqno(0), created_at_ts, send_mode=1, payload="hello world", need_deploy=True
|
||||
)['message'].to_boc(False)
|
||||
)
|
||||
await send_status("ton_daemon", "working: deploying highload wallet")
|
||||
await asyncio.sleep(15)
|
||||
return await main_fn(memory)
|
||||
|
||||
while True:
|
||||
async def close(self):
|
||||
"""Close HTTP client and cleanup resources."""
|
||||
if self.client:
|
||||
await self.client.aclose()
|
||||
|
||||
async def create_wallet(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Create new TON wallet with mnemonic generation.
|
||||
|
||||
Returns:
|
||||
Dict: Wallet creation result with address and private key
|
||||
"""
|
||||
try:
|
||||
sw_seqno_value = await get_sw_seqno()
|
||||
make_log("TON", f"Service running ({sw_seqno_value})", level="debug")
|
||||
|
||||
with db_session() as session:
|
||||
# Проверка отправленных сообщений
|
||||
await send_status("ton_daemon", f"working: processing in-txs (seqno={sw_seqno_value})")
|
||||
async def process_incoming_transaction(transaction: dict):
|
||||
transaction_hash = transaction['transaction_id']['hash']
|
||||
transaction_lt = str(transaction['transaction_id']['lt'])
|
||||
# transaction_success = bool(transaction['success'])
|
||||
|
||||
async def process_incoming_message(blockchain_message: dict):
|
||||
in_msg_cell = Cell.one_from_boc(b64decode(blockchain_message['msg_data']['body']))
|
||||
in_msg_slice = in_msg_cell.refs[0].begin_parse()
|
||||
in_msg_slice.read_uint(32)
|
||||
in_msg_slice.read_uint(8)
|
||||
in_msg_query_id = in_msg_slice.read_uint(23)
|
||||
in_msg_created_at = in_msg_slice.read_uint(64)
|
||||
in_msg_epoch = int(in_msg_created_at // (60 * 60))
|
||||
in_msg_seqno = HighloadQueryId.from_query_id(in_msg_query_id).to_seqno()
|
||||
in_msg_blockchain_task = (
|
||||
session.query(BlockchainTask).filter(
|
||||
and_(
|
||||
BlockchainTask.seqno == in_msg_seqno,
|
||||
BlockchainTask.epoch == in_msg_epoch,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if not in_msg_blockchain_task:
|
||||
return
|
||||
|
||||
if not (in_msg_blockchain_task.status in ['done']) or in_msg_blockchain_task.transaction_hash != transaction_hash:
|
||||
in_msg_blockchain_task.status = 'done'
|
||||
in_msg_blockchain_task.transaction_hash = transaction_hash
|
||||
in_msg_blockchain_task.transaction_lt = transaction_lt
|
||||
session.commit()
|
||||
|
||||
for blockchain_message in [transaction['in_msg']]:
|
||||
try:
|
||||
await process_incoming_message(blockchain_message)
|
||||
except BaseException as e:
|
||||
pass # make_log("TON_Daemon", f"Error while processing incoming message: {e}" + '\n' + traceback.format_exc(), level='debug')
|
||||
|
||||
# Generate mnemonic phrase
|
||||
mnemonic_response = await self.client.post(
|
||||
f"{self.api_endpoint}/wallet/generate",
|
||||
json={"testnet": self.testnet}
|
||||
)
|
||||
|
||||
if mnemonic_response.status_code != 200:
|
||||
error_msg = f"Failed to generate wallet: {mnemonic_response.text}"
|
||||
await logger.aerror("Wallet generation failed", error=error_msg)
|
||||
return {"error": error_msg}
|
||||
|
||||
mnemonic_data = mnemonic_response.json()
|
||||
|
||||
# Create wallet from mnemonic
|
||||
wallet_response = await self.client.post(
|
||||
f"{self.api_endpoint}/wallet/create",
|
||||
json={
|
||||
"mnemonic": mnemonic_data["mnemonic"],
|
||||
"testnet": self.testnet
|
||||
}
|
||||
)
|
||||
|
||||
if wallet_response.status_code != 200:
|
||||
error_msg = f"Failed to create wallet: {wallet_response.text}"
|
||||
await logger.aerror("Wallet creation failed", error=error_msg)
|
||||
return {"error": error_msg}
|
||||
|
||||
wallet_data = wallet_response.json()
|
||||
|
||||
await logger.ainfo(
|
||||
"Wallet created successfully",
|
||||
address=wallet_data.get("address"),
|
||||
testnet=self.testnet
|
||||
)
|
||||
|
||||
return {
|
||||
"address": wallet_data["address"],
|
||||
"private_key": wallet_data["private_key"],
|
||||
"mnemonic": mnemonic_data["mnemonic"],
|
||||
"testnet": self.testnet
|
||||
}
|
||||
|
||||
except httpx.TimeoutException:
|
||||
error_msg = "Wallet creation timeout"
|
||||
await logger.aerror(error_msg)
|
||||
return {"error": error_msg}
|
||||
except Exception as e:
|
||||
error_msg = f"Wallet creation error: {str(e)}"
|
||||
await logger.aerror("Wallet creation exception", error=str(e))
|
||||
return {"error": error_msg}
|
||||
|
||||
async def get_wallet_balance(self, address: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get wallet balance with caching for performance.
|
||||
|
||||
Args:
|
||||
address: TON wallet address
|
||||
|
||||
Returns:
|
||||
Dict: Balance information
|
||||
"""
|
||||
try:
|
||||
# Check cache first
|
||||
cache_key = f"ton_balance:{address}"
|
||||
cached_balance = await self.cache_manager.get(cache_key)
|
||||
|
||||
if cached_balance:
|
||||
return cached_balance
|
||||
|
||||
# Fetch from blockchain
|
||||
balance_response = await self.client.get(
|
||||
f"{self.api_endpoint}/wallet/{address}/balance"
|
||||
)
|
||||
|
||||
if balance_response.status_code != 200:
|
||||
error_msg = f"Failed to get balance: {balance_response.text}"
|
||||
return {"error": error_msg}
|
||||
|
||||
balance_data = balance_response.json()
|
||||
|
||||
result = {
|
||||
"balance": int(balance_data.get("balance", 0)), # nanotons
|
||||
"last_transaction_lt": balance_data.get("last_transaction_lt"),
|
||||
"account_state": balance_data.get("account_state", "unknown"),
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Cache for 30 seconds
|
||||
await self.cache_manager.set(cache_key, result, ttl=30)
|
||||
|
||||
return result
|
||||
|
||||
except httpx.TimeoutException:
|
||||
return {"error": "Balance fetch timeout"}
|
||||
except Exception as e:
|
||||
await logger.aerror("Balance fetch error", address=address, error=str(e))
|
||||
return {"error": f"Balance fetch error: {str(e)}"}
|
||||
|
||||
async def get_wallet_transactions(
|
||||
self,
|
||||
address: str,
|
||||
limit: int = 20,
|
||||
offset: int = 0
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get wallet transaction history with pagination.
|
||||
|
||||
Args:
|
||||
address: TON wallet address
|
||||
limit: Number of transactions to fetch
|
||||
offset: Pagination offset
|
||||
|
||||
Returns:
|
||||
Dict: Transaction history
|
||||
"""
|
||||
try:
|
||||
# Check cache
|
||||
cache_key = f"ton_transactions:{address}:{limit}:{offset}"
|
||||
cached_transactions = await self.cache_manager.get(cache_key)
|
||||
|
||||
if cached_transactions:
|
||||
return cached_transactions
|
||||
|
||||
transactions_response = await self.client.get(
|
||||
f"{self.api_endpoint}/wallet/{address}/transactions",
|
||||
params={"limit": limit, "offset": offset}
|
||||
)
|
||||
|
||||
if transactions_response.status_code != 200:
|
||||
error_msg = f"Failed to get transactions: {transactions_response.text}"
|
||||
return {"error": error_msg}
|
||||
|
||||
transactions_data = transactions_response.json()
|
||||
|
||||
result = {
|
||||
"transactions": transactions_data.get("transactions", []),
|
||||
"total": transactions_data.get("total", 0),
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Cache for 1 minute
|
||||
await self.cache_manager.set(cache_key, result, ttl=60)
|
||||
|
||||
return result
|
||||
|
||||
except httpx.TimeoutException:
|
||||
return {"error": "Transaction fetch timeout"}
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Transaction fetch error",
|
||||
address=address,
|
||||
error=str(e)
|
||||
)
|
||||
return {"error": f"Transaction fetch error: {str(e)}"}
|
||||
|
||||
async def send_transaction(
|
||||
self,
|
||||
private_key: str,
|
||||
recipient_address: str,
|
||||
amount: int,
|
||||
message: str = "",
|
||||
**kwargs
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Send TON transaction with validation and monitoring.
|
||||
|
||||
Args:
|
||||
private_key: Encrypted private key
|
||||
recipient_address: Recipient wallet address
|
||||
amount: Amount in nanotons
|
||||
message: Optional message
|
||||
**kwargs: Additional transaction parameters
|
||||
|
||||
Returns:
|
||||
Dict: Transaction result
|
||||
"""
|
||||
try:
|
||||
# Validate inputs
|
||||
if amount <= 0:
|
||||
return {"error": "Amount must be positive"}
|
||||
|
||||
if len(recipient_address) != 48:
|
||||
return {"error": "Invalid recipient address format"}
|
||||
|
||||
# Decrypt private key
|
||||
try:
|
||||
decrypted_key = decrypt_data(private_key, context="wallet")
|
||||
if isinstance(decrypted_key, bytes):
|
||||
decrypted_key = decrypted_key.decode('utf-8')
|
||||
except Exception as e:
|
||||
await logger.aerror("Private key decryption failed", error=str(e))
|
||||
return {"error": "Invalid private key"}
|
||||
|
||||
# Prepare transaction
|
||||
transaction_data = {
|
||||
"private_key": decrypted_key,
|
||||
"recipient": recipient_address,
|
||||
"amount": str(amount),
|
||||
"message": message,
|
||||
"testnet": self.testnet
|
||||
}
|
||||
|
||||
# Send transaction
|
||||
tx_response = await self.client.post(
|
||||
f"{self.api_endpoint}/transaction/send",
|
||||
json=transaction_data
|
||||
)
|
||||
|
||||
if tx_response.status_code != 200:
|
||||
error_msg = f"Transaction failed: {tx_response.text}"
|
||||
await logger.aerror(
|
||||
"Transaction submission failed",
|
||||
recipient=recipient_address,
|
||||
amount=amount,
|
||||
error=error_msg
|
||||
)
|
||||
return {"error": error_msg}
|
||||
|
||||
tx_data = tx_response.json()
|
||||
|
||||
result = {
|
||||
"hash": tx_data["hash"],
|
||||
"lt": tx_data.get("lt"),
|
||||
"fee": tx_data.get("fee", 0),
|
||||
"block_hash": tx_data.get("block_hash"),
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
await logger.ainfo(
|
||||
"Transaction sent successfully",
|
||||
hash=result["hash"],
|
||||
recipient=recipient_address,
|
||||
amount=amount
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except httpx.TimeoutException:
|
||||
return {"error": "Transaction timeout"}
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Transaction send error",
|
||||
recipient=recipient_address,
|
||||
amount=amount,
|
||||
error=str(e)
|
||||
)
|
||||
return {"error": f"Transaction error: {str(e)}"}
|
||||
|
||||
async def get_transaction_status(self, tx_hash: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get transaction status and confirmation details.
|
||||
|
||||
Args:
|
||||
tx_hash: Transaction hash
|
||||
|
||||
Returns:
|
||||
Dict: Transaction status information
|
||||
"""
|
||||
try:
|
||||
# Check cache
|
||||
cache_key = f"ton_tx_status:{tx_hash}"
|
||||
cached_status = await self.cache_manager.get(cache_key)
|
||||
|
||||
if cached_status and cached_status.get("confirmed"):
|
||||
return cached_status
|
||||
|
||||
status_response = await self.client.get(
|
||||
f"{self.api_endpoint}/transaction/{tx_hash}/status"
|
||||
)
|
||||
|
||||
if status_response.status_code != 200:
|
||||
return {"error": f"Failed to get status: {status_response.text}"}
|
||||
|
||||
status_data = status_response.json()
|
||||
|
||||
result = {
|
||||
"hash": tx_hash,
|
||||
"confirmed": status_data.get("confirmed", False),
|
||||
"failed": status_data.get("failed", False),
|
||||
"confirmations": status_data.get("confirmations", 0),
|
||||
"block_hash": status_data.get("block_hash"),
|
||||
"block_time": status_data.get("block_time"),
|
||||
"fee": status_data.get("fee"),
|
||||
"confirmed_at": status_data.get("confirmed_at"),
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Cache confirmed/failed transactions longer
|
||||
cache_ttl = 3600 if result["confirmed"] or result["failed"] else 30
|
||||
await self.cache_manager.set(cache_key, result, ttl=cache_ttl)
|
||||
|
||||
return result
|
||||
|
||||
except httpx.TimeoutException:
|
||||
return {"error": "Status check timeout"}
|
||||
except Exception as e:
|
||||
await logger.aerror("Status check error", tx_hash=tx_hash, error=str(e))
|
||||
return {"error": f"Status check error: {str(e)}"}
|
||||
|
||||
async def validate_address(self, address: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Validate TON address format and existence.
|
||||
|
||||
Args:
|
||||
address: TON address to validate
|
||||
|
||||
Returns:
|
||||
Dict: Validation result
|
||||
"""
|
||||
try:
|
||||
# Basic format validation
|
||||
if len(address) != 48:
|
||||
return {"valid": False, "error": "Invalid address length"}
|
||||
|
||||
# Check against blockchain
|
||||
validation_response = await self.client.post(
|
||||
f"{self.api_endpoint}/address/validate",
|
||||
json={"address": address}
|
||||
)
|
||||
|
||||
if validation_response.status_code != 200:
|
||||
return {"valid": False, "error": "Validation service error"}
|
||||
|
||||
validation_data = validation_response.json()
|
||||
|
||||
return {
|
||||
"valid": validation_data.get("valid", False),
|
||||
"exists": validation_data.get("exists", False),
|
||||
"account_type": validation_data.get("account_type"),
|
||||
"error": validation_data.get("error")
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Address validation error", address=address, error=str(e))
|
||||
return {"valid": False, "error": f"Validation error: {str(e)}"}
|
||||
|
||||
async def get_network_info(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get TON network information and statistics.
|
||||
|
||||
Returns:
|
||||
Dict: Network information
|
||||
"""
|
||||
try:
|
||||
cache_key = "ton_network_info"
|
||||
cached_info = await self.cache_manager.get(cache_key)
|
||||
|
||||
if cached_info:
|
||||
return cached_info
|
||||
|
||||
network_response = await self.client.get(
|
||||
f"{self.api_endpoint}/network/info"
|
||||
)
|
||||
|
||||
if network_response.status_code != 200:
|
||||
return {"error": f"Failed to get network info: {network_response.text}"}
|
||||
|
||||
network_data = network_response.json()
|
||||
|
||||
result = {
|
||||
"network": "testnet" if self.testnet else "mainnet",
|
||||
"last_block": network_data.get("last_block"),
|
||||
"last_block_time": network_data.get("last_block_time"),
|
||||
"total_accounts": network_data.get("total_accounts"),
|
||||
"total_transactions": network_data.get("total_transactions"),
|
||||
"tps": network_data.get("tps"), # Transactions per second
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Cache for 5 minutes
|
||||
await self.cache_manager.set(cache_key, result, ttl=300)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Network info error", error=str(e))
|
||||
return {"error": f"Network info error: {str(e)}"}
|
||||
|
||||
async def estimate_transaction_fee(
|
||||
self,
|
||||
sender_address: str,
|
||||
recipient_address: str,
|
||||
amount: int,
|
||||
message: str = ""
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Estimate transaction fee before sending.
|
||||
|
||||
Args:
|
||||
sender_address: Sender wallet address
|
||||
recipient_address: Recipient wallet address
|
||||
amount: Amount in nanotons
|
||||
message: Optional message
|
||||
|
||||
Returns:
|
||||
Dict: Fee estimation
|
||||
"""
|
||||
try:
|
||||
fee_response = await self.client.post(
|
||||
f"{self.api_endpoint}/transaction/estimate-fee",
|
||||
json={
|
||||
"sender": sender_address,
|
||||
"recipient": recipient_address,
|
||||
"amount": str(amount),
|
||||
"message": message
|
||||
}
|
||||
)
|
||||
|
||||
if fee_response.status_code != 200:
|
||||
return {"error": f"Fee estimation failed: {fee_response.text}"}
|
||||
|
||||
fee_data = fee_response.json()
|
||||
|
||||
return {
|
||||
"estimated_fee": fee_data.get("fee", 0),
|
||||
"estimated_fee_tons": str(Decimal(fee_data.get("fee", 0)) / Decimal("1000000000")),
|
||||
"gas_used": fee_data.get("gas_used"),
|
||||
"message_size": len(message.encode('utf-8')),
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Fee estimation error", error=str(e))
|
||||
return {"error": f"Fee estimation error: {str(e)}"}
|
||||
|
||||
async def monitor_transaction(self, tx_hash: str, max_wait_time: int = 300) -> Dict[str, Any]:
|
||||
"""
|
||||
Monitor transaction until confirmation or timeout.
|
||||
|
||||
Args:
|
||||
tx_hash: Transaction hash to monitor
|
||||
max_wait_time: Maximum wait time in seconds
|
||||
|
||||
Returns:
|
||||
Dict: Final transaction status
|
||||
"""
|
||||
start_time = datetime.utcnow()
|
||||
check_interval = 5 # Check every 5 seconds
|
||||
|
||||
while (datetime.utcnow() - start_time).seconds < max_wait_time:
|
||||
status = await self.get_transaction_status(tx_hash)
|
||||
|
||||
if status.get("error"):
|
||||
return status
|
||||
|
||||
if status.get("confirmed") or status.get("failed"):
|
||||
await logger.ainfo(
|
||||
"Transaction monitoring completed",
|
||||
tx_hash=tx_hash,
|
||||
confirmed=status.get("confirmed"),
|
||||
failed=status.get("failed"),
|
||||
duration=(datetime.utcnow() - start_time).seconds
|
||||
)
|
||||
return status
|
||||
|
||||
await asyncio.sleep(check_interval)
|
||||
|
||||
# Timeout reached
|
||||
await logger.awarning(
|
||||
"Transaction monitoring timeout",
|
||||
tx_hash=tx_hash,
|
||||
max_wait_time=max_wait_time
|
||||
)
|
||||
|
||||
return {
|
||||
"hash": tx_hash,
|
||||
"confirmed": False,
|
||||
"timeout": True,
|
||||
"error": "Monitoring timeout reached"
|
||||
}
|
||||
|
||||
async def get_smart_contract_info(self, address: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get smart contract information and ABI.
|
||||
|
||||
Args:
|
||||
address: Smart contract address
|
||||
|
||||
Returns:
|
||||
Dict: Contract information
|
||||
"""
|
||||
try:
|
||||
cache_key = f"ton_contract:{address}"
|
||||
cached_info = await self.cache_manager.get(cache_key)
|
||||
|
||||
if cached_info:
|
||||
return cached_info
|
||||
|
||||
contract_response = await self.client.get(
|
||||
f"{self.api_endpoint}/contract/{address}/info"
|
||||
)
|
||||
|
||||
if contract_response.status_code != 200:
|
||||
return {"error": f"Failed to get contract info: {contract_response.text}"}
|
||||
|
||||
contract_data = contract_response.json()
|
||||
|
||||
result = {
|
||||
"address": address,
|
||||
"contract_type": contract_data.get("contract_type"),
|
||||
"is_verified": contract_data.get("is_verified", False),
|
||||
"abi": contract_data.get("abi"),
|
||||
"source_code": contract_data.get("source_code"),
|
||||
"compiler_version": contract_data.get("compiler_version"),
|
||||
"deployment_block": contract_data.get("deployment_block"),
|
||||
"updated_at": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
# Cache for 1 hour
|
||||
await self.cache_manager.set(cache_key, result, ttl=3600)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Contract info error", address=address, error=str(e))
|
||||
return {"error": f"Contract info error: {str(e)}"}
|
||||
|
||||
async def call_smart_contract(
|
||||
self,
|
||||
contract_address: str,
|
||||
method: str,
|
||||
params: Dict[str, Any],
|
||||
private_key: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Call smart contract method.
|
||||
|
||||
Args:
|
||||
contract_address: Contract address
|
||||
method: Method name to call
|
||||
params: Method parameters
|
||||
private_key: Private key for write operations
|
||||
|
||||
Returns:
|
||||
Dict: Contract call result
|
||||
"""
|
||||
try:
|
||||
call_data = {
|
||||
"contract": contract_address,
|
||||
"method": method,
|
||||
"params": params
|
||||
}
|
||||
|
||||
# Add private key for write operations
|
||||
if private_key:
|
||||
try:
|
||||
sw_transactions = await toncenter.get_transactions(highload_wallet.address.to_string(1, 1, 1), limit=100)
|
||||
for sw_transaction in sw_transactions:
|
||||
try:
|
||||
await process_incoming_transaction(sw_transaction)
|
||||
except BaseException as e:
|
||||
make_log("TON_Daemon", f"Error while processing incoming transaction: {e}", level="debug")
|
||||
except BaseException as e:
|
||||
make_log("TON_Daemon", f"Error while getting service wallet transactions: {e}", level="ERROR")
|
||||
|
||||
await send_status("ton_daemon", f"working: processing out-txs (seqno={sw_seqno_value})")
|
||||
# Отправка подписанных сообщений
|
||||
for blockchain_task in (
|
||||
session.query(BlockchainTask).filter(
|
||||
BlockchainTask.status == 'processing',
|
||||
).order_by(BlockchainTask.updated.asc()).all()
|
||||
):
|
||||
make_log("TON_Daemon", f"Processing task (processing) {blockchain_task.id}")
|
||||
query_boc = bytes.fromhex(blockchain_task.meta['signed_message'])
|
||||
errors_list = []
|
||||
|
||||
try:
|
||||
await toncenter.send_boc(query_boc)
|
||||
except BaseException as e:
|
||||
errors_list.append(f"{e}")
|
||||
|
||||
try:
|
||||
make_log("TON_Daemon", str(
|
||||
httpx.post(
|
||||
'https://tonapi.io/v2/blockchain/message',
|
||||
json={
|
||||
'boc': query_boc.hex()
|
||||
}
|
||||
).text
|
||||
))
|
||||
except BaseException as e:
|
||||
make_log("TON_Daemon", f"Error while pushing task to tonkeeper ({blockchain_task.id}): {e}", level="ERROR")
|
||||
errors_list.append(f"{e}")
|
||||
|
||||
blockchain_task.updated = datetime.utcnow()
|
||||
|
||||
if blockchain_task.meta['sign_created'] + 10 * 60 < datetime.utcnow().timestamp():
|
||||
# or sum([int("terminating vm with exit code 36" in e) for e in errors_list]) > 0:
|
||||
make_log("TON_Daemon", f"Task {blockchain_task.id} done", level="DEBUG")
|
||||
blockchain_task.status = 'done'
|
||||
session.commit()
|
||||
continue
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
await send_status("ton_daemon", f"working: creating new messages (seqno={sw_seqno_value})")
|
||||
# Создание новых подписей
|
||||
for blockchain_task in (
|
||||
session.query(BlockchainTask).filter(BlockchainTask.status == 'wait').all()
|
||||
):
|
||||
try:
|
||||
# Check processing tasks in current epoch < 3_000_000
|
||||
if (
|
||||
session.query(BlockchainTask).filter(
|
||||
BlockchainTask.epoch == blockchain_task.epoch,
|
||||
).count() > 3_000_000
|
||||
):
|
||||
make_log("TON", f"Too many processing tasks in epoch {blockchain_task.epoch}", level="error")
|
||||
await send_status("ton_daemon", f"working: too many tasks in epoch {blockchain_task.epoch}")
|
||||
await asyncio.sleep(5)
|
||||
continue
|
||||
|
||||
sign_created = int(datetime.utcnow().timestamp()) - 60
|
||||
try:
|
||||
current_epoch = int(datetime.utcnow().timestamp() // (60 * 60))
|
||||
max_epoch_seqno = (
|
||||
session.query(func.max(BlockchainTask.seqno)).filter(
|
||||
BlockchainTask.epoch == current_epoch
|
||||
).scalar() or 0
|
||||
)
|
||||
current_epoch_shift = 3_000_000 if current_epoch % 2 == 0 else 0
|
||||
current_seqno = max_epoch_seqno + 1 + (current_epoch_shift if max_epoch_seqno == 0 else 0)
|
||||
except BaseException as e:
|
||||
make_log("CRITICAL", f"Error calculating epoch,seqno: {e}", level="error")
|
||||
current_epoch = 0
|
||||
current_seqno = 0
|
||||
|
||||
blockchain_task.seqno = current_seqno
|
||||
blockchain_task.epoch = current_epoch
|
||||
blockchain_task.status = 'processing'
|
||||
try:
|
||||
query = highload_wallet.create_transfer_message(
|
||||
blockchain_task.destination, int(blockchain_task.amount), HighloadQueryId.from_seqno(current_seqno),
|
||||
sign_created, send_mode=1,
|
||||
payload=Cell.one_from_boc(b64decode(blockchain_task.payload))
|
||||
)
|
||||
query_boc = query['message'].to_boc(False)
|
||||
except BaseException as e:
|
||||
make_log("TON", f"Error creating transfer message: {e}", level="error")
|
||||
query_boc = begin_cell().end_cell().to_boc(False)
|
||||
|
||||
blockchain_task.meta = {
|
||||
**blockchain_task.meta,
|
||||
'sign_created': sign_created,
|
||||
'signed_message': query_boc.hex(),
|
||||
}
|
||||
session.commit()
|
||||
make_log("TON", f"Created signed message for task {blockchain_task.id}" + '\n' + traceback.format_exc(), level="info")
|
||||
except BaseException as e:
|
||||
make_log("TON", f"Error processing task {blockchain_task.id}: {e}" + '\n' + traceback.format_exc(), level="error")
|
||||
continue
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
await asyncio.sleep(1)
|
||||
await send_status("ton_daemon", f"working (seqno={sw_seqno_value})")
|
||||
except BaseException as e:
|
||||
make_log("TON", f"Error: {e}", level="error")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# loop = asyncio.get_event_loop()
|
||||
# loop.run_until_complete(main())
|
||||
# loop.close()
|
||||
|
||||
decrypted_key = decrypt_data(private_key, context="wallet")
|
||||
if isinstance(decrypted_key, bytes):
|
||||
decrypted_key = decrypted_key.decode('utf-8')
|
||||
call_data["private_key"] = decrypted_key
|
||||
except Exception as e:
|
||||
return {"error": "Invalid private key"}
|
||||
|
||||
contract_response = await self.client.post(
|
||||
f"{self.api_endpoint}/contract/call",
|
||||
json=call_data
|
||||
)
|
||||
|
||||
if contract_response.status_code != 200:
|
||||
return {"error": f"Contract call failed: {contract_response.text}"}
|
||||
|
||||
call_result = contract_response.json()
|
||||
|
||||
await logger.ainfo(
|
||||
"Smart contract called",
|
||||
contract=contract_address,
|
||||
method=method,
|
||||
success=call_result.get("success", False)
|
||||
)
|
||||
|
||||
return call_result
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Contract call error",
|
||||
contract=contract_address,
|
||||
method=method,
|
||||
error=str(e)
|
||||
)
|
||||
return {"error": f"Contract call error: {str(e)}"}
|
||||
|
||||
# Global TON service instance
|
||||
_ton_service = None
|
||||
|
||||
async def get_ton_service() -> TONService:
|
||||
"""Get or create global TON service instance."""
|
||||
global _ton_service
|
||||
if _ton_service is None:
|
||||
_ton_service = TONService()
|
||||
return _ton_service
|
||||
|
||||
async def cleanup_ton_service():
|
||||
"""Cleanup global TON service instance."""
|
||||
global _ton_service
|
||||
if _ton_service:
|
||||
await _ton_service.close()
|
||||
_ton_service = None
|
||||
@@ -0,0 +1,385 @@
|
||||
"""Redis caching system with fallback support."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import pickle
|
||||
from typing import Any, Optional, Union, Dict, List
|
||||
from contextlib import asynccontextmanager
|
||||
from functools import wraps
|
||||
|
||||
import redis.asyncio as redis
|
||||
from redis.asyncio import ConnectionPool
|
||||
|
||||
from app.core.config_compatible import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Global Redis connection pool
|
||||
_redis_pool: Optional[ConnectionPool] = None
|
||||
_redis_client: Optional[redis.Redis] = None
|
||||
|
||||
|
||||
class CacheError(Exception):
|
||||
"""Custom cache error."""
|
||||
pass
|
||||
|
||||
|
||||
async def init_cache() -> None:
|
||||
"""Initialize Redis cache connection."""
|
||||
global _redis_pool, _redis_client
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
if not settings.redis_enabled or not settings.cache_enabled:
|
||||
logger.info("Redis caching is disabled")
|
||||
return
|
||||
|
||||
try:
|
||||
# Create connection pool
|
||||
_redis_pool = ConnectionPool(
|
||||
host=settings.redis_host,
|
||||
port=settings.redis_port,
|
||||
password=settings.redis_password,
|
||||
db=settings.redis_db,
|
||||
max_connections=settings.redis_max_connections,
|
||||
socket_timeout=settings.redis_socket_timeout,
|
||||
socket_connect_timeout=settings.redis_socket_connect_timeout,
|
||||
decode_responses=False, # We'll handle encoding manually for flexibility
|
||||
retry_on_timeout=True,
|
||||
health_check_interval=30,
|
||||
)
|
||||
|
||||
# Create Redis client
|
||||
_redis_client = redis.Redis(connection_pool=_redis_pool)
|
||||
|
||||
# Test connection
|
||||
await _redis_client.ping()
|
||||
|
||||
logger.info(f"Redis cache initialized successfully at {settings.redis_host}:{settings.redis_port}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to initialize Redis cache: {e}. Caching will be disabled.")
|
||||
_redis_pool = None
|
||||
_redis_client = None
|
||||
|
||||
|
||||
async def close_cache() -> None:
|
||||
"""Close Redis cache connection."""
|
||||
global _redis_pool, _redis_client
|
||||
|
||||
if _redis_client:
|
||||
try:
|
||||
await _redis_client.close()
|
||||
logger.info("Redis cache connection closed")
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing Redis cache: {e}")
|
||||
finally:
|
||||
_redis_client = None
|
||||
_redis_pool = None
|
||||
|
||||
|
||||
def get_redis_client() -> Optional[redis.Redis]:
|
||||
"""Get Redis client instance."""
|
||||
return _redis_client
|
||||
|
||||
|
||||
def is_cache_available() -> bool:
|
||||
"""Check if cache is available."""
|
||||
return _redis_client is not None
|
||||
|
||||
|
||||
class Cache:
|
||||
"""Redis cache manager with fallback support."""
|
||||
|
||||
def __init__(self):
|
||||
self.settings = get_settings()
|
||||
|
||||
def _serialize(self, value: Any) -> bytes:
|
||||
"""Serialize value for storage."""
|
||||
try:
|
||||
if isinstance(value, (str, int, float, bool)):
|
||||
return json.dumps(value).encode('utf-8')
|
||||
else:
|
||||
return pickle.dumps(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to serialize cache value: {e}")
|
||||
raise CacheError(f"Serialization error: {e}")
|
||||
|
||||
def _deserialize(self, data: bytes) -> Any:
|
||||
"""Deserialize value from storage."""
|
||||
try:
|
||||
# Try JSON first (for simple types)
|
||||
try:
|
||||
return json.loads(data.decode('utf-8'))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
# Fallback to pickle for complex objects
|
||||
return pickle.loads(data)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to deserialize cache value: {e}")
|
||||
raise CacheError(f"Deserialization error: {e}")
|
||||
|
||||
def _make_key(self, key: str, prefix: str = "myuploader") -> str:
|
||||
"""Create cache key with prefix."""
|
||||
return f"{prefix}:{key}"
|
||||
|
||||
async def get(self, key: str, default: Any = None) -> Any:
|
||||
"""Get value from cache."""
|
||||
if not is_cache_available():
|
||||
return default
|
||||
|
||||
try:
|
||||
redis_key = self._make_key(key)
|
||||
data = await _redis_client.get(redis_key)
|
||||
|
||||
if data is None:
|
||||
return default
|
||||
|
||||
return self._deserialize(data)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Cache get error for key '{key}': {e}")
|
||||
return default
|
||||
|
||||
async def set(self, key: str, value: Any, ttl: Optional[int] = None) -> bool:
|
||||
"""Set value in cache."""
|
||||
if not is_cache_available():
|
||||
return False
|
||||
|
||||
try:
|
||||
redis_key = self._make_key(key)
|
||||
data = self._serialize(value)
|
||||
|
||||
if ttl is None:
|
||||
ttl = self.settings.cache_default_ttl
|
||||
|
||||
await _redis_client.setex(redis_key, ttl, data)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Cache set error for key '{key}': {e}")
|
||||
return False
|
||||
|
||||
async def delete(self, key: str) -> bool:
|
||||
"""Delete value from cache."""
|
||||
if not is_cache_available():
|
||||
return False
|
||||
|
||||
try:
|
||||
redis_key = self._make_key(key)
|
||||
result = await _redis_client.delete(redis_key)
|
||||
return bool(result)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Cache delete error for key '{key}': {e}")
|
||||
return False
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
"""Check if key exists in cache."""
|
||||
if not is_cache_available():
|
||||
return False
|
||||
|
||||
try:
|
||||
redis_key = self._make_key(key)
|
||||
result = await _redis_client.exists(redis_key)
|
||||
return bool(result)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Cache exists error for key '{key}': {e}")
|
||||
return False
|
||||
|
||||
async def expire(self, key: str, ttl: int) -> bool:
|
||||
"""Set expiration time for key."""
|
||||
if not is_cache_available():
|
||||
return False
|
||||
|
||||
try:
|
||||
redis_key = self._make_key(key)
|
||||
result = await _redis_client.expire(redis_key, ttl)
|
||||
return bool(result)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Cache expire error for key '{key}': {e}")
|
||||
return False
|
||||
|
||||
async def clear_pattern(self, pattern: str) -> int:
|
||||
"""Clear all keys matching pattern."""
|
||||
if not is_cache_available():
|
||||
return 0
|
||||
|
||||
try:
|
||||
redis_pattern = self._make_key(pattern)
|
||||
keys = await _redis_client.keys(redis_pattern)
|
||||
|
||||
if keys:
|
||||
result = await _redis_client.delete(*keys)
|
||||
return result
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Cache clear pattern error for pattern '{pattern}': {e}")
|
||||
return 0
|
||||
|
||||
async def increment(self, key: str, amount: int = 1, ttl: Optional[int] = None) -> Optional[int]:
|
||||
"""Increment counter in cache."""
|
||||
if not is_cache_available():
|
||||
return None
|
||||
|
||||
try:
|
||||
redis_key = self._make_key(key)
|
||||
result = await _redis_client.incrby(redis_key, amount)
|
||||
|
||||
if ttl is not None:
|
||||
await _redis_client.expire(redis_key, ttl)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Cache increment error for key '{key}': {e}")
|
||||
return None
|
||||
|
||||
async def get_multiple(self, keys: List[str]) -> Dict[str, Any]:
|
||||
"""Get multiple values from cache."""
|
||||
if not is_cache_available():
|
||||
return {}
|
||||
|
||||
try:
|
||||
redis_keys = [self._make_key(key) for key in keys]
|
||||
values = await _redis_client.mget(redis_keys)
|
||||
|
||||
result = {}
|
||||
for i, (key, data) in enumerate(zip(keys, values)):
|
||||
if data is not None:
|
||||
try:
|
||||
result[key] = self._deserialize(data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to deserialize cached value for key '{key}': {e}")
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Cache get_multiple error: {e}")
|
||||
return {}
|
||||
|
||||
async def set_multiple(self, mapping: Dict[str, Any], ttl: Optional[int] = None) -> bool:
|
||||
"""Set multiple values in cache."""
|
||||
if not is_cache_available():
|
||||
return False
|
||||
|
||||
try:
|
||||
pipeline = _redis_client.pipeline()
|
||||
|
||||
for key, value in mapping.items():
|
||||
redis_key = self._make_key(key)
|
||||
data = self._serialize(value)
|
||||
|
||||
if ttl is None:
|
||||
ttl = self.settings.cache_default_ttl
|
||||
|
||||
pipeline.setex(redis_key, ttl, data)
|
||||
|
||||
await pipeline.execute()
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Cache set_multiple error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# Global cache instance
|
||||
cache = Cache()
|
||||
|
||||
|
||||
# Caching decorators
|
||||
def cached(ttl: Optional[int] = None, key_prefix: str = "func"):
|
||||
"""Decorator for caching function results."""
|
||||
def decorator(func):
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
if not is_cache_available():
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
# Create cache key from function name and arguments
|
||||
key_parts = [key_prefix, func.__name__]
|
||||
if args:
|
||||
key_parts.extend([str(arg) for arg in args])
|
||||
if kwargs:
|
||||
key_parts.extend([f"{k}={v}" for k, v in sorted(kwargs.items())])
|
||||
|
||||
cache_key = ":".join(key_parts)
|
||||
|
||||
# Try to get from cache
|
||||
result = await cache.get(cache_key)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# Call function and cache result
|
||||
result = await func(*args, **kwargs)
|
||||
await cache.set(cache_key, result, ttl)
|
||||
return result
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
def cache_user_data(ttl: Optional[int] = None):
|
||||
"""Decorator for caching user-specific data."""
|
||||
if ttl is None:
|
||||
ttl = get_settings().cache_user_ttl
|
||||
return cached(ttl=ttl, key_prefix="user")
|
||||
|
||||
|
||||
def cache_content_data(ttl: Optional[int] = None):
|
||||
"""Decorator for caching content data."""
|
||||
if ttl is None:
|
||||
ttl = get_settings().cache_content_ttl
|
||||
return cached(ttl=ttl, key_prefix="content")
|
||||
|
||||
|
||||
# Cache health check
|
||||
async def check_cache_health() -> Dict[str, Any]:
|
||||
"""Check cache health and return status."""
|
||||
if not is_cache_available():
|
||||
return {
|
||||
"status": "disabled",
|
||||
"available": False,
|
||||
"error": "Redis not initialized"
|
||||
}
|
||||
|
||||
try:
|
||||
# Test basic operations
|
||||
test_key = "health_check"
|
||||
test_value = {"timestamp": "test"}
|
||||
|
||||
await cache.set(test_key, test_value, 10)
|
||||
retrieved = await cache.get(test_key)
|
||||
await cache.delete(test_key)
|
||||
|
||||
# Get Redis info
|
||||
info = await _redis_client.info()
|
||||
|
||||
return {
|
||||
"status": "healthy",
|
||||
"available": True,
|
||||
"test_passed": retrieved == test_value,
|
||||
"connected_clients": info.get("connected_clients", 0),
|
||||
"used_memory": info.get("used_memory_human", "unknown"),
|
||||
"total_commands_processed": info.get("total_commands_processed", 0),
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": "error",
|
||||
"available": False,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
|
||||
# Context manager for cache operations
|
||||
@asynccontextmanager
|
||||
async def cache_context():
|
||||
"""Context manager for cache operations."""
|
||||
try:
|
||||
yield cache
|
||||
except Exception as e:
|
||||
logger.error(f"Cache context error: {e}")
|
||||
raise
|
||||
@@ -0,0 +1,253 @@
|
||||
"""
|
||||
Application configuration with security improvements and validation
|
||||
"""
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Dict, Any
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseSettings, validator, Field
|
||||
from pydantic.networks import AnyHttpUrl, PostgresDsn, RedisDsn
|
||||
import structlog
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings with validation"""
|
||||
|
||||
# Application
|
||||
PROJECT_NAME: str = "My Uploader Bot"
|
||||
PROJECT_VERSION: str = "2.0.0"
|
||||
PROJECT_HOST: AnyHttpUrl = Field(default="http://127.0.0.1:15100")
|
||||
SANIC_PORT: int = Field(default=15100, ge=1000, le=65535)
|
||||
DEBUG: bool = Field(default=False)
|
||||
|
||||
# Security
|
||||
SECRET_KEY: str = Field(default_factory=lambda: secrets.token_urlsafe(32))
|
||||
JWT_SECRET_KEY: str = Field(default_factory=lambda: secrets.token_urlsafe(32))
|
||||
JWT_EXPIRE_MINUTES: int = Field(default=60 * 24 * 7) # 7 days
|
||||
ENCRYPTION_KEY: Optional[str] = None
|
||||
|
||||
# Rate Limiting
|
||||
RATE_LIMIT_REQUESTS: int = Field(default=100)
|
||||
RATE_LIMIT_WINDOW: int = Field(default=60) # seconds
|
||||
RATE_LIMIT_ENABLED: bool = Field(default=True)
|
||||
|
||||
# Database
|
||||
DATABASE_URL: PostgresDsn = Field(
|
||||
default="postgresql+asyncpg://user:password@localhost:5432/uploader_bot"
|
||||
)
|
||||
DATABASE_POOL_SIZE: int = Field(default=10, ge=1, le=100)
|
||||
DATABASE_MAX_OVERFLOW: int = Field(default=20, ge=0, le=100)
|
||||
DATABASE_ECHO: bool = Field(default=False)
|
||||
|
||||
# Redis
|
||||
REDIS_URL: RedisDsn = Field(default="redis://localhost:6379/0")
|
||||
REDIS_POOL_SIZE: int = Field(default=10, ge=1, le=100)
|
||||
REDIS_TTL_DEFAULT: int = Field(default=3600) # 1 hour
|
||||
REDIS_TTL_SHORT: int = Field(default=300) # 5 minutes
|
||||
REDIS_TTL_LONG: int = Field(default=86400) # 24 hours
|
||||
|
||||
# File Storage
|
||||
UPLOADS_DIR: Path = Field(default=Path("/app/data"))
|
||||
MAX_FILE_SIZE: int = Field(default=100 * 1024 * 1024) # 100MB
|
||||
ALLOWED_CONTENT_TYPES: List[str] = Field(default=[
|
||||
'image/jpeg', 'image/png', 'image/gif', 'image/webp',
|
||||
'video/mp4', 'video/webm', 'video/ogg', 'video/quicktime',
|
||||
'audio/mpeg', 'audio/ogg', 'audio/wav', 'audio/mp4',
|
||||
'text/plain', 'application/json'
|
||||
])
|
||||
|
||||
# Telegram
|
||||
TELEGRAM_API_KEY: str = Field(..., min_length=40)
|
||||
CLIENT_TELEGRAM_API_KEY: str = Field(..., min_length=40)
|
||||
TELEGRAM_WEBHOOK_ENABLED: bool = Field(default=False)
|
||||
TELEGRAM_WEBHOOK_URL: Optional[AnyHttpUrl] = None
|
||||
TELEGRAM_WEBHOOK_SECRET: str = Field(default_factory=lambda: secrets.token_urlsafe(32))
|
||||
|
||||
# TON Blockchain
|
||||
TESTNET: bool = Field(default=False)
|
||||
TONCENTER_HOST: AnyHttpUrl = Field(default="https://toncenter.com/api/v2/")
|
||||
TONCENTER_API_KEY: Optional[str] = None
|
||||
TONCENTER_V3_HOST: AnyHttpUrl = Field(default="https://toncenter.com/api/v3/")
|
||||
MY_PLATFORM_CONTRACT: str = Field(default="EQDmWp6hbJlYUrXZKb9N88sOrTit630ZuRijfYdXEHLtheMY")
|
||||
MY_FUND_ADDRESS: str = Field(default="UQDarChHFMOI2On9IdHJNeEKttqepgo0AY4bG1trw8OAAwMY")
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL: str = Field(default="INFO", regex="^(DEBUG|INFO|WARNING|ERROR|CRITICAL)$")
|
||||
LOG_DIR: Path = Field(default=Path("logs"))
|
||||
LOG_FORMAT: str = Field(default="json")
|
||||
LOG_ROTATION: str = Field(default="1 day")
|
||||
LOG_RETENTION: str = Field(default="30 days")
|
||||
|
||||
# Monitoring
|
||||
METRICS_ENABLED: bool = Field(default=True)
|
||||
METRICS_PORT: int = Field(default=9090, ge=1000, le=65535)
|
||||
HEALTH_CHECK_ENABLED: bool = Field(default=True)
|
||||
|
||||
# Background Services
|
||||
INDEXER_ENABLED: bool = Field(default=True)
|
||||
INDEXER_INTERVAL: int = Field(default=5, ge=1, le=3600)
|
||||
TON_DAEMON_ENABLED: bool = Field(default=True)
|
||||
TON_DAEMON_INTERVAL: int = Field(default=3, ge=1, le=3600)
|
||||
LICENSE_SERVICE_ENABLED: bool = Field(default=True)
|
||||
LICENSE_SERVICE_INTERVAL: int = Field(default=10, ge=1, le=3600)
|
||||
CONVERT_SERVICE_ENABLED: bool = Field(default=True)
|
||||
CONVERT_SERVICE_INTERVAL: int = Field(default=30, ge=1, le=3600)
|
||||
|
||||
# Web App URLs
|
||||
WEB_APP_URLS: Dict[str, str] = Field(default={
|
||||
'uploadContent': "https://web2-client.vercel.app/uploadContent"
|
||||
})
|
||||
|
||||
# Maintenance
|
||||
MAINTENANCE_MODE: bool = Field(default=False)
|
||||
MAINTENANCE_MESSAGE: str = Field(default="System is under maintenance")
|
||||
|
||||
# Development
|
||||
MOCK_EXTERNAL_SERVICES: bool = Field(default=False)
|
||||
DISABLE_WEBHOOKS: bool = Field(default=False)
|
||||
|
||||
@validator('UPLOADS_DIR')
|
||||
def create_uploads_dir(cls, v):
|
||||
"""Create uploads directory if it doesn't exist"""
|
||||
if not v.exists():
|
||||
v.mkdir(parents=True, exist_ok=True)
|
||||
return v
|
||||
|
||||
@validator('LOG_DIR')
|
||||
def create_log_dir(cls, v):
|
||||
"""Create log directory if it doesn't exist"""
|
||||
if not v.exists():
|
||||
v.mkdir(parents=True, exist_ok=True)
|
||||
return v
|
||||
|
||||
@validator('DATABASE_URL')
|
||||
def validate_database_url(cls, v):
|
||||
"""Validate database URL format"""
|
||||
if not str(v).startswith('postgresql+asyncpg://'):
|
||||
raise ValueError('Database URL must use asyncpg driver')
|
||||
return v
|
||||
|
||||
@validator('TELEGRAM_API_KEY', 'CLIENT_TELEGRAM_API_KEY')
|
||||
def validate_telegram_keys(cls, v):
|
||||
"""Validate Telegram bot tokens format"""
|
||||
parts = v.split(':')
|
||||
if len(parts) != 2 or not parts[0].isdigit() or len(parts[1]) != 35:
|
||||
raise ValueError('Invalid Telegram bot token format')
|
||||
return v
|
||||
|
||||
@validator('SECRET_KEY', 'JWT_SECRET_KEY')
|
||||
def validate_secret_keys(cls, v):
|
||||
"""Validate secret keys length"""
|
||||
if len(v) < 32:
|
||||
raise ValueError('Secret keys must be at least 32 characters long')
|
||||
return v
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
case_sensitive = True
|
||||
validate_assignment = True
|
||||
|
||||
|
||||
class SecurityConfig:
|
||||
"""Security-related configurations"""
|
||||
|
||||
# CORS settings
|
||||
CORS_ORIGINS = [
|
||||
"https://web2-client.vercel.app",
|
||||
"https://t.me",
|
||||
"https://web.telegram.org"
|
||||
]
|
||||
|
||||
# Content Security Policy
|
||||
CSP_DIRECTIVES = {
|
||||
'default-src': ["'self'"],
|
||||
'script-src': ["'self'", "'unsafe-inline'", "https://cdn.jsdelivr.net"],
|
||||
'style-src': ["'self'", "'unsafe-inline'", "https://cdn.jsdelivr.net"],
|
||||
'img-src': ["'self'", "data:", "https:"],
|
||||
'connect-src': ["'self'", "https://api.telegram.org"],
|
||||
'frame-ancestors': ["'none'"],
|
||||
'form-action': ["'self'"],
|
||||
'base-uri': ["'self'"]
|
||||
}
|
||||
|
||||
# Request size limits
|
||||
MAX_REQUEST_SIZE = 100 * 1024 * 1024 # 100MB
|
||||
MAX_JSON_SIZE = 10 * 1024 * 1024 # 10MB
|
||||
|
||||
# Session settings
|
||||
SESSION_COOKIE_SECURE = True
|
||||
SESSION_COOKIE_HTTPONLY = True
|
||||
SESSION_COOKIE_SAMESITE = "Strict"
|
||||
|
||||
# Rate limiting patterns
|
||||
RATE_LIMIT_PATTERNS = {
|
||||
"auth": {"requests": 5, "window": 300}, # 5 requests per 5 minutes
|
||||
"upload": {"requests": 10, "window": 3600}, # 10 uploads per hour
|
||||
"api": {"requests": 100, "window": 60}, # 100 API calls per minute
|
||||
"heavy": {"requests": 1, "window": 60} # 1 heavy operation per minute
|
||||
}
|
||||
|
||||
|
||||
# Create settings instance
|
||||
settings = Settings()
|
||||
|
||||
# Expose commonly used settings
|
||||
DATABASE_URL = str(settings.DATABASE_URL)
|
||||
REDIS_URL = str(settings.REDIS_URL)
|
||||
DATABASE_POOL_SIZE = settings.DATABASE_POOL_SIZE
|
||||
DATABASE_MAX_OVERFLOW = settings.DATABASE_MAX_OVERFLOW
|
||||
REDIS_POOL_SIZE = settings.REDIS_POOL_SIZE
|
||||
|
||||
TELEGRAM_API_KEY = settings.TELEGRAM_API_KEY
|
||||
CLIENT_TELEGRAM_API_KEY = settings.CLIENT_TELEGRAM_API_KEY
|
||||
PROJECT_HOST = str(settings.PROJECT_HOST)
|
||||
SANIC_PORT = settings.SANIC_PORT
|
||||
UPLOADS_DIR = settings.UPLOADS_DIR
|
||||
ALLOWED_CONTENT_TYPES = settings.ALLOWED_CONTENT_TYPES
|
||||
|
||||
TESTNET = settings.TESTNET
|
||||
TONCENTER_HOST = str(settings.TONCENTER_HOST)
|
||||
TONCENTER_API_KEY = settings.TONCENTER_API_KEY
|
||||
TONCENTER_V3_HOST = str(settings.TONCENTER_V3_HOST)
|
||||
MY_PLATFORM_CONTRACT = settings.MY_PLATFORM_CONTRACT
|
||||
MY_FUND_ADDRESS = settings.MY_FUND_ADDRESS
|
||||
|
||||
LOG_LEVEL = settings.LOG_LEVEL
|
||||
LOG_DIR = settings.LOG_DIR
|
||||
MAINTENANCE_MODE = settings.MAINTENANCE_MODE
|
||||
|
||||
# Cache keys patterns
|
||||
CACHE_KEYS = {
|
||||
"user_session": "user:session:{user_id}",
|
||||
"user_data": "user:data:{user_id}",
|
||||
"content_metadata": "content:meta:{content_id}",
|
||||
"rate_limit": "rate_limit:{pattern}:{identifier}",
|
||||
"blockchain_task": "blockchain:task:{task_id}",
|
||||
"temp_upload": "upload:temp:{upload_id}",
|
||||
"wallet_connection": "wallet:conn:{wallet_address}",
|
||||
"ton_price": "ton:price:usd",
|
||||
"system_status": "system:status:{service}",
|
||||
}
|
||||
|
||||
# Log current configuration (without secrets)
|
||||
def log_config():
|
||||
"""Log current configuration without sensitive data"""
|
||||
safe_config = {
|
||||
"project_name": settings.PROJECT_NAME,
|
||||
"project_version": settings.PROJECT_VERSION,
|
||||
"debug": settings.DEBUG,
|
||||
"sanic_port": settings.SANIC_PORT,
|
||||
"testnet": settings.TESTNET,
|
||||
"maintenance_mode": settings.MAINTENANCE_MODE,
|
||||
"metrics_enabled": settings.METRICS_ENABLED,
|
||||
"uploads_dir": str(settings.UPLOADS_DIR),
|
||||
"log_level": settings.LOG_LEVEL,
|
||||
}
|
||||
logger.info("Configuration loaded", **safe_config)
|
||||
|
||||
# Initialize logging configuration
|
||||
log_config()
|
||||
@@ -0,0 +1,257 @@
|
||||
"""Compatible configuration management with MariaDB and Redis support."""
|
||||
|
||||
import os
|
||||
from functools import lru_cache
|
||||
from typing import Optional, Dict, Any
|
||||
from pydantic import BaseSettings, Field, validator
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings with backward compatibility."""
|
||||
|
||||
# Application settings
|
||||
app_name: str = Field(default="My Uploader Bot", env="APP_NAME")
|
||||
debug: bool = Field(default=False, env="DEBUG")
|
||||
environment: str = Field(default="production", env="ENVIRONMENT")
|
||||
host: str = Field(default="0.0.0.0", env="HOST")
|
||||
port: int = Field(default=15100, env="PORT")
|
||||
|
||||
# Security settings
|
||||
secret_key: str = Field(env="SECRET_KEY", default="your-secret-key-change-this")
|
||||
jwt_secret_key: str = Field(env="JWT_SECRET_KEY", default="jwt-secret-change-this")
|
||||
jwt_algorithm: str = Field(default="HS256", env="JWT_ALGORITHM")
|
||||
jwt_expire_minutes: int = Field(default=30, env="JWT_EXPIRE_MINUTES")
|
||||
|
||||
# MariaDB/MySQL settings (preserving existing configuration)
|
||||
mysql_host: str = Field(default="maria_db", env="MYSQL_HOST")
|
||||
mysql_port: int = Field(default=3306, env="MYSQL_PORT")
|
||||
mysql_user: str = Field(default="myuploader", env="MYSQL_USER")
|
||||
mysql_password: str = Field(default="password", env="MYSQL_PASSWORD")
|
||||
mysql_database: str = Field(default="myuploader", env="MYSQL_DATABASE")
|
||||
mysql_root_password: str = Field(default="password", env="MYSQL_ROOT_PASSWORD")
|
||||
|
||||
# Database pool settings
|
||||
database_pool_size: int = Field(default=20, env="DATABASE_POOL_SIZE")
|
||||
database_max_overflow: int = Field(default=30, env="DATABASE_MAX_OVERFLOW")
|
||||
database_pool_timeout: int = Field(default=30, env="DATABASE_POOL_TIMEOUT")
|
||||
database_pool_recycle: int = Field(default=3600, env="DATABASE_POOL_RECYCLE")
|
||||
|
||||
# Optional new database URL (for future migration)
|
||||
database_url: Optional[str] = Field(default=None, env="DATABASE_URL")
|
||||
|
||||
# Redis settings (new addition)
|
||||
redis_enabled: bool = Field(default=True, env="REDIS_ENABLED")
|
||||
redis_host: str = Field(default="redis", env="REDIS_HOST")
|
||||
redis_port: int = Field(default=6379, env="REDIS_PORT")
|
||||
redis_password: Optional[str] = Field(default=None, env="REDIS_PASSWORD")
|
||||
redis_db: int = Field(default=0, env="REDIS_DB")
|
||||
redis_max_connections: int = Field(default=50, env="REDIS_MAX_CONNECTIONS")
|
||||
redis_socket_timeout: int = Field(default=30, env="REDIS_SOCKET_TIMEOUT")
|
||||
redis_socket_connect_timeout: int = Field(default=30, env="REDIS_SOCKET_CONNECT_TIMEOUT")
|
||||
|
||||
# Cache settings
|
||||
cache_enabled: bool = Field(default=True, env="CACHE_ENABLED")
|
||||
cache_default_ttl: int = Field(default=300, env="CACHE_DEFAULT_TTL") # 5 minutes
|
||||
cache_user_ttl: int = Field(default=600, env="CACHE_USER_TTL") # 10 minutes
|
||||
cache_content_ttl: int = Field(default=1800, env="CACHE_CONTENT_TTL") # 30 minutes
|
||||
|
||||
# Storage settings (preserving existing paths)
|
||||
storage_path: str = Field(default="/Storage/storedContent", env="STORAGE_PATH")
|
||||
logs_path: str = Field(default="/Storage/logs", env="LOGS_PATH")
|
||||
sql_storage_path: str = Field(default="/Storage/sqlStorage", env="SQL_STORAGE_PATH")
|
||||
|
||||
# File upload settings
|
||||
max_file_size: int = Field(default=100 * 1024 * 1024, env="MAX_FILE_SIZE") # 100MB
|
||||
allowed_extensions: str = Field(default=".jpg,.jpeg,.png,.gif,.pdf,.doc,.docx,.txt", env="ALLOWED_EXTENSIONS")
|
||||
|
||||
# Rate limiting
|
||||
rate_limit_enabled: bool = Field(default=True, env="RATE_LIMIT_ENABLED")
|
||||
rate_limit_requests: int = Field(default=100, env="RATE_LIMIT_REQUESTS")
|
||||
rate_limit_window: int = Field(default=3600, env="RATE_LIMIT_WINDOW") # 1 hour
|
||||
|
||||
# TON Blockchain settings (preserving existing)
|
||||
ton_network: str = Field(default="mainnet", env="TON_NETWORK")
|
||||
ton_api_key: Optional[str] = Field(default=None, env="TON_API_KEY")
|
||||
ton_wallet_address: Optional[str] = Field(default=None, env="TON_WALLET_ADDRESS")
|
||||
|
||||
# License settings
|
||||
license_check_enabled: bool = Field(default=True, env="LICENSE_CHECK_ENABLED")
|
||||
license_server_url: Optional[str] = Field(default=None, env="LICENSE_SERVER_URL")
|
||||
|
||||
# Indexer settings
|
||||
indexer_enabled: bool = Field(default=True, env="INDEXER_ENABLED")
|
||||
indexer_interval: int = Field(default=300, env="INDEXER_INTERVAL") # 5 minutes
|
||||
|
||||
# Convert process settings
|
||||
convert_enabled: bool = Field(default=True, env="CONVERT_ENABLED")
|
||||
convert_queue_size: int = Field(default=10, env="CONVERT_QUEUE_SIZE")
|
||||
|
||||
# Logging settings
|
||||
log_level: str = Field(default="INFO", env="LOG_LEVEL")
|
||||
log_format: str = Field(default="json", env="LOG_FORMAT")
|
||||
log_file_enabled: bool = Field(default=True, env="LOG_FILE_ENABLED")
|
||||
log_file_max_size: int = Field(default=10 * 1024 * 1024, env="LOG_FILE_MAX_SIZE") # 10MB
|
||||
log_file_backup_count: int = Field(default=5, env="LOG_FILE_BACKUP_COUNT")
|
||||
|
||||
# API settings
|
||||
api_title: str = Field(default="My Uploader Bot API", env="API_TITLE")
|
||||
api_version: str = Field(default="1.0.0", env="API_VERSION")
|
||||
api_description: str = Field(default="File upload and management API", env="API_DESCRIPTION")
|
||||
cors_enabled: bool = Field(default=True, env="CORS_ENABLED")
|
||||
cors_origins: str = Field(default="*", env="CORS_ORIGINS")
|
||||
|
||||
# Health check settings
|
||||
health_check_enabled: bool = Field(default=True, env="HEALTH_CHECK_ENABLED")
|
||||
health_check_interval: int = Field(default=60, env="HEALTH_CHECK_INTERVAL")
|
||||
|
||||
# Metrics settings
|
||||
metrics_enabled: bool = Field(default=True, env="METRICS_ENABLED")
|
||||
metrics_endpoint: str = Field(default="/metrics", env="METRICS_ENDPOINT")
|
||||
|
||||
@validator("allowed_extensions")
|
||||
def validate_extensions(cls, v):
|
||||
"""Validate and normalize file extensions."""
|
||||
if isinstance(v, str):
|
||||
return [ext.strip().lower() for ext in v.split(",") if ext.strip()]
|
||||
return v
|
||||
|
||||
@validator("cors_origins")
|
||||
def validate_cors_origins(cls, v):
|
||||
"""Validate and normalize CORS origins."""
|
||||
if isinstance(v, str) and v != "*":
|
||||
return [origin.strip() for origin in v.split(",") if origin.strip()]
|
||||
return v
|
||||
|
||||
@validator("log_level")
|
||||
def validate_log_level(cls, v):
|
||||
"""Validate log level."""
|
||||
valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
if v.upper() not in valid_levels:
|
||||
raise ValueError(f"Log level must be one of: {valid_levels}")
|
||||
return v.upper()
|
||||
|
||||
def get_database_url(self) -> str:
|
||||
"""Get complete database URL."""
|
||||
if self.database_url:
|
||||
return self.database_url
|
||||
return f"mysql+aiomysql://{self.mysql_user}:{self.mysql_password}@{self.mysql_host}:{self.mysql_port}/{self.mysql_database}"
|
||||
|
||||
def get_redis_url(self) -> str:
|
||||
"""Get complete Redis URL."""
|
||||
if self.redis_password:
|
||||
return f"redis://:{self.redis_password}@{self.redis_host}:{self.redis_port}/{self.redis_db}"
|
||||
return f"redis://{self.redis_host}:{self.redis_port}/{self.redis_db}"
|
||||
|
||||
def get_allowed_extensions_set(self) -> set:
|
||||
"""Get allowed extensions as a set."""
|
||||
if isinstance(self.allowed_extensions, list):
|
||||
return set(self.allowed_extensions)
|
||||
return set(ext.strip().lower() for ext in self.allowed_extensions.split(",") if ext.strip())
|
||||
|
||||
def get_cors_origins_list(self) -> list:
|
||||
"""Get CORS origins as a list."""
|
||||
if self.cors_origins == "*":
|
||||
return ["*"]
|
||||
if isinstance(self.cors_origins, list):
|
||||
return self.cors_origins
|
||||
return [origin.strip() for origin in self.cors_origins.split(",") if origin.strip()]
|
||||
|
||||
def is_development(self) -> bool:
|
||||
"""Check if running in development mode."""
|
||||
return self.environment.lower() in ["development", "dev", "local"]
|
||||
|
||||
def is_production(self) -> bool:
|
||||
"""Check if running in production mode."""
|
||||
return self.environment.lower() in ["production", "prod"]
|
||||
|
||||
def get_cache_config(self) -> Dict[str, Any]:
|
||||
"""Get cache configuration dictionary."""
|
||||
return {
|
||||
"enabled": self.cache_enabled and self.redis_enabled,
|
||||
"default_ttl": self.cache_default_ttl,
|
||||
"user_ttl": self.cache_user_ttl,
|
||||
"content_ttl": self.cache_content_ttl,
|
||||
"redis_url": self.get_redis_url(),
|
||||
"max_connections": self.redis_max_connections,
|
||||
}
|
||||
|
||||
def get_database_config(self) -> Dict[str, Any]:
|
||||
"""Get database configuration dictionary."""
|
||||
return {
|
||||
"url": self.get_database_url(),
|
||||
"pool_size": self.database_pool_size,
|
||||
"max_overflow": self.database_max_overflow,
|
||||
"pool_timeout": self.database_pool_timeout,
|
||||
"pool_recycle": self.database_pool_recycle,
|
||||
}
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
env_file_encoding = "utf-8"
|
||||
case_sensitive = False
|
||||
|
||||
|
||||
@lru_cache()
|
||||
def get_settings() -> Settings:
|
||||
"""Get cached settings instance."""
|
||||
return Settings()
|
||||
|
||||
|
||||
# Backward compatibility functions
|
||||
def get_mysql_config() -> Dict[str, Any]:
|
||||
"""Get MySQL configuration for backward compatibility."""
|
||||
settings = get_settings()
|
||||
return {
|
||||
"host": settings.mysql_host,
|
||||
"port": settings.mysql_port,
|
||||
"user": settings.mysql_user,
|
||||
"password": settings.mysql_password,
|
||||
"database": settings.mysql_database,
|
||||
}
|
||||
|
||||
|
||||
def get_storage_config() -> Dict[str, str]:
|
||||
"""Get storage configuration for backward compatibility."""
|
||||
settings = get_settings()
|
||||
return {
|
||||
"storage_path": settings.storage_path,
|
||||
"logs_path": settings.logs_path,
|
||||
"sql_storage_path": settings.sql_storage_path,
|
||||
}
|
||||
|
||||
|
||||
def get_redis_config() -> Dict[str, Any]:
|
||||
"""Get Redis configuration."""
|
||||
settings = get_settings()
|
||||
return {
|
||||
"enabled": settings.redis_enabled,
|
||||
"host": settings.redis_host,
|
||||
"port": settings.redis_port,
|
||||
"password": settings.redis_password,
|
||||
"db": settings.redis_db,
|
||||
"max_connections": settings.redis_max_connections,
|
||||
"socket_timeout": settings.redis_socket_timeout,
|
||||
"socket_connect_timeout": settings.redis_socket_connect_timeout,
|
||||
}
|
||||
|
||||
|
||||
# Environment variables validation
|
||||
def validate_environment():
|
||||
"""Validate required environment variables."""
|
||||
settings = get_settings()
|
||||
|
||||
required_vars = [
|
||||
"SECRET_KEY",
|
||||
"JWT_SECRET_KEY",
|
||||
"MYSQL_PASSWORD",
|
||||
]
|
||||
|
||||
missing_vars = []
|
||||
for var in required_vars:
|
||||
if not os.getenv(var):
|
||||
missing_vars.append(var)
|
||||
|
||||
if missing_vars:
|
||||
raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}")
|
||||
|
||||
return True
|
||||
@@ -0,0 +1,262 @@
|
||||
"""
|
||||
Async SQLAlchemy configuration with connection pooling and Redis integration
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncGenerator, Optional
|
||||
from datetime import timedelta
|
||||
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
create_async_engine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
AsyncEngine
|
||||
)
|
||||
from sqlalchemy.pool import NullPool, QueuePool
|
||||
from sqlalchemy.sql import text
|
||||
import redis.asyncio as redis
|
||||
from redis.asyncio.connection import ConnectionPool
|
||||
import structlog
|
||||
|
||||
from app.core.config import (
|
||||
DATABASE_URL,
|
||||
REDIS_URL,
|
||||
DATABASE_POOL_SIZE,
|
||||
DATABASE_MAX_OVERFLOW,
|
||||
REDIS_POOL_SIZE
|
||||
)
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
class DatabaseManager:
|
||||
"""Async database manager with connection pooling"""
|
||||
|
||||
def __init__(self):
|
||||
self._engine: Optional[AsyncEngine] = None
|
||||
self._session_factory: Optional[async_sessionmaker[AsyncSession]] = None
|
||||
self._redis_pool: Optional[ConnectionPool] = None
|
||||
self._redis: Optional[redis.Redis] = None
|
||||
self._initialized = False
|
||||
|
||||
async def initialize(self) -> None:
|
||||
"""Initialize database connections and Redis"""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
# Initialize async SQLAlchemy engine
|
||||
self._engine = create_async_engine(
|
||||
DATABASE_URL,
|
||||
poolclass=QueuePool,
|
||||
pool_size=DATABASE_POOL_SIZE,
|
||||
max_overflow=DATABASE_MAX_OVERFLOW,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=3600, # 1 hour
|
||||
echo=False, # Set to True for SQL debugging
|
||||
future=True,
|
||||
json_serializer=lambda obj: obj,
|
||||
json_deserializer=lambda obj: obj,
|
||||
)
|
||||
|
||||
# Create session factory
|
||||
self._session_factory = async_sessionmaker(
|
||||
self._engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
autoflush=False,
|
||||
autocommit=False
|
||||
)
|
||||
|
||||
# Initialize Redis connection pool
|
||||
self._redis_pool = ConnectionPool.from_url(
|
||||
REDIS_URL,
|
||||
max_connections=REDIS_POOL_SIZE,
|
||||
retry_on_timeout=True,
|
||||
health_check_interval=30
|
||||
)
|
||||
|
||||
self._redis = redis.Redis(
|
||||
connection_pool=self._redis_pool,
|
||||
decode_responses=True
|
||||
)
|
||||
|
||||
# Test connections
|
||||
await self._test_connections()
|
||||
self._initialized = True
|
||||
|
||||
logger.info("Database and Redis connections initialized")
|
||||
|
||||
async def _test_connections(self) -> None:
|
||||
"""Test database and Redis connections"""
|
||||
# Test database
|
||||
async with self._engine.begin() as conn:
|
||||
result = await conn.execute(text("SELECT 1"))
|
||||
assert result.scalar() == 1
|
||||
|
||||
# Test Redis
|
||||
await self._redis.ping()
|
||||
logger.info("Database and Redis connections tested successfully")
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close all connections gracefully"""
|
||||
if self._engine:
|
||||
await self._engine.dispose()
|
||||
|
||||
if self._redis_pool:
|
||||
await self._redis_pool.disconnect()
|
||||
|
||||
self._initialized = False
|
||||
logger.info("Database and Redis connections closed")
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_session(self) -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Get async database session with automatic cleanup"""
|
||||
if not self._initialized:
|
||||
await self.initialize()
|
||||
|
||||
async with self._session_factory() as session:
|
||||
try:
|
||||
yield session
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error("Database session error", error=str(e))
|
||||
raise
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_transaction(self) -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Get async database session with automatic transaction management"""
|
||||
async with self.get_session() as session:
|
||||
async with session.begin():
|
||||
yield session
|
||||
|
||||
async def get_redis(self) -> redis.Redis:
|
||||
"""Get Redis client"""
|
||||
if not self._initialized:
|
||||
await self.initialize()
|
||||
return self._redis
|
||||
|
||||
@property
|
||||
def engine(self) -> AsyncEngine:
|
||||
"""Get SQLAlchemy engine"""
|
||||
if not self._engine:
|
||||
raise RuntimeError("Database not initialized")
|
||||
return self._engine
|
||||
|
||||
|
||||
class CacheManager:
|
||||
"""Redis-based cache manager with TTL and serialization"""
|
||||
|
||||
def __init__(self, redis_client: redis.Redis):
|
||||
self.redis = redis_client
|
||||
|
||||
async def get(self, key: str, default=None):
|
||||
"""Get value from cache"""
|
||||
try:
|
||||
value = await self.redis.get(key)
|
||||
return value if value is not None else default
|
||||
except Exception as e:
|
||||
logger.error("Cache get error", key=key, error=str(e))
|
||||
return default
|
||||
|
||||
async def set(
|
||||
self,
|
||||
key: str,
|
||||
value: str,
|
||||
ttl: Optional[int] = None,
|
||||
nx: bool = False
|
||||
) -> bool:
|
||||
"""Set value in cache with optional TTL"""
|
||||
try:
|
||||
return await self.redis.set(key, value, ex=ttl, nx=nx)
|
||||
except Exception as e:
|
||||
logger.error("Cache set error", key=key, error=str(e))
|
||||
return False
|
||||
|
||||
async def delete(self, key: str) -> bool:
|
||||
"""Delete key from cache"""
|
||||
try:
|
||||
return bool(await self.redis.delete(key))
|
||||
except Exception as e:
|
||||
logger.error("Cache delete error", key=key, error=str(e))
|
||||
return False
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
"""Check if key exists in cache"""
|
||||
try:
|
||||
return bool(await self.redis.exists(key))
|
||||
except Exception as e:
|
||||
logger.error("Cache exists error", key=key, error=str(e))
|
||||
return False
|
||||
|
||||
async def incr(self, key: str, amount: int = 1) -> int:
|
||||
"""Increment counter in cache"""
|
||||
try:
|
||||
return await self.redis.incr(key, amount)
|
||||
except Exception as e:
|
||||
logger.error("Cache incr error", key=key, error=str(e))
|
||||
return 0
|
||||
|
||||
async def expire(self, key: str, ttl: int) -> bool:
|
||||
"""Set TTL for existing key"""
|
||||
try:
|
||||
return await self.redis.expire(key, ttl)
|
||||
except Exception as e:
|
||||
logger.error("Cache expire error", key=key, error=str(e))
|
||||
return False
|
||||
|
||||
async def hget(self, name: str, key: str):
|
||||
"""Get hash field value"""
|
||||
try:
|
||||
return await self.redis.hget(name, key)
|
||||
except Exception as e:
|
||||
logger.error("Cache hget error", name=name, key=key, error=str(e))
|
||||
return None
|
||||
|
||||
async def hset(self, name: str, key: str, value: str) -> bool:
|
||||
"""Set hash field value"""
|
||||
try:
|
||||
return bool(await self.redis.hset(name, key, value))
|
||||
except Exception as e:
|
||||
logger.error("Cache hset error", name=name, key=key, error=str(e))
|
||||
return False
|
||||
|
||||
async def hdel(self, name: str, key: str) -> bool:
|
||||
"""Delete hash field"""
|
||||
try:
|
||||
return bool(await self.redis.hdel(name, key))
|
||||
except Exception as e:
|
||||
logger.error("Cache hdel error", name=name, key=key, error=str(e))
|
||||
return False
|
||||
|
||||
|
||||
# Global instances
|
||||
db_manager = DatabaseManager()
|
||||
cache_manager: Optional[CacheManager] = None
|
||||
|
||||
|
||||
async def get_db_session() -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Dependency for getting database session"""
|
||||
async with db_manager.get_session() as session:
|
||||
yield session
|
||||
|
||||
|
||||
async def get_cache() -> CacheManager:
|
||||
"""Dependency for getting cache manager"""
|
||||
global cache_manager
|
||||
if not cache_manager:
|
||||
redis_client = await db_manager.get_redis()
|
||||
cache_manager = CacheManager(redis_client)
|
||||
return cache_manager
|
||||
|
||||
|
||||
async def init_database():
|
||||
"""Initialize database connections"""
|
||||
await db_manager.initialize()
|
||||
|
||||
|
||||
async def close_database():
|
||||
"""Close database connections"""
|
||||
await db_manager.close()
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Compatible database configuration with MariaDB support."""
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncGenerator, Optional
|
||||
|
||||
from sqlalchemy import MetaData
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine
|
||||
)
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app.core.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Global variables for database engine and session
|
||||
_engine: Optional[AsyncEngine] = None
|
||||
_async_session: Optional[async_sessionmaker[AsyncSession]] = None
|
||||
|
||||
# Naming convention for consistent constraint names
|
||||
naming_convention = {
|
||||
"ix": "ix_%(column_0_label)s",
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s"
|
||||
}
|
||||
|
||||
metadata = MetaData(naming_convention=naming_convention)
|
||||
|
||||
|
||||
def get_database_url() -> str:
|
||||
"""Get database URL from settings."""
|
||||
settings = get_settings()
|
||||
|
||||
# Support both new DATABASE_URL and legacy MariaDB settings
|
||||
if hasattr(settings, 'database_url') and settings.database_url:
|
||||
return settings.database_url
|
||||
|
||||
# Fallback to MariaDB configuration
|
||||
mysql_host = getattr(settings, 'mysql_host', 'maria_db')
|
||||
mysql_port = getattr(settings, 'mysql_port', 3306)
|
||||
mysql_user = getattr(settings, 'mysql_user', 'myuploader')
|
||||
mysql_password = getattr(settings, 'mysql_password', 'password')
|
||||
mysql_database = getattr(settings, 'mysql_database', 'myuploader')
|
||||
|
||||
return f"mysql+aiomysql://{mysql_user}:{mysql_password}@{mysql_host}:{mysql_port}/{mysql_database}"
|
||||
|
||||
|
||||
async def init_database() -> None:
|
||||
"""Initialize database connection."""
|
||||
global _engine, _async_session
|
||||
|
||||
if _engine is not None:
|
||||
logger.warning("Database already initialized")
|
||||
return
|
||||
|
||||
try:
|
||||
settings = get_settings()
|
||||
database_url = get_database_url()
|
||||
|
||||
logger.info(f"Connecting to database: {database_url.split('@')[1] if '@' in database_url else 'unknown'}")
|
||||
|
||||
# Create async engine with MariaDB/MySQL optimizations
|
||||
_engine = create_async_engine(
|
||||
database_url,
|
||||
echo=settings.debug if hasattr(settings, 'debug') else False,
|
||||
pool_size=getattr(settings, 'database_pool_size', 20),
|
||||
max_overflow=getattr(settings, 'database_max_overflow', 30),
|
||||
pool_timeout=getattr(settings, 'database_pool_timeout', 30),
|
||||
pool_recycle=getattr(settings, 'database_pool_recycle', 3600),
|
||||
pool_pre_ping=True, # Verify connections before use
|
||||
# MariaDB specific settings
|
||||
connect_args={
|
||||
"charset": "utf8mb4",
|
||||
"use_unicode": True,
|
||||
"autocommit": False,
|
||||
}
|
||||
)
|
||||
|
||||
# Create async session factory
|
||||
_async_session = async_sessionmaker(
|
||||
bind=_engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
autoflush=True,
|
||||
autocommit=False
|
||||
)
|
||||
|
||||
# Test the connection
|
||||
async with _engine.begin() as conn:
|
||||
await conn.execute("SELECT 1")
|
||||
|
||||
logger.info("Database connection established successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize database: {e}")
|
||||
raise
|
||||
|
||||
|
||||
async def close_database() -> None:
|
||||
"""Close database connection."""
|
||||
global _engine, _async_session
|
||||
|
||||
if _engine is not None:
|
||||
logger.info("Closing database connection")
|
||||
await _engine.dispose()
|
||||
_engine = None
|
||||
_async_session = None
|
||||
logger.info("Database connection closed")
|
||||
|
||||
|
||||
def get_engine() -> AsyncEngine:
|
||||
"""Get database engine."""
|
||||
if _engine is None:
|
||||
raise RuntimeError("Database not initialized. Call init_database() first.")
|
||||
return _engine
|
||||
|
||||
|
||||
def get_session_factory() -> async_sessionmaker[AsyncSession]:
|
||||
"""Get session factory."""
|
||||
if _async_session is None:
|
||||
raise RuntimeError("Database not initialized. Call init_database() first.")
|
||||
return _async_session
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_async_session() -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Get async database session with automatic cleanup."""
|
||||
if _async_session is None:
|
||||
raise RuntimeError("Database not initialized. Call init_database() first.")
|
||||
|
||||
async with _async_session() as session:
|
||||
try:
|
||||
yield session
|
||||
except Exception as e:
|
||||
logger.error(f"Database session error: {e}")
|
||||
await session.rollback()
|
||||
raise
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def check_database_health() -> bool:
|
||||
"""Check database connection health."""
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
await session.execute("SELECT 1")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Database health check failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def get_database_info() -> dict:
|
||||
"""Get database information."""
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
# Get database version
|
||||
result = await session.execute("SELECT VERSION() as version")
|
||||
version_row = result.fetchone()
|
||||
version = version_row[0] if version_row else "Unknown"
|
||||
|
||||
# Get connection count (MariaDB specific)
|
||||
try:
|
||||
result = await session.execute("SHOW STATUS LIKE 'Threads_connected'")
|
||||
conn_row = result.fetchone()
|
||||
connections = int(conn_row[1]) if conn_row else 0
|
||||
except:
|
||||
connections = 0
|
||||
|
||||
# Get database size
|
||||
try:
|
||||
result = await session.execute("""
|
||||
SELECT
|
||||
ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) as size_mb
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = DATABASE()
|
||||
""")
|
||||
size_row = result.fetchone()
|
||||
size_mb = float(size_row[0]) if size_row and size_row[0] else 0
|
||||
except:
|
||||
size_mb = 0
|
||||
|
||||
return {
|
||||
"version": version,
|
||||
"connections": connections,
|
||||
"size_mb": size_mb,
|
||||
"engine_pool_size": _engine.pool.size() if _engine else 0,
|
||||
"engine_checked_out": _engine.pool.checkedout() if _engine else 0,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get database info: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
# Database session dependency for dependency injection
|
||||
async def get_db_session() -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Database session dependency for API routes."""
|
||||
async with get_async_session() as session:
|
||||
yield session
|
||||
|
||||
|
||||
# Backward compatibility functions
|
||||
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Legacy function name for backward compatibility."""
|
||||
async with get_async_session() as session:
|
||||
yield session
|
||||
|
||||
|
||||
# Transaction context manager
|
||||
@asynccontextmanager
|
||||
async def transaction():
|
||||
"""Transaction context manager."""
|
||||
async with get_async_session() as session:
|
||||
async with session.begin():
|
||||
yield session
|
||||
@@ -0,0 +1,363 @@
|
||||
"""
|
||||
Structured logging configuration with monitoring and observability
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Union
|
||||
from contextvars import ContextVar
|
||||
import json
|
||||
|
||||
import structlog
|
||||
from structlog.stdlib import LoggerFactory
|
||||
from structlog.typing import EventDict, Processor
|
||||
import structlog.dev
|
||||
|
||||
from app.core.config import settings, LOG_DIR, LOG_LEVEL
|
||||
|
||||
# Context variables for request tracking
|
||||
request_id_var: ContextVar[Optional[str]] = ContextVar('request_id', default=None)
|
||||
user_id_var: ContextVar[Optional[int]] = ContextVar('user_id', default=None)
|
||||
operation_var: ContextVar[Optional[str]] = ContextVar('operation', default=None)
|
||||
|
||||
|
||||
class RequestContextProcessor:
|
||||
"""Add request context to log records"""
|
||||
|
||||
def __call__(self, logger, method_name, event_dict: EventDict) -> EventDict:
|
||||
"""Add context variables to event dict"""
|
||||
if request_id := request_id_var.get(None):
|
||||
event_dict['request_id'] = request_id
|
||||
|
||||
if user_id := user_id_var.get(None):
|
||||
event_dict['user_id'] = user_id
|
||||
|
||||
if operation := operation_var.get(None):
|
||||
event_dict['operation'] = operation
|
||||
|
||||
return event_dict
|
||||
|
||||
|
||||
class TimestampProcessor:
|
||||
"""Add consistent timestamp to log records"""
|
||||
|
||||
def __call__(self, logger, method_name, event_dict: EventDict) -> EventDict:
|
||||
"""Add timestamp to event dict"""
|
||||
event_dict['timestamp'] = datetime.utcnow().isoformat() + 'Z'
|
||||
return event_dict
|
||||
|
||||
|
||||
class SecurityProcessor:
|
||||
"""Filter sensitive data from logs"""
|
||||
|
||||
SENSITIVE_KEYS = {
|
||||
'password', 'token', 'key', 'secret', 'auth', 'credential',
|
||||
'private_key', 'seed', 'mnemonic', 'api_key', 'authorization'
|
||||
}
|
||||
|
||||
def __call__(self, logger, method_name, event_dict: EventDict) -> EventDict:
|
||||
"""Remove or mask sensitive data"""
|
||||
return self._filter_dict(event_dict)
|
||||
|
||||
def _filter_dict(self, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Recursively filter sensitive data"""
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
filtered = {}
|
||||
for key, value in data.items():
|
||||
if any(sensitive in key.lower() for sensitive in self.SENSITIVE_KEYS):
|
||||
filtered[key] = '***REDACTED***'
|
||||
elif isinstance(value, dict):
|
||||
filtered[key] = self._filter_dict(value)
|
||||
elif isinstance(value, list):
|
||||
filtered[key] = [
|
||||
self._filter_dict(item) if isinstance(item, dict) else item
|
||||
for item in value
|
||||
]
|
||||
else:
|
||||
filtered[key] = value
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
class PerformanceProcessor:
|
||||
"""Add performance metrics to log records"""
|
||||
|
||||
def __call__(self, logger, method_name, event_dict: EventDict) -> EventDict:
|
||||
"""Add performance data to event dict"""
|
||||
# Add memory usage if available
|
||||
try:
|
||||
import psutil
|
||||
process = psutil.Process()
|
||||
event_dict['memory_mb'] = round(process.memory_info().rss / 1024 / 1024, 2)
|
||||
event_dict['cpu_percent'] = process.cpu_percent()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
return event_dict
|
||||
|
||||
|
||||
class MetricsCollector:
|
||||
"""Collect metrics from log events"""
|
||||
|
||||
def __init__(self):
|
||||
self.counters: Dict[str, int] = {}
|
||||
self.timers: Dict[str, float] = {}
|
||||
self.errors: Dict[str, int] = {}
|
||||
|
||||
def increment_counter(self, metric: str, value: int = 1):
|
||||
"""Increment counter metric"""
|
||||
self.counters[metric] = self.counters.get(metric, 0) + value
|
||||
|
||||
def record_timer(self, metric: str, duration: float):
|
||||
"""Record timer metric"""
|
||||
self.timers[metric] = duration
|
||||
|
||||
def record_error(self, error_type: str):
|
||||
"""Record error metric"""
|
||||
self.errors[error_type] = self.errors.get(error_type, 0) + 1
|
||||
|
||||
def get_metrics(self) -> Dict[str, Any]:
|
||||
"""Get all collected metrics"""
|
||||
return {
|
||||
'counters': self.counters,
|
||||
'timers': self.timers,
|
||||
'errors': self.errors
|
||||
}
|
||||
|
||||
|
||||
# Global metrics collector
|
||||
metrics_collector = MetricsCollector()
|
||||
|
||||
|
||||
class DatabaseLogHandler(logging.Handler):
|
||||
"""Log handler that stores critical logs in database"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.setLevel(logging.ERROR)
|
||||
self._queue = asyncio.Queue(maxsize=1000)
|
||||
self._task = None
|
||||
|
||||
def emit(self, record: logging.LogRecord):
|
||||
"""Add log record to queue"""
|
||||
try:
|
||||
log_entry = {
|
||||
'timestamp': datetime.utcnow(),
|
||||
'level': record.levelname,
|
||||
'logger': record.name,
|
||||
'message': record.getMessage(),
|
||||
'module': record.module,
|
||||
'function': record.funcName,
|
||||
'line': record.lineno,
|
||||
'request_id': getattr(record, 'request_id', None),
|
||||
'user_id': getattr(record, 'user_id', None),
|
||||
'extra': getattr(record, '__dict__', {})
|
||||
}
|
||||
|
||||
if not self._queue.full():
|
||||
self._queue.put_nowait(log_entry)
|
||||
|
||||
except Exception:
|
||||
# Don't let logging errors break the application
|
||||
pass
|
||||
|
||||
async def process_logs(self):
|
||||
"""Process logs from queue and store in database"""
|
||||
from app.core.database import get_db_session
|
||||
|
||||
while True:
|
||||
try:
|
||||
log_entry = await self._queue.get()
|
||||
|
||||
# Store in database (implement based on your log model)
|
||||
# async with get_db_session() as session:
|
||||
# log_record = LogRecord(**log_entry)
|
||||
# session.add(log_record)
|
||||
# await session.commit()
|
||||
|
||||
except Exception as e:
|
||||
# Log to stderr to avoid infinite recursion
|
||||
print(f"Database log handler error: {e}", file=sys.stderr)
|
||||
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
|
||||
def configure_logging():
|
||||
"""Configure structured logging"""
|
||||
|
||||
# Configure standard library logging
|
||||
logging.basicConfig(
|
||||
format="%(message)s",
|
||||
stream=sys.stdout,
|
||||
level=getattr(logging, LOG_LEVEL.upper())
|
||||
)
|
||||
|
||||
# Silence noisy loggers
|
||||
logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING)
|
||||
logging.getLogger("aioredis").setLevel(logging.WARNING)
|
||||
logging.getLogger("aiogram").setLevel(logging.WARNING)
|
||||
|
||||
# Configure processors based on environment
|
||||
processors: list[Processor] = [
|
||||
structlog.contextvars.merge_contextvars,
|
||||
RequestContextProcessor(),
|
||||
TimestampProcessor(),
|
||||
SecurityProcessor(),
|
||||
structlog.processors.add_log_level,
|
||||
structlog.processors.StackInfoRenderer(),
|
||||
]
|
||||
|
||||
if settings.DEBUG:
|
||||
processors.extend([
|
||||
PerformanceProcessor(),
|
||||
structlog.dev.ConsoleRenderer(colors=True)
|
||||
])
|
||||
else:
|
||||
processors.append(structlog.processors.JSONRenderer())
|
||||
|
||||
# Configure structlog
|
||||
structlog.configure(
|
||||
processors=processors,
|
||||
wrapper_class=structlog.make_filtering_bound_logger(
|
||||
getattr(logging, LOG_LEVEL.upper())
|
||||
),
|
||||
logger_factory=LoggerFactory(),
|
||||
cache_logger_on_first_use=True,
|
||||
)
|
||||
|
||||
# Add file handler for persistent logging
|
||||
if not settings.DEBUG:
|
||||
log_file = LOG_DIR / f"app_{datetime.now().strftime('%Y%m%d')}.log"
|
||||
file_handler = logging.FileHandler(log_file, encoding='utf-8')
|
||||
file_handler.setFormatter(
|
||||
logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
)
|
||||
logging.getLogger().addHandler(file_handler)
|
||||
|
||||
|
||||
class LoggerMixin:
|
||||
"""Mixin to add structured logging to classes"""
|
||||
|
||||
@property
|
||||
def logger(self):
|
||||
"""Get logger for this class"""
|
||||
return structlog.get_logger(self.__class__.__name__)
|
||||
|
||||
|
||||
class AsyncContextLogger:
|
||||
"""Context manager for async operations with automatic logging"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
operation: str,
|
||||
logger: Optional[structlog.BoundLogger] = None,
|
||||
log_args: bool = True,
|
||||
log_result: bool = True
|
||||
):
|
||||
self.operation = operation
|
||||
self.logger = logger or structlog.get_logger()
|
||||
self.log_args = log_args
|
||||
self.log_result = log_result
|
||||
self.start_time = None
|
||||
|
||||
async def __aenter__(self):
|
||||
"""Enter async context"""
|
||||
self.start_time = time.time()
|
||||
operation_var.set(self.operation)
|
||||
|
||||
self.logger.info(
|
||||
"Operation started",
|
||||
operation=self.operation,
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Exit async context with performance logging"""
|
||||
duration = time.time() - self.start_time
|
||||
|
||||
if exc_type:
|
||||
self.logger.error(
|
||||
"Operation failed",
|
||||
operation=self.operation,
|
||||
duration_ms=round(duration * 1000, 2),
|
||||
error_type=exc_type.__name__,
|
||||
error_message=str(exc_val)
|
||||
)
|
||||
metrics_collector.record_error(f"{self.operation}_error")
|
||||
else:
|
||||
self.logger.info(
|
||||
"Operation completed",
|
||||
operation=self.operation,
|
||||
duration_ms=round(duration * 1000, 2)
|
||||
)
|
||||
|
||||
metrics_collector.record_timer(f"{self.operation}_duration", duration)
|
||||
operation_var.set(None)
|
||||
|
||||
|
||||
def get_logger(name: str = None) -> structlog.BoundLogger:
|
||||
"""Get configured structured logger"""
|
||||
return structlog.get_logger(name)
|
||||
|
||||
|
||||
# Compatibility wrapper for old logging
|
||||
def make_log(
|
||||
component: Optional[str],
|
||||
message: str,
|
||||
level: str = 'info',
|
||||
**kwargs
|
||||
):
|
||||
"""Legacy logging function for backward compatibility"""
|
||||
logger = get_logger(component or 'Legacy')
|
||||
log_func = getattr(logger, level.lower(), logger.info)
|
||||
log_func(message, **kwargs)
|
||||
|
||||
|
||||
# Performance monitoring decorator
|
||||
def log_performance(operation: str = None):
|
||||
"""Decorator to log function performance"""
|
||||
def decorator(func):
|
||||
async def async_wrapper(*args, **kwargs):
|
||||
op_name = operation or f"{func.__module__}.{func.__name__}"
|
||||
async with AsyncContextLogger(op_name):
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
def sync_wrapper(*args, **kwargs):
|
||||
op_name = operation or f"{func.__module__}.{func.__name__}"
|
||||
start_time = time.time()
|
||||
logger = get_logger(func.__module__)
|
||||
|
||||
try:
|
||||
logger.info("Function started", function=op_name)
|
||||
result = func(*args, **kwargs)
|
||||
duration = time.time() - start_time
|
||||
logger.info(
|
||||
"Function completed",
|
||||
function=op_name,
|
||||
duration_ms=round(duration * 1000, 2)
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
logger.error(
|
||||
"Function failed",
|
||||
function=op_name,
|
||||
duration_ms=round(duration * 1000, 2),
|
||||
error=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
return async_wrapper if asyncio.iscoroutinefunction(func) else sync_wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
# Initialize logging
|
||||
configure_logging()
|
||||
@@ -0,0 +1,566 @@
|
||||
"""Prometheus metrics collection for my-uploader-bot."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime
|
||||
from functools import wraps
|
||||
from typing import Dict, Any, Optional, Callable
|
||||
|
||||
from prometheus_client import Counter, Histogram, Gauge, Info, generate_latest, CONTENT_TYPE_LATEST
|
||||
from sanic import Request, Response
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Application info
|
||||
APP_INFO = Info('myuploader_app_info', 'Application information')
|
||||
APP_INFO.info({
|
||||
'version': '2.0.0',
|
||||
'name': 'my-uploader-bot',
|
||||
'python_version': '3.11+'
|
||||
})
|
||||
|
||||
# HTTP request metrics
|
||||
HTTP_REQUESTS_TOTAL = Counter(
|
||||
'http_requests_total',
|
||||
'Total HTTP requests',
|
||||
['method', 'endpoint', 'status_code']
|
||||
)
|
||||
|
||||
HTTP_REQUEST_DURATION = Histogram(
|
||||
'http_request_duration_seconds',
|
||||
'HTTP request duration in seconds',
|
||||
['method', 'endpoint']
|
||||
)
|
||||
|
||||
HTTP_REQUEST_SIZE = Histogram(
|
||||
'http_request_size_bytes',
|
||||
'HTTP request size in bytes',
|
||||
['method', 'endpoint']
|
||||
)
|
||||
|
||||
HTTP_RESPONSE_SIZE = Histogram(
|
||||
'http_response_size_bytes',
|
||||
'HTTP response size in bytes',
|
||||
['method', 'endpoint']
|
||||
)
|
||||
|
||||
# Authentication metrics
|
||||
AUTH_LOGIN_ATTEMPTS_TOTAL = Counter(
|
||||
'auth_login_attempts_total',
|
||||
'Total login attempts',
|
||||
['status']
|
||||
)
|
||||
|
||||
AUTH_LOGIN_FAILURES_TOTAL = Counter(
|
||||
'auth_login_failures_total',
|
||||
'Total login failures',
|
||||
['reason']
|
||||
)
|
||||
|
||||
AUTH_API_KEY_USAGE_TOTAL = Counter(
|
||||
'auth_api_key_usage_total',
|
||||
'Total API key usage',
|
||||
['key_id', 'status']
|
||||
)
|
||||
|
||||
# File upload metrics
|
||||
UPLOAD_REQUESTS_TOTAL = Counter(
|
||||
'upload_requests_total',
|
||||
'Total upload requests',
|
||||
['status', 'file_type']
|
||||
)
|
||||
|
||||
UPLOAD_SIZE_BYTES = Histogram(
|
||||
'upload_size_bytes',
|
||||
'File upload size in bytes',
|
||||
['file_type']
|
||||
)
|
||||
|
||||
UPLOAD_DURATION_SECONDS = Histogram(
|
||||
'upload_duration_seconds',
|
||||
'File upload duration in seconds',
|
||||
['file_type']
|
||||
)
|
||||
|
||||
UPLOAD_QUEUE_SIZE = Gauge(
|
||||
'upload_queue_size',
|
||||
'Number of files in upload queue'
|
||||
)
|
||||
|
||||
UPLOAD_FAILURES_TOTAL = Counter(
|
||||
'upload_failures_total',
|
||||
'Total upload failures',
|
||||
['reason', 'file_type']
|
||||
)
|
||||
|
||||
# File processing metrics
|
||||
PROCESSING_QUEUE_SIZE = Gauge(
|
||||
'processing_queue_size',
|
||||
'Number of files in processing queue'
|
||||
)
|
||||
|
||||
PROCESSING_DURATION_SECONDS = Histogram(
|
||||
'processing_duration_seconds',
|
||||
'File processing duration in seconds',
|
||||
['file_type', 'operation']
|
||||
)
|
||||
|
||||
PROCESSING_FAILURES_TOTAL = Counter(
|
||||
'processing_failures_total',
|
||||
'Total processing failures',
|
||||
['file_type', 'operation']
|
||||
)
|
||||
|
||||
# Database metrics
|
||||
DB_CONNECTIONS_ACTIVE = Gauge(
|
||||
'db_connections_active',
|
||||
'Number of active database connections'
|
||||
)
|
||||
|
||||
DB_CONNECTIONS_IDLE = Gauge(
|
||||
'db_connections_idle',
|
||||
'Number of idle database connections'
|
||||
)
|
||||
|
||||
DB_QUERY_DURATION_SECONDS = Histogram(
|
||||
'db_query_duration_seconds',
|
||||
'Database query duration in seconds',
|
||||
['operation']
|
||||
)
|
||||
|
||||
DB_TRANSACTIONS_TOTAL = Counter(
|
||||
'db_transactions_total',
|
||||
'Total database transactions',
|
||||
['status']
|
||||
)
|
||||
|
||||
# Cache metrics
|
||||
CACHE_OPERATIONS_TOTAL = Counter(
|
||||
'cache_operations_total',
|
||||
'Total cache operations',
|
||||
['operation', 'status']
|
||||
)
|
||||
|
||||
CACHE_HIT_RATIO = Gauge(
|
||||
'cache_hit_ratio',
|
||||
'Cache hit ratio'
|
||||
)
|
||||
|
||||
CACHE_KEYS_TOTAL = Gauge(
|
||||
'cache_keys_total',
|
||||
'Total number of cache keys'
|
||||
)
|
||||
|
||||
CACHE_MEMORY_USAGE_BYTES = Gauge(
|
||||
'cache_memory_usage_bytes',
|
||||
'Cache memory usage in bytes'
|
||||
)
|
||||
|
||||
# Storage metrics
|
||||
STORAGE_OPERATIONS_TOTAL = Counter(
|
||||
'storage_operations_total',
|
||||
'Total storage operations',
|
||||
['operation', 'backend', 'status']
|
||||
)
|
||||
|
||||
STORAGE_AVAILABLE_BYTES = Gauge(
|
||||
'storage_available_bytes',
|
||||
'Available storage space in bytes',
|
||||
['backend']
|
||||
)
|
||||
|
||||
STORAGE_TOTAL_BYTES = Gauge(
|
||||
'storage_total_bytes',
|
||||
'Total storage space in bytes',
|
||||
['backend']
|
||||
)
|
||||
|
||||
STORAGE_FILES_TOTAL = Gauge(
|
||||
'storage_files_total',
|
||||
'Total number of stored files',
|
||||
['backend']
|
||||
)
|
||||
|
||||
# Blockchain metrics
|
||||
BLOCKCHAIN_TRANSACTIONS_TOTAL = Counter(
|
||||
'blockchain_transactions_total',
|
||||
'Total blockchain transactions',
|
||||
['status', 'network']
|
||||
)
|
||||
|
||||
BLOCKCHAIN_TRANSACTION_FEES = Histogram(
|
||||
'blockchain_transaction_fees',
|
||||
'Blockchain transaction fees',
|
||||
['network']
|
||||
)
|
||||
|
||||
BLOCKCHAIN_PENDING_TRANSACTIONS = Gauge(
|
||||
'blockchain_pending_transactions',
|
||||
'Number of pending blockchain transactions'
|
||||
)
|
||||
|
||||
BLOCKCHAIN_WALLET_BALANCES = Gauge(
|
||||
'blockchain_wallet_balances',
|
||||
'Wallet balances',
|
||||
['wallet_id', 'currency']
|
||||
)
|
||||
|
||||
TON_SERVICE_UP = Gauge(
|
||||
'ton_service_up',
|
||||
'TON service availability (1 = up, 0 = down)'
|
||||
)
|
||||
|
||||
# Security metrics
|
||||
RATE_LIMIT_HITS_TOTAL = Counter(
|
||||
'rate_limit_hits_total',
|
||||
'Total rate limit hits',
|
||||
['endpoint', 'user_id']
|
||||
)
|
||||
|
||||
SECURITY_EVENTS_TOTAL = Counter(
|
||||
'security_events_total',
|
||||
'Total security events',
|
||||
['event_type', 'severity']
|
||||
)
|
||||
|
||||
SECURITY_SUSPICIOUS_EVENTS = Gauge(
|
||||
'security_suspicious_events',
|
||||
'Number of suspicious security events in the last hour'
|
||||
)
|
||||
|
||||
FAILED_LOGIN_ATTEMPTS = Counter(
|
||||
'failed_login_attempts_total',
|
||||
'Total failed login attempts',
|
||||
['ip_address', 'reason']
|
||||
)
|
||||
|
||||
# System metrics
|
||||
SYSTEM_UPTIME_SECONDS = Gauge(
|
||||
'system_uptime_seconds',
|
||||
'System uptime in seconds'
|
||||
)
|
||||
|
||||
BACKGROUND_TASKS_ACTIVE = Gauge(
|
||||
'background_tasks_active',
|
||||
'Number of active background tasks',
|
||||
['service']
|
||||
)
|
||||
|
||||
BACKGROUND_TASKS_COMPLETED = Counter(
|
||||
'background_tasks_completed_total',
|
||||
'Total completed background tasks',
|
||||
['service', 'status']
|
||||
)
|
||||
|
||||
# Error metrics
|
||||
ERROR_RATE = Gauge(
|
||||
'error_rate',
|
||||
'Application error rate'
|
||||
)
|
||||
|
||||
EXCEPTIONS_TOTAL = Counter(
|
||||
'exceptions_total',
|
||||
'Total exceptions',
|
||||
['exception_type', 'handler']
|
||||
)
|
||||
|
||||
|
||||
class MetricsCollector:
|
||||
"""Centralized metrics collection and management."""
|
||||
|
||||
def __init__(self):
|
||||
self.start_time = time.time()
|
||||
self._cache_stats = {
|
||||
'hits': 0,
|
||||
'misses': 0,
|
||||
'operations': 0
|
||||
}
|
||||
|
||||
def record_http_request(
|
||||
self,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
status_code: int,
|
||||
duration: float,
|
||||
request_size: int = 0,
|
||||
response_size: int = 0
|
||||
):
|
||||
"""Record HTTP request metrics."""
|
||||
HTTP_REQUESTS_TOTAL.labels(
|
||||
method=method,
|
||||
endpoint=endpoint,
|
||||
status_code=status_code
|
||||
).inc()
|
||||
|
||||
HTTP_REQUEST_DURATION.labels(
|
||||
method=method,
|
||||
endpoint=endpoint
|
||||
).observe(duration)
|
||||
|
||||
if request_size > 0:
|
||||
HTTP_REQUEST_SIZE.labels(
|
||||
method=method,
|
||||
endpoint=endpoint
|
||||
).observe(request_size)
|
||||
|
||||
if response_size > 0:
|
||||
HTTP_RESPONSE_SIZE.labels(
|
||||
method=method,
|
||||
endpoint=endpoint
|
||||
).observe(response_size)
|
||||
|
||||
def record_auth_event(self, event_type: str, status: str, **labels):
|
||||
"""Record authentication events."""
|
||||
if event_type == 'login':
|
||||
AUTH_LOGIN_ATTEMPTS_TOTAL.labels(status=status).inc()
|
||||
if status == 'failed':
|
||||
reason = labels.get('reason', 'unknown')
|
||||
AUTH_LOGIN_FAILURES_TOTAL.labels(reason=reason).inc()
|
||||
elif event_type == 'api_key':
|
||||
key_id = labels.get('key_id', 'unknown')
|
||||
AUTH_API_KEY_USAGE_TOTAL.labels(key_id=key_id, status=status).inc()
|
||||
|
||||
def record_upload_event(
|
||||
self,
|
||||
status: str,
|
||||
file_type: str,
|
||||
file_size: int = 0,
|
||||
duration: float = 0,
|
||||
**kwargs
|
||||
):
|
||||
"""Record file upload events."""
|
||||
UPLOAD_REQUESTS_TOTAL.labels(status=status, file_type=file_type).inc()
|
||||
|
||||
if file_size > 0:
|
||||
UPLOAD_SIZE_BYTES.labels(file_type=file_type).observe(file_size)
|
||||
|
||||
if duration > 0:
|
||||
UPLOAD_DURATION_SECONDS.labels(file_type=file_type).observe(duration)
|
||||
|
||||
if status == 'failed':
|
||||
reason = kwargs.get('reason', 'unknown')
|
||||
UPLOAD_FAILURES_TOTAL.labels(reason=reason, file_type=file_type).inc()
|
||||
|
||||
def record_processing_event(
|
||||
self,
|
||||
file_type: str,
|
||||
operation: str,
|
||||
duration: float = 0,
|
||||
status: str = 'success'
|
||||
):
|
||||
"""Record file processing events."""
|
||||
if duration > 0:
|
||||
PROCESSING_DURATION_SECONDS.labels(
|
||||
file_type=file_type,
|
||||
operation=operation
|
||||
).observe(duration)
|
||||
|
||||
if status == 'failed':
|
||||
PROCESSING_FAILURES_TOTAL.labels(
|
||||
file_type=file_type,
|
||||
operation=operation
|
||||
).inc()
|
||||
|
||||
def record_db_event(self, operation: str, duration: float = 0, status: str = 'success'):
|
||||
"""Record database events."""
|
||||
if duration > 0:
|
||||
DB_QUERY_DURATION_SECONDS.labels(operation=operation).observe(duration)
|
||||
|
||||
DB_TRANSACTIONS_TOTAL.labels(status=status).inc()
|
||||
|
||||
def record_cache_event(self, operation: str, status: str):
|
||||
"""Record cache events."""
|
||||
CACHE_OPERATIONS_TOTAL.labels(operation=operation, status=status).inc()
|
||||
|
||||
# Update cache stats
|
||||
self._cache_stats['operations'] += 1
|
||||
if status == 'hit':
|
||||
self._cache_stats['hits'] += 1
|
||||
elif status == 'miss':
|
||||
self._cache_stats['misses'] += 1
|
||||
|
||||
# Update hit ratio
|
||||
if self._cache_stats['operations'] > 0:
|
||||
hit_ratio = self._cache_stats['hits'] / self._cache_stats['operations']
|
||||
CACHE_HIT_RATIO.set(hit_ratio)
|
||||
|
||||
def record_blockchain_event(
|
||||
self,
|
||||
event_type: str,
|
||||
status: str,
|
||||
network: str = 'mainnet',
|
||||
**kwargs
|
||||
):
|
||||
"""Record blockchain events."""
|
||||
if event_type == 'transaction':
|
||||
BLOCKCHAIN_TRANSACTIONS_TOTAL.labels(status=status, network=network).inc()
|
||||
|
||||
if 'fee' in kwargs:
|
||||
BLOCKCHAIN_TRANSACTION_FEES.labels(network=network).observe(kwargs['fee'])
|
||||
|
||||
def record_security_event(self, event_type: str, severity: str = 'info', **kwargs):
|
||||
"""Record security events."""
|
||||
SECURITY_EVENTS_TOTAL.labels(event_type=event_type, severity=severity).inc()
|
||||
|
||||
if event_type == 'rate_limit':
|
||||
endpoint = kwargs.get('endpoint', 'unknown')
|
||||
user_id = kwargs.get('user_id', 'anonymous')
|
||||
RATE_LIMIT_HITS_TOTAL.labels(endpoint=endpoint, user_id=user_id).inc()
|
||||
|
||||
elif event_type == 'failed_login':
|
||||
ip_address = kwargs.get('ip_address', 'unknown')
|
||||
reason = kwargs.get('reason', 'unknown')
|
||||
FAILED_LOGIN_ATTEMPTS.labels(ip_address=ip_address, reason=reason).inc()
|
||||
|
||||
def update_system_metrics(self):
|
||||
"""Update system-level metrics."""
|
||||
uptime = time.time() - self.start_time
|
||||
SYSTEM_UPTIME_SECONDS.set(uptime)
|
||||
|
||||
def update_gauge_metrics(self, metrics_data: Dict[str, Any]):
|
||||
"""Update gauge metrics from external data."""
|
||||
# Database metrics
|
||||
if 'db_connections' in metrics_data:
|
||||
db_conn = metrics_data['db_connections']
|
||||
DB_CONNECTIONS_ACTIVE.set(db_conn.get('active', 0))
|
||||
DB_CONNECTIONS_IDLE.set(db_conn.get('idle', 0))
|
||||
|
||||
# Cache metrics
|
||||
if 'cache' in metrics_data:
|
||||
cache_data = metrics_data['cache']
|
||||
CACHE_KEYS_TOTAL.set(cache_data.get('keys', 0))
|
||||
CACHE_MEMORY_USAGE_BYTES.set(cache_data.get('memory_usage', 0))
|
||||
|
||||
# Storage metrics
|
||||
if 'storage' in metrics_data:
|
||||
storage_data = metrics_data['storage']
|
||||
for backend, data in storage_data.items():
|
||||
STORAGE_AVAILABLE_BYTES.labels(backend=backend).set(data.get('available', 0))
|
||||
STORAGE_TOTAL_BYTES.labels(backend=backend).set(data.get('total', 0))
|
||||
STORAGE_FILES_TOTAL.labels(backend=backend).set(data.get('files', 0))
|
||||
|
||||
# Queue metrics
|
||||
if 'queues' in metrics_data:
|
||||
queues = metrics_data['queues']
|
||||
UPLOAD_QUEUE_SIZE.set(queues.get('upload', 0))
|
||||
PROCESSING_QUEUE_SIZE.set(queues.get('processing', 0))
|
||||
|
||||
# Blockchain metrics
|
||||
if 'blockchain' in metrics_data:
|
||||
blockchain_data = metrics_data['blockchain']
|
||||
BLOCKCHAIN_PENDING_TRANSACTIONS.set(blockchain_data.get('pending_transactions', 0))
|
||||
TON_SERVICE_UP.set(1 if blockchain_data.get('ton_service_up') else 0)
|
||||
|
||||
# Wallet balances
|
||||
for wallet_id, balance_data in blockchain_data.get('wallet_balances', {}).items():
|
||||
for currency, balance in balance_data.items():
|
||||
BLOCKCHAIN_WALLET_BALANCES.labels(
|
||||
wallet_id=wallet_id,
|
||||
currency=currency
|
||||
).set(balance)
|
||||
|
||||
# Background tasks
|
||||
if 'background_tasks' in metrics_data:
|
||||
tasks_data = metrics_data['background_tasks']
|
||||
for service, count in tasks_data.items():
|
||||
BACKGROUND_TASKS_ACTIVE.labels(service=service).set(count)
|
||||
|
||||
|
||||
# Global metrics collector instance
|
||||
metrics_collector = MetricsCollector()
|
||||
|
||||
|
||||
def metrics_middleware(request: Request, response: Response):
|
||||
"""Middleware to collect HTTP metrics."""
|
||||
start_time = time.time()
|
||||
|
||||
# After request processing
|
||||
duration = time.time() - start_time
|
||||
|
||||
# Get endpoint info
|
||||
endpoint = request.path
|
||||
method = request.method
|
||||
status_code = response.status
|
||||
|
||||
# Get request/response sizes
|
||||
request_size = len(request.body) if request.body else 0
|
||||
response_size = len(response.body) if hasattr(response, 'body') and response.body else 0
|
||||
|
||||
# Record metrics
|
||||
metrics_collector.record_http_request(
|
||||
method=method,
|
||||
endpoint=endpoint,
|
||||
status_code=status_code,
|
||||
duration=duration,
|
||||
request_size=request_size,
|
||||
response_size=response_size
|
||||
)
|
||||
|
||||
|
||||
def track_function_calls(func_name: str, labels: Optional[Dict[str, str]] = None):
|
||||
"""Decorator to track function call metrics."""
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
async def async_wrapper(*args, **kwargs):
|
||||
start_time = time.time()
|
||||
status = 'success'
|
||||
|
||||
try:
|
||||
result = await func(*args, **kwargs)
|
||||
return result
|
||||
except Exception as e:
|
||||
status = 'error'
|
||||
EXCEPTIONS_TOTAL.labels(
|
||||
exception_type=type(e).__name__,
|
||||
handler=func_name
|
||||
).inc()
|
||||
raise
|
||||
finally:
|
||||
duration = time.time() - start_time
|
||||
# Record custom metrics based on function type
|
||||
if func_name.startswith('db_'):
|
||||
metrics_collector.record_db_event(func_name, duration, status)
|
||||
elif func_name.startswith('cache_'):
|
||||
metrics_collector.record_cache_event(func_name, status)
|
||||
|
||||
@wraps(func)
|
||||
def sync_wrapper(*args, **kwargs):
|
||||
start_time = time.time()
|
||||
status = 'success'
|
||||
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
return result
|
||||
except Exception as e:
|
||||
status = 'error'
|
||||
EXCEPTIONS_TOTAL.labels(
|
||||
exception_type=type(e).__name__,
|
||||
handler=func_name
|
||||
).inc()
|
||||
raise
|
||||
finally:
|
||||
duration = time.time() - start_time
|
||||
# Record custom metrics based on function type
|
||||
if func_name.startswith('db_'):
|
||||
metrics_collector.record_db_event(func_name, duration, status)
|
||||
elif func_name.startswith('cache_'):
|
||||
metrics_collector.record_cache_event(func_name, status)
|
||||
|
||||
return async_wrapper if asyncio.iscoroutinefunction(func) else sync_wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
async def get_metrics():
|
||||
"""Get Prometheus metrics."""
|
||||
# Update system metrics before generating output
|
||||
metrics_collector.update_system_metrics()
|
||||
|
||||
# Generate metrics in Prometheus format
|
||||
return generate_latest()
|
||||
|
||||
|
||||
def get_metrics_content_type():
|
||||
"""Get the content type for metrics."""
|
||||
return CONTENT_TYPE_LATEST
|
||||
+276
-2
@@ -1,3 +1,277 @@
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
"""
|
||||
Base model classes with async SQLAlchemy support
|
||||
"""
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional, Type, TypeVar, Union
|
||||
|
||||
AlchemyBase = declarative_base()
|
||||
from sqlalchemy import Column, DateTime, String, Boolean, Integer, Text, JSON
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from pydantic import BaseModel
|
||||
import structlog
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# Create declarative base
|
||||
Base = declarative_base()
|
||||
|
||||
# Type variable for model classes
|
||||
ModelType = TypeVar("ModelType", bound="BaseModel")
|
||||
|
||||
|
||||
class TimestampMixin:
|
||||
"""Mixin for automatic timestamp fields"""
|
||||
|
||||
created_at = Column(
|
||||
DateTime,
|
||||
nullable=False,
|
||||
default=datetime.utcnow,
|
||||
comment="Record creation timestamp"
|
||||
)
|
||||
updated_at = Column(
|
||||
DateTime,
|
||||
nullable=False,
|
||||
default=datetime.utcnow,
|
||||
onupdate=datetime.utcnow,
|
||||
comment="Record last update timestamp"
|
||||
)
|
||||
|
||||
|
||||
class UUIDMixin:
|
||||
"""Mixin for UUID primary key"""
|
||||
|
||||
id = Column(
|
||||
UUID(as_uuid=True),
|
||||
primary_key=True,
|
||||
default=uuid.uuid4,
|
||||
comment="Unique identifier"
|
||||
)
|
||||
|
||||
|
||||
class SoftDeleteMixin:
|
||||
"""Mixin for soft delete functionality"""
|
||||
|
||||
deleted_at = Column(
|
||||
DateTime,
|
||||
nullable=True,
|
||||
comment="Soft delete timestamp"
|
||||
)
|
||||
|
||||
@property
|
||||
def is_deleted(self) -> bool:
|
||||
"""Check if record is soft deleted"""
|
||||
return self.deleted_at is not None
|
||||
|
||||
def soft_delete(self):
|
||||
"""Mark record as soft deleted"""
|
||||
self.deleted_at = datetime.utcnow()
|
||||
|
||||
def restore(self):
|
||||
"""Restore soft deleted record"""
|
||||
self.deleted_at = None
|
||||
|
||||
|
||||
class MetadataMixin:
|
||||
"""Mixin for flexible metadata storage"""
|
||||
|
||||
metadata = Column(
|
||||
JSON,
|
||||
nullable=False,
|
||||
default=dict,
|
||||
comment="Flexible metadata storage"
|
||||
)
|
||||
|
||||
def set_meta(self, key: str, value: Any) -> None:
|
||||
"""Set metadata value"""
|
||||
if self.metadata is None:
|
||||
self.metadata = {}
|
||||
self.metadata[key] = value
|
||||
|
||||
def get_meta(self, key: str, default: Any = None) -> Any:
|
||||
"""Get metadata value"""
|
||||
if self.metadata is None:
|
||||
return default
|
||||
return self.metadata.get(key, default)
|
||||
|
||||
def update_meta(self, updates: Dict[str, Any]) -> None:
|
||||
"""Update multiple metadata values"""
|
||||
if self.metadata is None:
|
||||
self.metadata = {}
|
||||
self.metadata.update(updates)
|
||||
|
||||
|
||||
class StatusMixin:
|
||||
"""Mixin for status tracking"""
|
||||
|
||||
status = Column(
|
||||
String(64),
|
||||
nullable=False,
|
||||
default="active",
|
||||
index=True,
|
||||
comment="Record status"
|
||||
)
|
||||
|
||||
def set_status(self, status: str, reason: Optional[str] = None):
|
||||
"""Set status with optional reason"""
|
||||
self.status = status
|
||||
if reason:
|
||||
self.set_meta("status_reason", reason)
|
||||
self.set_meta("status_changed_at", datetime.utcnow().isoformat())
|
||||
|
||||
|
||||
class BaseModelMixin:
|
||||
"""Base mixin with common functionality"""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert model to dictionary"""
|
||||
result = {}
|
||||
for column in self.__table__.columns:
|
||||
value = getattr(self, column.name)
|
||||
if isinstance(value, datetime):
|
||||
value = value.isoformat()
|
||||
elif hasattr(value, '__dict__'):
|
||||
value = str(value)
|
||||
result[column.name] = value
|
||||
return result
|
||||
|
||||
def update_from_dict(self, data: Dict[str, Any]) -> None:
|
||||
"""Update model from dictionary"""
|
||||
for key, value in data.items():
|
||||
if hasattr(self, key):
|
||||
setattr(self, key, value)
|
||||
|
||||
@classmethod
|
||||
async def get_by_id(
|
||||
cls: Type[ModelType],
|
||||
session: AsyncSession,
|
||||
id_value: Union[int, str, uuid.UUID]
|
||||
) -> Optional[ModelType]:
|
||||
"""Get record by ID"""
|
||||
try:
|
||||
stmt = select(cls).where(cls.id == id_value)
|
||||
result = await session.execute(stmt)
|
||||
return result.scalar_one_or_none()
|
||||
except Exception as e:
|
||||
logger.error("Error getting record by ID", model=cls.__name__, id=id_value, error=str(e))
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def get_all(
|
||||
cls: Type[ModelType],
|
||||
session: AsyncSession,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None
|
||||
) -> list[ModelType]:
|
||||
"""Get all records with optional pagination"""
|
||||
try:
|
||||
stmt = select(cls)
|
||||
if offset:
|
||||
stmt = stmt.offset(offset)
|
||||
if limit:
|
||||
stmt = stmt.limit(limit)
|
||||
result = await session.execute(stmt)
|
||||
return result.scalars().all()
|
||||
except Exception as e:
|
||||
logger.error("Error getting all records", model=cls.__name__, error=str(e))
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def count(cls: Type[ModelType], session: AsyncSession) -> int:
|
||||
"""Get total count of records"""
|
||||
try:
|
||||
from sqlalchemy import func
|
||||
stmt = select(func.count(cls.id))
|
||||
result = await session.execute(stmt)
|
||||
return result.scalar() or 0
|
||||
except Exception as e:
|
||||
logger.error("Error counting records", model=cls.__name__, error=str(e))
|
||||
return 0
|
||||
|
||||
async def save(self, session: AsyncSession) -> None:
|
||||
"""Save model to database"""
|
||||
try:
|
||||
session.add(self)
|
||||
await session.commit()
|
||||
await session.refresh(self)
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error("Error saving model", model=self.__class__.__name__, error=str(e))
|
||||
raise
|
||||
|
||||
async def delete(self, session: AsyncSession) -> None:
|
||||
"""Delete model from database"""
|
||||
try:
|
||||
await session.delete(self)
|
||||
await session.commit()
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error("Error deleting model", model=self.__class__.__name__, error=str(e))
|
||||
raise
|
||||
|
||||
|
||||
class AuditMixin:
|
||||
"""Mixin for audit trail"""
|
||||
|
||||
created_by = Column(
|
||||
UUID(as_uuid=True),
|
||||
nullable=True,
|
||||
comment="User who created the record"
|
||||
)
|
||||
updated_by = Column(
|
||||
UUID(as_uuid=True),
|
||||
nullable=True,
|
||||
comment="User who last updated the record"
|
||||
)
|
||||
|
||||
def set_audit_info(self, user_id: Optional[uuid.UUID] = None):
|
||||
"""Set audit information"""
|
||||
if user_id:
|
||||
if not hasattr(self, 'created_at') or not self.created_at:
|
||||
self.created_by = user_id
|
||||
self.updated_by = user_id
|
||||
|
||||
|
||||
class CacheableMixin:
|
||||
"""Mixin for cacheable models"""
|
||||
|
||||
@property
|
||||
def cache_key(self) -> str:
|
||||
"""Generate cache key for this model"""
|
||||
return f"{self.__class__.__name__.lower()}:{self.id}"
|
||||
|
||||
@property
|
||||
def cache_ttl(self) -> int:
|
||||
"""Default cache TTL in seconds"""
|
||||
return 3600 # 1 hour
|
||||
|
||||
def get_cache_data(self) -> Dict[str, Any]:
|
||||
"""Get data for caching"""
|
||||
return self.to_dict()
|
||||
|
||||
|
||||
# Combined base model class
|
||||
class BaseModel(
|
||||
Base,
|
||||
BaseModelMixin,
|
||||
TimestampMixin,
|
||||
UUIDMixin,
|
||||
SoftDeleteMixin,
|
||||
MetadataMixin,
|
||||
StatusMixin,
|
||||
AuditMixin,
|
||||
CacheableMixin
|
||||
):
|
||||
"""Base model with all mixins"""
|
||||
__abstract__ = True
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""String representation of model"""
|
||||
return f"<{self.__class__.__name__}(id={self.id})>"
|
||||
|
||||
|
||||
# Compatibility with old model base
|
||||
AlchemyBase = Base
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Compatible SQLAlchemy base models for MariaDB."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any
|
||||
from sqlalchemy import Column, Integer, DateTime, text
|
||||
from sqlalchemy.ext.declarative import declarative_base, declared_attr
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
# Create base class
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class TimestampMixin:
|
||||
"""Mixin for adding timestamp fields."""
|
||||
|
||||
@declared_attr
|
||||
def created_at(cls):
|
||||
return Column(
|
||||
DateTime,
|
||||
nullable=False,
|
||||
default=datetime.utcnow,
|
||||
server_default=text('CURRENT_TIMESTAMP')
|
||||
)
|
||||
|
||||
@declared_attr
|
||||
def updated_at(cls):
|
||||
return Column(
|
||||
DateTime,
|
||||
nullable=False,
|
||||
default=datetime.utcnow,
|
||||
onupdate=datetime.utcnow,
|
||||
server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP')
|
||||
)
|
||||
|
||||
|
||||
class BaseModel(Base, TimestampMixin):
|
||||
"""Base model with common fields for all entities."""
|
||||
|
||||
__abstract__ = True
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
|
||||
def to_dict(self, exclude: Optional[set] = None) -> Dict[str, Any]:
|
||||
"""Convert model instance to dictionary."""
|
||||
exclude = exclude or set()
|
||||
result = {}
|
||||
|
||||
for column in self.__table__.columns:
|
||||
if column.name not in exclude:
|
||||
value = getattr(self, column.name)
|
||||
# Handle datetime serialization
|
||||
if isinstance(value, datetime):
|
||||
result[column.name] = value.isoformat()
|
||||
else:
|
||||
result[column.name] = value
|
||||
|
||||
return result
|
||||
|
||||
def update_from_dict(self, data: Dict[str, Any], exclude: Optional[set] = None) -> None:
|
||||
"""Update model instance from dictionary."""
|
||||
exclude = exclude or {"id", "created_at", "updated_at"}
|
||||
|
||||
for key, value in data.items():
|
||||
if key not in exclude and hasattr(self, key):
|
||||
setattr(self, key, value)
|
||||
|
||||
@classmethod
|
||||
def get_table_name(cls) -> str:
|
||||
"""Get table name."""
|
||||
return cls.__tablename__
|
||||
|
||||
@classmethod
|
||||
def get_columns(cls) -> list:
|
||||
"""Get list of column names."""
|
||||
return [column.name for column in cls.__table__.columns]
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""String representation of model."""
|
||||
return f"<{self.__class__.__name__}(id={getattr(self, 'id', None)})>"
|
||||
|
||||
|
||||
# Legacy session factory for backward compatibility
|
||||
SessionLocal = sessionmaker()
|
||||
|
||||
|
||||
def get_session():
|
||||
"""Get database session (legacy function for compatibility)."""
|
||||
return SessionLocal()
|
||||
@@ -0,0 +1,445 @@
|
||||
"""
|
||||
Blockchain-related models for TON network integration.
|
||||
Handles transaction records, wallet management, and smart contract interactions.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import Dict, List, Optional, Any
|
||||
from uuid import UUID
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Column, String, Integer, DateTime, Boolean, Text, JSON, ForeignKey, Index
|
||||
from sqlalchemy.orm import relationship, validates
|
||||
from sqlalchemy.dialects.postgresql import UUID as PostgreSQLUUID
|
||||
|
||||
from app.core.models.base import Base, TimestampMixin, UUIDMixin
|
||||
|
||||
class BlockchainTransaction(Base, UUIDMixin, TimestampMixin):
|
||||
"""Model for storing blockchain transaction records."""
|
||||
__tablename__ = "blockchain_transactions"
|
||||
|
||||
# User relationship
|
||||
user_id = Column(PostgreSQLUUID(as_uuid=True), ForeignKey("users.id"), nullable=False)
|
||||
user = relationship("User", back_populates="blockchain_transactions")
|
||||
|
||||
# Transaction details
|
||||
transaction_hash = Column(String(64), unique=True, nullable=False, index=True)
|
||||
transaction_type = Column(String(20), nullable=False) # transfer, mint, burn, stake, etc.
|
||||
status = Column(String(20), nullable=False, default="pending") # pending, confirmed, failed
|
||||
|
||||
# Amount and fees
|
||||
amount = Column(sa.BIGINT, nullable=False, default=0) # Amount in nanotons
|
||||
network_fee = Column(sa.BIGINT, nullable=False, default=0) # Network fee in nanotons
|
||||
|
||||
# Addresses
|
||||
sender_address = Column(String(48), nullable=True, index=True)
|
||||
recipient_address = Column(String(48), nullable=True, index=True)
|
||||
|
||||
# Message and metadata
|
||||
message = Column(Text, nullable=True)
|
||||
metadata = Column(JSON, nullable=True)
|
||||
|
||||
# Blockchain specific fields
|
||||
block_hash = Column(String(64), nullable=True)
|
||||
logical_time = Column(sa.BIGINT, nullable=True) # TON logical time
|
||||
confirmations = Column(Integer, nullable=False, default=0)
|
||||
|
||||
# Timing
|
||||
confirmed_at = Column(DateTime, nullable=True)
|
||||
failed_at = Column(DateTime, nullable=True)
|
||||
|
||||
# Smart contract interaction
|
||||
contract_address = Column(String(48), nullable=True)
|
||||
contract_method = Column(String(100), nullable=True)
|
||||
contract_data = Column(JSON, nullable=True)
|
||||
|
||||
# Internal tracking
|
||||
retry_count = Column(Integer, nullable=False, default=0)
|
||||
last_retry_at = Column(DateTime, nullable=True)
|
||||
error_message = Column(Text, nullable=True)
|
||||
|
||||
# Indexes for performance
|
||||
__table_args__ = (
|
||||
Index("idx_blockchain_tx_user_status", "user_id", "status"),
|
||||
Index("idx_blockchain_tx_hash", "transaction_hash"),
|
||||
Index("idx_blockchain_tx_addresses", "sender_address", "recipient_address"),
|
||||
Index("idx_blockchain_tx_created", "created_at"),
|
||||
Index("idx_blockchain_tx_type_status", "transaction_type", "status"),
|
||||
)
|
||||
|
||||
@validates('transaction_type')
|
||||
def validate_transaction_type(self, key, transaction_type):
|
||||
"""Validate transaction type."""
|
||||
allowed_types = {
|
||||
'transfer', 'mint', 'burn', 'stake', 'unstake',
|
||||
'contract_call', 'deploy', 'withdraw', 'deposit'
|
||||
}
|
||||
if transaction_type not in allowed_types:
|
||||
raise ValueError(f"Invalid transaction type: {transaction_type}")
|
||||
return transaction_type
|
||||
|
||||
@validates('status')
|
||||
def validate_status(self, key, status):
|
||||
"""Validate transaction status."""
|
||||
allowed_statuses = {'pending', 'confirmed', 'failed', 'cancelled'}
|
||||
if status not in allowed_statuses:
|
||||
raise ValueError(f"Invalid status: {status}")
|
||||
return status
|
||||
|
||||
@property
|
||||
def amount_tons(self) -> Decimal:
|
||||
"""Convert nanotons to TON."""
|
||||
return Decimal(self.amount) / Decimal("1000000000")
|
||||
|
||||
@property
|
||||
def fee_tons(self) -> Decimal:
|
||||
"""Convert fee nanotons to TON."""
|
||||
return Decimal(self.network_fee) / Decimal("1000000000")
|
||||
|
||||
@property
|
||||
def is_incoming(self) -> bool:
|
||||
"""Check if transaction is incoming to user's wallet."""
|
||||
return self.transaction_type in {'transfer', 'mint', 'deposit'} and self.recipient_address
|
||||
|
||||
@property
|
||||
def is_outgoing(self) -> bool:
|
||||
"""Check if transaction is outgoing from user's wallet."""
|
||||
return self.transaction_type in {'transfer', 'burn', 'withdraw'} and self.sender_address
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert transaction to dictionary."""
|
||||
return {
|
||||
"id": str(self.id),
|
||||
"hash": self.transaction_hash,
|
||||
"type": self.transaction_type,
|
||||
"status": self.status,
|
||||
"amount": self.amount,
|
||||
"amount_tons": str(self.amount_tons),
|
||||
"fee": self.network_fee,
|
||||
"fee_tons": str(self.fee_tons),
|
||||
"sender": self.sender_address,
|
||||
"recipient": self.recipient_address,
|
||||
"message": self.message,
|
||||
"block_hash": self.block_hash,
|
||||
"confirmations": self.confirmations,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"confirmed_at": self.confirmed_at.isoformat() if self.confirmed_at else None,
|
||||
"is_incoming": self.is_incoming,
|
||||
"is_outgoing": self.is_outgoing
|
||||
}
|
||||
|
||||
class SmartContract(Base, UUIDMixin, TimestampMixin):
|
||||
"""Model for smart contract management."""
|
||||
__tablename__ = "smart_contracts"
|
||||
|
||||
# Contract details
|
||||
address = Column(String(48), unique=True, nullable=False, index=True)
|
||||
name = Column(String(100), nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
contract_type = Column(String(50), nullable=False) # nft, token, defi, etc.
|
||||
|
||||
# Contract metadata
|
||||
abi = Column(JSON, nullable=True) # Contract ABI if available
|
||||
source_code = Column(Text, nullable=True)
|
||||
compiler_version = Column(String(20), nullable=True)
|
||||
|
||||
# Deployment info
|
||||
deployer_address = Column(String(48), nullable=True)
|
||||
deployment_tx_hash = Column(String(64), nullable=True)
|
||||
deployment_block = Column(sa.BIGINT, nullable=True)
|
||||
|
||||
# Status and verification
|
||||
is_verified = Column(Boolean, nullable=False, default=False)
|
||||
is_active = Column(Boolean, nullable=False, default=True)
|
||||
verification_date = Column(DateTime, nullable=True)
|
||||
|
||||
# Usage statistics
|
||||
interaction_count = Column(Integer, nullable=False, default=0)
|
||||
last_interaction_at = Column(DateTime, nullable=True)
|
||||
|
||||
# Relationships
|
||||
transactions = relationship(
|
||||
"BlockchainTransaction",
|
||||
foreign_keys="BlockchainTransaction.contract_address",
|
||||
primaryjoin="SmartContract.address == BlockchainTransaction.contract_address",
|
||||
back_populates=None
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_smart_contract_address", "address"),
|
||||
Index("idx_smart_contract_type", "contract_type"),
|
||||
Index("idx_smart_contract_active", "is_active"),
|
||||
)
|
||||
|
||||
@validates('contract_type')
|
||||
def validate_contract_type(self, key, contract_type):
|
||||
"""Validate contract type."""
|
||||
allowed_types = {
|
||||
'nft', 'token', 'defi', 'game', 'dao', 'bridge',
|
||||
'oracle', 'multisig', 'custom'
|
||||
}
|
||||
if contract_type not in allowed_types:
|
||||
raise ValueError(f"Invalid contract type: {contract_type}")
|
||||
return contract_type
|
||||
|
||||
class TokenBalance(Base, UUIDMixin, TimestampMixin):
|
||||
"""Model for tracking user token balances."""
|
||||
__tablename__ = "token_balances"
|
||||
|
||||
# User relationship
|
||||
user_id = Column(PostgreSQLUUID(as_uuid=True), ForeignKey("users.id"), nullable=False)
|
||||
user = relationship("User", back_populates="token_balances")
|
||||
|
||||
# Token details
|
||||
token_address = Column(String(48), nullable=False, index=True)
|
||||
token_name = Column(String(100), nullable=True)
|
||||
token_symbol = Column(String(10), nullable=True)
|
||||
token_decimals = Column(Integer, nullable=False, default=9)
|
||||
|
||||
# Balance information
|
||||
balance = Column(sa.BIGINT, nullable=False, default=0) # Raw balance
|
||||
locked_balance = Column(sa.BIGINT, nullable=False, default=0) # Locked in contracts
|
||||
|
||||
# Metadata
|
||||
last_update_block = Column(sa.BIGINT, nullable=True)
|
||||
last_update_tx = Column(String(64), nullable=True)
|
||||
|
||||
# Unique constraint
|
||||
__table_args__ = (
|
||||
sa.UniqueConstraint("user_id", "token_address", name="uq_user_token"),
|
||||
Index("idx_token_balance_user", "user_id"),
|
||||
Index("idx_token_balance_token", "token_address"),
|
||||
Index("idx_token_balance_updated", "updated_at"),
|
||||
)
|
||||
|
||||
@property
|
||||
def available_balance(self) -> int:
|
||||
"""Get available (unlocked) balance."""
|
||||
return max(0, self.balance - self.locked_balance)
|
||||
|
||||
@property
|
||||
def formatted_balance(self) -> Decimal:
|
||||
"""Get balance formatted with decimals."""
|
||||
return Decimal(self.balance) / Decimal(10 ** self.token_decimals)
|
||||
|
||||
@property
|
||||
def formatted_available_balance(self) -> Decimal:
|
||||
"""Get available balance formatted with decimals."""
|
||||
return Decimal(self.available_balance) / Decimal(10 ** self.token_decimals)
|
||||
|
||||
class StakingPosition(Base, UUIDMixin, TimestampMixin):
|
||||
"""Model for staking positions."""
|
||||
__tablename__ = "staking_positions"
|
||||
|
||||
# User relationship
|
||||
user_id = Column(PostgreSQLUUID(as_uuid=True), ForeignKey("users.id"), nullable=False)
|
||||
user = relationship("User", back_populates="staking_positions")
|
||||
|
||||
# Staking details
|
||||
validator_address = Column(String(48), nullable=False, index=True)
|
||||
pool_address = Column(String(48), nullable=True)
|
||||
|
||||
# Amount and timing
|
||||
staked_amount = Column(sa.BIGINT, nullable=False) # Amount in nanotons
|
||||
stake_tx_hash = Column(String(64), nullable=False)
|
||||
stake_block = Column(sa.BIGINT, nullable=True)
|
||||
|
||||
# Status
|
||||
status = Column(String(20), nullable=False, default="active") # active, unstaking, withdrawn
|
||||
unstake_tx_hash = Column(String(64), nullable=True)
|
||||
unstake_requested_at = Column(DateTime, nullable=True)
|
||||
withdrawn_at = Column(DateTime, nullable=True)
|
||||
|
||||
# Rewards
|
||||
rewards_earned = Column(sa.BIGINT, nullable=False, default=0)
|
||||
last_reward_claim = Column(DateTime, nullable=True)
|
||||
last_reward_tx = Column(String(64), nullable=True)
|
||||
|
||||
# Lock period
|
||||
lock_period_days = Column(Integer, nullable=False, default=0)
|
||||
unlock_date = Column(DateTime, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_staking_user_status", "user_id", "status"),
|
||||
Index("idx_staking_validator", "validator_address"),
|
||||
Index("idx_staking_unlock", "unlock_date"),
|
||||
)
|
||||
|
||||
@validates('status')
|
||||
def validate_status(self, key, status):
|
||||
"""Validate staking status."""
|
||||
allowed_statuses = {'active', 'unstaking', 'withdrawn', 'slashed'}
|
||||
if status not in allowed_statuses:
|
||||
raise ValueError(f"Invalid staking status: {status}")
|
||||
return status
|
||||
|
||||
@property
|
||||
def staked_tons(self) -> Decimal:
|
||||
"""Get staked amount in TON."""
|
||||
return Decimal(self.staked_amount) / Decimal("1000000000")
|
||||
|
||||
@property
|
||||
def rewards_tons(self) -> Decimal:
|
||||
"""Get rewards amount in TON."""
|
||||
return Decimal(self.rewards_earned) / Decimal("1000000000")
|
||||
|
||||
@property
|
||||
def is_locked(self) -> bool:
|
||||
"""Check if staking position is still locked."""
|
||||
if not self.unlock_date:
|
||||
return False
|
||||
return datetime.utcnow() < self.unlock_date
|
||||
|
||||
class NFTCollection(Base, UUIDMixin, TimestampMixin):
|
||||
"""Model for NFT collections."""
|
||||
__tablename__ = "nft_collections"
|
||||
|
||||
# Collection details
|
||||
contract_address = Column(String(48), unique=True, nullable=False, index=True)
|
||||
name = Column(String(100), nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
symbol = Column(String(10), nullable=True)
|
||||
|
||||
# Creator and metadata
|
||||
creator_address = Column(String(48), nullable=False)
|
||||
metadata_uri = Column(String(500), nullable=True)
|
||||
base_uri = Column(String(500), nullable=True)
|
||||
|
||||
# Collection stats
|
||||
total_supply = Column(Integer, nullable=False, default=0)
|
||||
max_supply = Column(Integer, nullable=True)
|
||||
floor_price = Column(sa.BIGINT, nullable=True) # In nanotons
|
||||
|
||||
# Status
|
||||
is_verified = Column(Boolean, nullable=False, default=False)
|
||||
is_active = Column(Boolean, nullable=False, default=True)
|
||||
|
||||
# Relationships
|
||||
nfts = relationship("NFTToken", back_populates="collection")
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_nft_collection_address", "contract_address"),
|
||||
Index("idx_nft_collection_creator", "creator_address"),
|
||||
Index("idx_nft_collection_verified", "is_verified"),
|
||||
)
|
||||
|
||||
class NFTToken(Base, UUIDMixin, TimestampMixin):
|
||||
"""Model for individual NFT tokens."""
|
||||
__tablename__ = "nft_tokens"
|
||||
|
||||
# Token identification
|
||||
collection_id = Column(PostgreSQLUUID(as_uuid=True), ForeignKey("nft_collections.id"), nullable=False)
|
||||
collection = relationship("NFTCollection", back_populates="nfts")
|
||||
|
||||
token_id = Column(String(100), nullable=False) # Token ID within collection
|
||||
token_address = Column(String(48), unique=True, nullable=False, index=True)
|
||||
|
||||
# Ownership
|
||||
owner_address = Column(String(48), nullable=False, index=True)
|
||||
|
||||
# Metadata
|
||||
name = Column(String(200), nullable=True)
|
||||
description = Column(Text, nullable=True)
|
||||
image_uri = Column(String(500), nullable=True)
|
||||
metadata_uri = Column(String(500), nullable=True)
|
||||
attributes = Column(JSON, nullable=True)
|
||||
|
||||
# Trading
|
||||
last_sale_price = Column(sa.BIGINT, nullable=True) # In nanotons
|
||||
last_sale_tx = Column(String(64), nullable=True)
|
||||
last_sale_date = Column(DateTime, nullable=True)
|
||||
|
||||
# Status
|
||||
is_burned = Column(Boolean, nullable=False, default=False)
|
||||
burned_at = Column(DateTime, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
sa.UniqueConstraint("collection_id", "token_id", name="uq_collection_token"),
|
||||
Index("idx_nft_token_address", "token_address"),
|
||||
Index("idx_nft_token_owner", "owner_address"),
|
||||
Index("idx_nft_token_collection", "collection_id"),
|
||||
)
|
||||
|
||||
@property
|
||||
def last_sale_tons(self) -> Optional[Decimal]:
|
||||
"""Get last sale price in TON."""
|
||||
if self.last_sale_price is None:
|
||||
return None
|
||||
return Decimal(self.last_sale_price) / Decimal("1000000000")
|
||||
|
||||
class DeFiPosition(Base, UUIDMixin, TimestampMixin):
|
||||
"""Model for DeFi protocol positions."""
|
||||
__tablename__ = "defi_positions"
|
||||
|
||||
# User relationship
|
||||
user_id = Column(PostgreSQLUUID(as_uuid=True), ForeignKey("users.id"), nullable=False)
|
||||
user = relationship("User", back_populates="defi_positions")
|
||||
|
||||
# Protocol details
|
||||
protocol_name = Column(String(100), nullable=False)
|
||||
protocol_address = Column(String(48), nullable=False)
|
||||
position_type = Column(String(50), nullable=False) # liquidity, lending, borrowing, etc.
|
||||
|
||||
# Position details
|
||||
token_a_address = Column(String(48), nullable=True)
|
||||
token_a_amount = Column(sa.BIGINT, nullable=False, default=0)
|
||||
token_b_address = Column(String(48), nullable=True)
|
||||
token_b_amount = Column(sa.BIGINT, nullable=False, default=0)
|
||||
|
||||
# Value tracking
|
||||
initial_value = Column(sa.BIGINT, nullable=False, default=0) # In nanotons
|
||||
current_value = Column(sa.BIGINT, nullable=False, default=0)
|
||||
last_value_update = Column(DateTime, nullable=True)
|
||||
|
||||
# Rewards and fees
|
||||
rewards_earned = Column(sa.BIGINT, nullable=False, default=0)
|
||||
fees_paid = Column(sa.BIGINT, nullable=False, default=0)
|
||||
|
||||
# Status
|
||||
status = Column(String(20), nullable=False, default="active") # active, closed, liquidated
|
||||
opened_tx = Column(String(64), nullable=False)
|
||||
closed_tx = Column(String(64), nullable=True)
|
||||
closed_at = Column(DateTime, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_defi_user_protocol", "user_id", "protocol_name"),
|
||||
Index("idx_defi_position_type", "position_type"),
|
||||
Index("idx_defi_status", "status"),
|
||||
)
|
||||
|
||||
@validates('position_type')
|
||||
def validate_position_type(self, key, position_type):
|
||||
"""Validate position type."""
|
||||
allowed_types = {
|
||||
'liquidity', 'lending', 'borrowing', 'farming',
|
||||
'staking', 'options', 'futures', 'insurance'
|
||||
}
|
||||
if position_type not in allowed_types:
|
||||
raise ValueError(f"Invalid position type: {position_type}")
|
||||
return position_type
|
||||
|
||||
@validates('status')
|
||||
def validate_status(self, key, status):
|
||||
"""Validate position status."""
|
||||
allowed_statuses = {'active', 'closed', 'liquidated', 'expired'}
|
||||
if status not in allowed_statuses:
|
||||
raise ValueError(f"Invalid position status: {status}")
|
||||
return status
|
||||
|
||||
@property
|
||||
def current_value_tons(self) -> Decimal:
|
||||
"""Get current value in TON."""
|
||||
return Decimal(self.current_value) / Decimal("1000000000")
|
||||
|
||||
@property
|
||||
def pnl_tons(self) -> Decimal:
|
||||
"""Get profit/loss in TON."""
|
||||
return Decimal(self.current_value - self.initial_value) / Decimal("1000000000")
|
||||
|
||||
@property
|
||||
def pnl_percentage(self) -> Decimal:
|
||||
"""Get profit/loss percentage."""
|
||||
if self.initial_value == 0:
|
||||
return Decimal("0")
|
||||
return (Decimal(self.current_value - self.initial_value) / Decimal(self.initial_value)) * 100
|
||||
@@ -0,0 +1,731 @@
|
||||
"""
|
||||
Content models with async support and enhanced features
|
||||
"""
|
||||
import hashlib
|
||||
import mimetypes
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any, Union
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from sqlalchemy import Column, String, Integer, BigInteger, Boolean, Text, ForeignKey, Index, text
|
||||
from sqlalchemy.dialects.postgresql import JSONB, ARRAY
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy.orm import relationship
|
||||
import structlog
|
||||
|
||||
from app.core.models.base import BaseModel
|
||||
from app.core.config import settings, PROJECT_HOST
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
class ContentType(str, Enum):
|
||||
"""Content type enumeration"""
|
||||
AUDIO = "audio"
|
||||
VIDEO = "video"
|
||||
IMAGE = "image"
|
||||
TEXT = "text"
|
||||
DOCUMENT = "document"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class ContentStatus(str, Enum):
|
||||
"""Content status enumeration"""
|
||||
UPLOADING = "uploading"
|
||||
PROCESSING = "processing"
|
||||
READY = "ready"
|
||||
FAILED = "failed"
|
||||
DISABLED = "disabled"
|
||||
DELETED = "deleted"
|
||||
|
||||
|
||||
class StorageType(str, Enum):
|
||||
"""Storage type enumeration"""
|
||||
LOCAL = "local"
|
||||
ONCHAIN = "onchain"
|
||||
IPFS = "ipfs"
|
||||
HYBRID = "hybrid"
|
||||
|
||||
|
||||
class LicenseType(str, Enum):
|
||||
"""License type enumeration"""
|
||||
LISTEN = "listen"
|
||||
USE = "use"
|
||||
RESALE = "resale"
|
||||
EXCLUSIVE = "exclusive"
|
||||
|
||||
|
||||
class StoredContent(BaseModel):
|
||||
"""Enhanced content storage model"""
|
||||
|
||||
__tablename__ = 'stored_content'
|
||||
|
||||
# Content identification
|
||||
hash = Column(
|
||||
String(128),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
index=True,
|
||||
comment="Content hash (SHA-256 or custom)"
|
||||
)
|
||||
content_id = Column(
|
||||
String(256),
|
||||
nullable=True,
|
||||
index=True,
|
||||
comment="Content identifier (CID for IPFS)"
|
||||
)
|
||||
|
||||
# File information
|
||||
filename = Column(
|
||||
String(512),
|
||||
nullable=False,
|
||||
comment="Original filename"
|
||||
)
|
||||
file_size = Column(
|
||||
BigInteger,
|
||||
nullable=False,
|
||||
default=0,
|
||||
comment="File size in bytes"
|
||||
)
|
||||
mime_type = Column(
|
||||
String(128),
|
||||
nullable=True,
|
||||
comment="MIME type of the content"
|
||||
)
|
||||
|
||||
# Content type and storage
|
||||
content_type = Column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default=ContentType.UNKNOWN.value,
|
||||
index=True,
|
||||
comment="Content type category"
|
||||
)
|
||||
storage_type = Column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default=StorageType.LOCAL.value,
|
||||
index=True,
|
||||
comment="Storage type"
|
||||
)
|
||||
|
||||
# File path and URLs
|
||||
file_path = Column(
|
||||
String(1024),
|
||||
nullable=True,
|
||||
comment="Local file path"
|
||||
)
|
||||
external_url = Column(
|
||||
String(2048),
|
||||
nullable=True,
|
||||
comment="External URL for remote content"
|
||||
)
|
||||
|
||||
# Blockchain related
|
||||
onchain_index = Column(
|
||||
Integer,
|
||||
nullable=True,
|
||||
index=True,
|
||||
comment="On-chain index number"
|
||||
)
|
||||
owner_address = Column(
|
||||
String(256),
|
||||
nullable=True,
|
||||
index=True,
|
||||
comment="Blockchain owner address"
|
||||
)
|
||||
|
||||
# User and access
|
||||
user_id = Column(
|
||||
String(36), # UUID
|
||||
ForeignKey('users.id'),
|
||||
nullable=True,
|
||||
index=True,
|
||||
comment="User who uploaded the content"
|
||||
)
|
||||
|
||||
# Encryption and security
|
||||
encrypted = Column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
comment="Whether content is encrypted"
|
||||
)
|
||||
encryption_key_id = Column(
|
||||
String(36), # UUID
|
||||
ForeignKey('encryption_keys.id'),
|
||||
nullable=True,
|
||||
comment="Encryption key reference"
|
||||
)
|
||||
|
||||
# Processing status
|
||||
disabled = Column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
index=True,
|
||||
comment="Whether content is disabled"
|
||||
)
|
||||
|
||||
# Content metadata
|
||||
title = Column(
|
||||
String(512),
|
||||
nullable=True,
|
||||
comment="Content title"
|
||||
)
|
||||
description = Column(
|
||||
Text,
|
||||
nullable=True,
|
||||
comment="Content description"
|
||||
)
|
||||
tags = Column(
|
||||
ARRAY(String),
|
||||
nullable=False,
|
||||
default=list,
|
||||
comment="Content tags"
|
||||
)
|
||||
|
||||
# Media-specific metadata
|
||||
duration = Column(
|
||||
Integer,
|
||||
nullable=True,
|
||||
comment="Duration in seconds (for audio/video)"
|
||||
)
|
||||
width = Column(
|
||||
Integer,
|
||||
nullable=True,
|
||||
comment="Width in pixels (for images/video)"
|
||||
)
|
||||
height = Column(
|
||||
Integer,
|
||||
nullable=True,
|
||||
comment="Height in pixels (for images/video)"
|
||||
)
|
||||
bitrate = Column(
|
||||
Integer,
|
||||
nullable=True,
|
||||
comment="Bitrate (for audio/video)"
|
||||
)
|
||||
|
||||
# Conversion and processing
|
||||
processing_status = Column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default=ContentStatus.READY.value,
|
||||
index=True,
|
||||
comment="Processing status"
|
||||
)
|
||||
conversion_data = Column(
|
||||
JSONB,
|
||||
nullable=False,
|
||||
default=dict,
|
||||
comment="Conversion and processing data"
|
||||
)
|
||||
|
||||
# Statistics
|
||||
download_count = Column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
default=0,
|
||||
comment="Number of downloads"
|
||||
)
|
||||
view_count = Column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
default=0,
|
||||
comment="Number of views"
|
||||
)
|
||||
|
||||
# Relationships
|
||||
user = relationship('User', back_populates='content_items')
|
||||
encryption_key = relationship('EncryptionKey', back_populates='content_items')
|
||||
user_contents = relationship('UserContent', back_populates='content')
|
||||
user_actions = relationship('UserAction', back_populates='content')
|
||||
|
||||
# Indexes for performance
|
||||
__table_args__ = (
|
||||
Index('idx_content_hash', 'hash'),
|
||||
Index('idx_content_user_type', 'user_id', 'content_type'),
|
||||
Index('idx_content_storage_status', 'storage_type', 'status'),
|
||||
Index('idx_content_onchain', 'onchain_index'),
|
||||
Index('idx_content_created', 'created_at'),
|
||||
Index('idx_content_disabled', 'disabled'),
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""String representation"""
|
||||
return f"StoredContent({self.id}, hash={self.hash[:8]}..., filename={self.filename})"
|
||||
|
||||
@property
|
||||
def file_extension(self) -> str:
|
||||
"""Get file extension"""
|
||||
return Path(self.filename).suffix.lower()
|
||||
|
||||
@property
|
||||
def web_url(self) -> str:
|
||||
"""Get web accessible URL"""
|
||||
if self.external_url:
|
||||
return self.external_url
|
||||
if self.hash:
|
||||
return urljoin(str(PROJECT_HOST), f"/api/v1.5/storage/{self.hash}")
|
||||
return ""
|
||||
|
||||
@property
|
||||
def download_url(self) -> str:
|
||||
"""Get download URL"""
|
||||
if self.hash:
|
||||
return urljoin(str(PROJECT_HOST), f"/api/v1/storage/{self.hash}")
|
||||
return ""
|
||||
|
||||
@property
|
||||
def is_media(self) -> bool:
|
||||
"""Check if content is media (audio/video/image)"""
|
||||
return self.content_type in [ContentType.AUDIO, ContentType.VIDEO, ContentType.IMAGE]
|
||||
|
||||
@property
|
||||
def is_processed(self) -> bool:
|
||||
"""Check if content is fully processed"""
|
||||
return self.processing_status == ContentStatus.READY.value
|
||||
|
||||
@property
|
||||
def cache_key(self) -> str:
|
||||
"""Override cache key to use hash"""
|
||||
return f"content:hash:{self.hash}"
|
||||
|
||||
def detect_content_type(self) -> ContentType:
|
||||
"""Detect content type from MIME type"""
|
||||
if not self.mime_type:
|
||||
# Try to guess from extension
|
||||
mime_type, _ = mimetypes.guess_type(self.filename)
|
||||
self.mime_type = mime_type
|
||||
|
||||
if self.mime_type:
|
||||
if self.mime_type.startswith('audio/'):
|
||||
return ContentType.AUDIO
|
||||
elif self.mime_type.startswith('video/'):
|
||||
return ContentType.VIDEO
|
||||
elif self.mime_type.startswith('image/'):
|
||||
return ContentType.IMAGE
|
||||
elif self.mime_type.startswith('text/'):
|
||||
return ContentType.TEXT
|
||||
elif 'application/' in self.mime_type:
|
||||
return ContentType.DOCUMENT
|
||||
|
||||
return ContentType.UNKNOWN
|
||||
|
||||
def calculate_hash(self, file_data: bytes) -> str:
|
||||
"""Calculate hash for file data"""
|
||||
return hashlib.sha256(file_data).hexdigest()
|
||||
|
||||
def set_conversion_data(self, key: str, value: Any) -> None:
|
||||
"""Set conversion data"""
|
||||
if not self.conversion_data:
|
||||
self.conversion_data = {}
|
||||
self.conversion_data[key] = value
|
||||
|
||||
def get_conversion_data(self, key: str, default: Any = None) -> Any:
|
||||
"""Get conversion data"""
|
||||
if not self.conversion_data:
|
||||
return default
|
||||
return self.conversion_data.get(key, default)
|
||||
|
||||
def add_tag(self, tag: str) -> None:
|
||||
"""Add tag to content"""
|
||||
if not self.tags:
|
||||
self.tags = []
|
||||
tag = tag.strip().lower()
|
||||
if tag and tag not in self.tags:
|
||||
self.tags.append(tag)
|
||||
|
||||
def remove_tag(self, tag: str) -> None:
|
||||
"""Remove tag from content"""
|
||||
if self.tags:
|
||||
tag = tag.strip().lower()
|
||||
if tag in self.tags:
|
||||
self.tags.remove(tag)
|
||||
|
||||
def increment_download_count(self) -> None:
|
||||
"""Increment download counter"""
|
||||
self.download_count += 1
|
||||
|
||||
def increment_view_count(self) -> None:
|
||||
"""Increment view counter"""
|
||||
self.view_count += 1
|
||||
|
||||
@classmethod
|
||||
async def get_by_hash(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
content_hash: str
|
||||
) -> Optional['StoredContent']:
|
||||
"""Get content by hash"""
|
||||
try:
|
||||
stmt = select(cls).where(cls.hash == content_hash)
|
||||
result = await session.execute(stmt)
|
||||
return result.scalar_one_or_none()
|
||||
except Exception as e:
|
||||
logger.error("Error getting content by hash", hash=content_hash, error=str(e))
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def get_by_user(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
user_id: str,
|
||||
content_type: Optional[ContentType] = None,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None
|
||||
) -> List['StoredContent']:
|
||||
"""Get content by user"""
|
||||
try:
|
||||
stmt = select(cls).where(cls.user_id == user_id)
|
||||
|
||||
if content_type:
|
||||
stmt = stmt.where(cls.content_type == content_type.value)
|
||||
|
||||
stmt = stmt.order_by(cls.created_at.desc())
|
||||
|
||||
if offset:
|
||||
stmt = stmt.offset(offset)
|
||||
if limit:
|
||||
stmt = stmt.limit(limit)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
return result.scalars().all()
|
||||
except Exception as e:
|
||||
logger.error("Error getting content by user", user_id=user_id, error=str(e))
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def get_recent(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
days: int = 7,
|
||||
content_type: Optional[ContentType] = None,
|
||||
limit: Optional[int] = None
|
||||
) -> List['StoredContent']:
|
||||
"""Get recent content"""
|
||||
try:
|
||||
cutoff_date = datetime.utcnow() - timedelta(days=days)
|
||||
stmt = select(cls).where(
|
||||
cls.created_at >= cutoff_date,
|
||||
cls.disabled == False,
|
||||
cls.processing_status == ContentStatus.READY.value
|
||||
)
|
||||
|
||||
if content_type:
|
||||
stmt = stmt.where(cls.content_type == content_type.value)
|
||||
|
||||
stmt = stmt.order_by(cls.created_at.desc())
|
||||
|
||||
if limit:
|
||||
stmt = stmt.limit(limit)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
return result.scalars().all()
|
||||
except Exception as e:
|
||||
logger.error("Error getting recent content", days=days, error=str(e))
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def search(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
query: str,
|
||||
content_type: Optional[ContentType] = None,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None
|
||||
) -> List['StoredContent']:
|
||||
"""Search content by title and description"""
|
||||
try:
|
||||
search_pattern = f"%{query.lower()}%"
|
||||
stmt = select(cls).where(
|
||||
(cls.title.ilike(search_pattern)) |
|
||||
(cls.description.ilike(search_pattern)) |
|
||||
(cls.filename.ilike(search_pattern)),
|
||||
cls.disabled == False,
|
||||
cls.processing_status == ContentStatus.READY.value
|
||||
)
|
||||
|
||||
if content_type:
|
||||
stmt = stmt.where(cls.content_type == content_type.value)
|
||||
|
||||
stmt = stmt.order_by(cls.created_at.desc())
|
||||
|
||||
if offset:
|
||||
stmt = stmt.offset(offset)
|
||||
if limit:
|
||||
stmt = stmt.limit(limit)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
return result.scalars().all()
|
||||
except Exception as e:
|
||||
logger.error("Error searching content", query=query, error=str(e))
|
||||
return []
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary with additional computed fields"""
|
||||
data = super().to_dict()
|
||||
data.update({
|
||||
'web_url': self.web_url,
|
||||
'download_url': self.download_url,
|
||||
'file_extension': self.file_extension,
|
||||
'is_media': self.is_media,
|
||||
'is_processed': self.is_processed
|
||||
})
|
||||
return data
|
||||
|
||||
|
||||
class UserContent(BaseModel):
|
||||
"""User content ownership and licensing"""
|
||||
|
||||
__tablename__ = 'user_content'
|
||||
|
||||
# Content relationship
|
||||
content_id = Column(
|
||||
String(36), # UUID
|
||||
ForeignKey('stored_content.id'),
|
||||
nullable=False,
|
||||
index=True,
|
||||
comment="Reference to stored content"
|
||||
)
|
||||
user_id = Column(
|
||||
String(36), # UUID
|
||||
ForeignKey('users.id'),
|
||||
nullable=False,
|
||||
index=True,
|
||||
comment="User who owns this content"
|
||||
)
|
||||
|
||||
# License information
|
||||
license_type = Column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default=LicenseType.LISTEN.value,
|
||||
comment="Type of license"
|
||||
)
|
||||
|
||||
# Blockchain data
|
||||
onchain_address = Column(
|
||||
String(256),
|
||||
nullable=True,
|
||||
index=True,
|
||||
comment="On-chain contract address"
|
||||
)
|
||||
owner_address = Column(
|
||||
String(256),
|
||||
nullable=True,
|
||||
index=True,
|
||||
comment="Blockchain owner address"
|
||||
)
|
||||
|
||||
# Transaction data
|
||||
purchase_transaction = Column(
|
||||
String(128),
|
||||
nullable=True,
|
||||
comment="Purchase transaction hash"
|
||||
)
|
||||
purchase_amount = Column(
|
||||
BigInteger,
|
||||
nullable=True,
|
||||
comment="Purchase amount in minimal units"
|
||||
)
|
||||
|
||||
# Wallet connection
|
||||
wallet_connection_id = Column(
|
||||
String(36), # UUID
|
||||
ForeignKey('wallet_connections.id'),
|
||||
nullable=True,
|
||||
comment="Wallet connection used for purchase"
|
||||
)
|
||||
|
||||
# Access control
|
||||
access_granted = Column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
comment="Whether access is granted"
|
||||
)
|
||||
access_expires_at = Column(
|
||||
DateTime,
|
||||
nullable=True,
|
||||
comment="When access expires (for temporary licenses)"
|
||||
)
|
||||
|
||||
# Usage tracking
|
||||
download_count = Column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
default=0,
|
||||
comment="Number of downloads by this user"
|
||||
)
|
||||
last_accessed = Column(
|
||||
DateTime,
|
||||
nullable=True,
|
||||
comment="Last access timestamp"
|
||||
)
|
||||
|
||||
# Relationships
|
||||
user = relationship('User', back_populates='content_items')
|
||||
content = relationship('StoredContent', back_populates='user_contents')
|
||||
wallet_connection = relationship('WalletConnection', back_populates='user_contents')
|
||||
|
||||
# Indexes
|
||||
__table_args__ = (
|
||||
Index('idx_user_content_user', 'user_id'),
|
||||
Index('idx_user_content_content', 'content_id'),
|
||||
Index('idx_user_content_onchain', 'onchain_address'),
|
||||
Index('idx_user_content_owner', 'owner_address'),
|
||||
Index('idx_user_content_status', 'status'),
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""String representation"""
|
||||
return f"UserContent({self.id}, user={self.user_id}, content={self.content_id})"
|
||||
|
||||
@property
|
||||
def is_expired(self) -> bool:
|
||||
"""Check if access has expired"""
|
||||
if not self.access_expires_at:
|
||||
return False
|
||||
return datetime.utcnow() > self.access_expires_at
|
||||
|
||||
@property
|
||||
def is_accessible(self) -> bool:
|
||||
"""Check if content is accessible"""
|
||||
return self.access_granted and not self.is_expired and self.status == 'active'
|
||||
|
||||
def grant_access(self, expires_at: Optional[datetime] = None) -> None:
|
||||
"""Grant access to content"""
|
||||
self.access_granted = True
|
||||
self.access_expires_at = expires_at
|
||||
self.last_accessed = datetime.utcnow()
|
||||
|
||||
def revoke_access(self) -> None:
|
||||
"""Revoke access to content"""
|
||||
self.access_granted = False
|
||||
|
||||
def record_download(self) -> None:
|
||||
"""Record a download"""
|
||||
self.download_count += 1
|
||||
self.last_accessed = datetime.utcnow()
|
||||
|
||||
@classmethod
|
||||
async def get_user_access(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
user_id: str,
|
||||
content_id: str
|
||||
) -> Optional['UserContent']:
|
||||
"""Get user access to specific content"""
|
||||
try:
|
||||
stmt = select(cls).where(
|
||||
cls.user_id == user_id,
|
||||
cls.content_id == content_id,
|
||||
cls.status == 'active'
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
return result.scalar_one_or_none()
|
||||
except Exception as e:
|
||||
logger.error("Error getting user access", user_id=user_id, content_id=content_id, error=str(e))
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def get_user_content(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
user_id: str,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None
|
||||
) -> List['UserContent']:
|
||||
"""Get all content accessible by user"""
|
||||
try:
|
||||
stmt = select(cls).where(
|
||||
cls.user_id == user_id,
|
||||
cls.status == 'active',
|
||||
cls.access_granted == True
|
||||
).order_by(cls.created_at.desc())
|
||||
|
||||
if offset:
|
||||
stmt = stmt.offset(offset)
|
||||
if limit:
|
||||
stmt = stmt.limit(limit)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
return result.scalars().all()
|
||||
except Exception as e:
|
||||
logger.error("Error getting user content", user_id=user_id, error=str(e))
|
||||
return []
|
||||
|
||||
|
||||
class EncryptionKey(BaseModel):
|
||||
"""Encryption key management"""
|
||||
|
||||
__tablename__ = 'encryption_keys'
|
||||
|
||||
# Key identification
|
||||
key_hash = Column(
|
||||
String(128),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
index=True,
|
||||
comment="Hash of the encryption key"
|
||||
)
|
||||
algorithm = Column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="AES-256-GCM",
|
||||
comment="Encryption algorithm used"
|
||||
)
|
||||
|
||||
# Key metadata
|
||||
purpose = Column(
|
||||
String(64),
|
||||
nullable=False,
|
||||
comment="Purpose of the key (content, user_data, etc.)"
|
||||
)
|
||||
|
||||
# Access control
|
||||
owner_id = Column(
|
||||
String(36), # UUID
|
||||
ForeignKey('users.id'),
|
||||
nullable=True,
|
||||
comment="Key owner (if user-specific)"
|
||||
)
|
||||
|
||||
# Key lifecycle
|
||||
expires_at = Column(
|
||||
DateTime,
|
||||
nullable=True,
|
||||
comment="Key expiration timestamp"
|
||||
)
|
||||
revoked_at = Column(
|
||||
DateTime,
|
||||
nullable=True,
|
||||
comment="Key revocation timestamp"
|
||||
)
|
||||
|
||||
# Relationships
|
||||
owner = relationship('User', back_populates='encryption_keys')
|
||||
content_items = relationship('StoredContent', back_populates='encryption_key')
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""String representation"""
|
||||
return f"EncryptionKey({self.id}, hash={self.key_hash[:8]}...)"
|
||||
|
||||
@property
|
||||
def is_valid(self) -> bool:
|
||||
"""Check if key is valid (not expired or revoked)"""
|
||||
now = datetime.utcnow()
|
||||
if self.revoked_at and self.revoked_at <= now:
|
||||
return False
|
||||
if self.expires_at and self.expires_at <= now:
|
||||
return False
|
||||
return True
|
||||
|
||||
def revoke(self) -> None:
|
||||
"""Revoke the key"""
|
||||
self.revoked_at = datetime.utcnow()
|
||||
@@ -0,0 +1,388 @@
|
||||
"""Compatible content models for MariaDB."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional, List, Dict, Any
|
||||
from sqlalchemy import Column, String, Boolean, Text, Integer, DateTime, BigInteger, Index, ForeignKey
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from app.core.models.base_compatible import BaseModel
|
||||
|
||||
|
||||
class Content(BaseModel):
|
||||
"""Content model compatible with existing MariaDB schema."""
|
||||
|
||||
__tablename__ = "content"
|
||||
|
||||
# Basic content information
|
||||
user_id = Column(Integer, ForeignKey('users.id'), nullable=False, index=True)
|
||||
filename = Column(String(255), nullable=False)
|
||||
original_filename = Column(String(255), nullable=False)
|
||||
file_path = Column(String(500), nullable=False)
|
||||
|
||||
# File metadata
|
||||
file_size = Column(BigInteger, nullable=False) # bytes
|
||||
file_type = Column(String(100), nullable=False)
|
||||
mime_type = Column(String(100), nullable=False)
|
||||
file_extension = Column(String(10), nullable=False)
|
||||
|
||||
# Content metadata
|
||||
title = Column(String(255), nullable=True)
|
||||
description = Column(Text, nullable=True)
|
||||
tags = Column(Text, nullable=True) # JSON or comma-separated
|
||||
|
||||
# Status and visibility
|
||||
is_public = Column(Boolean, default=False, nullable=False)
|
||||
is_active = Column(Boolean, default=True, nullable=False)
|
||||
is_indexed = Column(Boolean, default=False, nullable=False)
|
||||
is_converted = Column(Boolean, default=False, nullable=False)
|
||||
|
||||
# Access and security
|
||||
access_password = Column(String(255), nullable=True)
|
||||
download_count = Column(Integer, default=0, nullable=False)
|
||||
view_count = Column(Integer, default=0, nullable=False)
|
||||
|
||||
# Processing status
|
||||
processing_status = Column(String(50), default="pending", nullable=False)
|
||||
processing_error = Column(Text, nullable=True)
|
||||
processing_started = Column(DateTime, nullable=True)
|
||||
processing_completed = Column(DateTime, nullable=True)
|
||||
|
||||
# File hashes for integrity
|
||||
md5_hash = Column(String(32), nullable=True, index=True)
|
||||
sha256_hash = Column(String(64), nullable=True, index=True)
|
||||
|
||||
# Thumbnails and previews
|
||||
thumbnail_path = Column(String(500), nullable=True)
|
||||
preview_path = Column(String(500), nullable=True)
|
||||
|
||||
# TON Blockchain integration
|
||||
ton_transaction_hash = Column(String(100), nullable=True, index=True)
|
||||
ton_storage_proof = Column(Text, nullable=True)
|
||||
ton_storage_fee = Column(BigInteger, default=0, nullable=False) # nanotons
|
||||
|
||||
# Expiration and cleanup
|
||||
expires_at = Column(DateTime, nullable=True)
|
||||
auto_delete = Column(Boolean, default=False, nullable=False)
|
||||
|
||||
# Relationships
|
||||
user = relationship("User", back_populates="content")
|
||||
|
||||
# Table indexes for performance
|
||||
__table_args__ = (
|
||||
Index('idx_content_user_active', 'user_id', 'is_active'),
|
||||
Index('idx_content_public_indexed', 'is_public', 'is_indexed'),
|
||||
Index('idx_content_file_type', 'file_type', 'mime_type'),
|
||||
Index('idx_content_created', 'created_at'),
|
||||
Index('idx_content_size', 'file_size'),
|
||||
Index('idx_content_processing', 'processing_status'),
|
||||
Index('idx_content_ton_tx', 'ton_transaction_hash'),
|
||||
Index('idx_content_expires', 'expires_at', 'auto_delete'),
|
||||
)
|
||||
|
||||
def is_expired(self) -> bool:
|
||||
"""Check if content is expired."""
|
||||
if not self.expires_at:
|
||||
return False
|
||||
return datetime.utcnow() > self.expires_at
|
||||
|
||||
def is_image(self) -> bool:
|
||||
"""Check if content is an image."""
|
||||
return self.file_type.lower() in ['image', 'img'] or \
|
||||
self.mime_type.startswith('image/')
|
||||
|
||||
def is_video(self) -> bool:
|
||||
"""Check if content is a video."""
|
||||
return self.file_type.lower() == 'video' or \
|
||||
self.mime_type.startswith('video/')
|
||||
|
||||
def is_document(self) -> bool:
|
||||
"""Check if content is a document."""
|
||||
return self.file_type.lower() in ['document', 'doc', 'pdf'] or \
|
||||
self.mime_type in ['application/pdf', 'application/msword', 'text/plain']
|
||||
|
||||
def get_file_size_human(self) -> str:
|
||||
"""Get human-readable file size."""
|
||||
size = self.file_size
|
||||
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
|
||||
if size < 1024.0:
|
||||
return f"{size:.1f} {unit}"
|
||||
size /= 1024.0
|
||||
return f"{size:.1f} PB"
|
||||
|
||||
def increment_download_count(self) -> None:
|
||||
"""Increment download counter."""
|
||||
self.download_count += 1
|
||||
|
||||
def increment_view_count(self) -> None:
|
||||
"""Increment view counter."""
|
||||
self.view_count += 1
|
||||
|
||||
def mark_as_indexed(self) -> None:
|
||||
"""Mark content as indexed."""
|
||||
self.is_indexed = True
|
||||
|
||||
def mark_as_converted(self) -> None:
|
||||
"""Mark content as converted."""
|
||||
self.is_converted = True
|
||||
self.processing_status = "completed"
|
||||
self.processing_completed = datetime.utcnow()
|
||||
|
||||
def set_processing_error(self, error: str) -> None:
|
||||
"""Set processing error."""
|
||||
self.processing_status = "error"
|
||||
self.processing_error = error
|
||||
self.processing_completed = datetime.utcnow()
|
||||
|
||||
def start_processing(self) -> None:
|
||||
"""Mark processing as started."""
|
||||
self.processing_status = "processing"
|
||||
self.processing_started = datetime.utcnow()
|
||||
self.processing_error = None
|
||||
|
||||
def get_tags_list(self) -> List[str]:
|
||||
"""Get tags as list."""
|
||||
if not self.tags:
|
||||
return []
|
||||
|
||||
# Try to parse as JSON first, fallback to comma-separated
|
||||
try:
|
||||
import json
|
||||
return json.loads(self.tags)
|
||||
except:
|
||||
return [tag.strip() for tag in self.tags.split(',') if tag.strip()]
|
||||
|
||||
def set_tags_list(self, tags: List[str]) -> None:
|
||||
"""Set tags from list."""
|
||||
import json
|
||||
self.tags = json.dumps(tags) if tags else None
|
||||
|
||||
def to_dict(self, include_sensitive: bool = False) -> Dict[str, Any]:
|
||||
"""Convert to dictionary with option to exclude sensitive data."""
|
||||
exclude = set()
|
||||
if not include_sensitive:
|
||||
exclude.update({"access_password", "file_path", "processing_error"})
|
||||
|
||||
data = super().to_dict(exclude=exclude)
|
||||
|
||||
# Add computed fields
|
||||
data.update({
|
||||
"file_size_human": self.get_file_size_human(),
|
||||
"is_image": self.is_image(),
|
||||
"is_video": self.is_video(),
|
||||
"is_document": self.is_document(),
|
||||
"is_expired": self.is_expired(),
|
||||
"tags_list": self.get_tags_list(),
|
||||
})
|
||||
|
||||
return data
|
||||
|
||||
def to_public_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to public dictionary (minimal content info)."""
|
||||
return {
|
||||
"id": self.id,
|
||||
"filename": self.filename,
|
||||
"title": self.title,
|
||||
"description": self.description,
|
||||
"file_type": self.file_type,
|
||||
"file_size": self.file_size,
|
||||
"file_size_human": self.get_file_size_human(),
|
||||
"is_image": self.is_image(),
|
||||
"is_video": self.is_video(),
|
||||
"is_document": self.is_document(),
|
||||
"download_count": self.download_count,
|
||||
"view_count": self.view_count,
|
||||
"tags_list": self.get_tags_list(),
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
class ContentShare(BaseModel):
|
||||
"""Content sharing model for tracking shared content."""
|
||||
|
||||
__tablename__ = "content_shares"
|
||||
|
||||
content_id = Column(Integer, ForeignKey('content.id'), nullable=False, index=True)
|
||||
user_id = Column(Integer, ForeignKey('users.id'), nullable=True, index=True) # Can be null for anonymous shares
|
||||
|
||||
# Share metadata
|
||||
share_token = Column(String(100), unique=True, nullable=False, index=True)
|
||||
share_url = Column(String(500), nullable=False)
|
||||
|
||||
# Share settings
|
||||
is_active = Column(Boolean, default=True, nullable=False)
|
||||
is_password_protected = Column(Boolean, default=False, nullable=False)
|
||||
share_password = Column(String(255), nullable=True)
|
||||
|
||||
# Access control
|
||||
max_downloads = Column(Integer, nullable=True) # Null = unlimited
|
||||
download_count = Column(Integer, default=0, nullable=False)
|
||||
view_count = Column(Integer, default=0, nullable=False)
|
||||
|
||||
# Time limits
|
||||
expires_at = Column(DateTime, nullable=True)
|
||||
|
||||
# Tracking
|
||||
ip_address = Column(String(45), nullable=True)
|
||||
user_agent = Column(Text, nullable=True)
|
||||
|
||||
# Relationships
|
||||
content = relationship("Content")
|
||||
user = relationship("User")
|
||||
|
||||
__table_args__ = (
|
||||
Index('idx_shares_content_active', 'content_id', 'is_active'),
|
||||
Index('idx_shares_token', 'share_token'),
|
||||
Index('idx_shares_expires', 'expires_at'),
|
||||
)
|
||||
|
||||
def is_expired(self) -> bool:
|
||||
"""Check if share is expired."""
|
||||
if not self.expires_at:
|
||||
return False
|
||||
return datetime.utcnow() > self.expires_at
|
||||
|
||||
def is_download_limit_reached(self) -> bool:
|
||||
"""Check if download limit is reached."""
|
||||
if not self.max_downloads:
|
||||
return False
|
||||
return self.download_count >= self.max_downloads
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""Check if share is valid."""
|
||||
return (self.is_active and
|
||||
not self.is_expired() and
|
||||
not self.is_download_limit_reached())
|
||||
|
||||
def increment_download(self) -> bool:
|
||||
"""Increment download count and return if still valid."""
|
||||
if not self.is_valid():
|
||||
return False
|
||||
|
||||
self.download_count += 1
|
||||
return self.is_valid()
|
||||
|
||||
def increment_view(self) -> None:
|
||||
"""Increment view count."""
|
||||
self.view_count += 1
|
||||
|
||||
|
||||
class ContentMetadata(BaseModel):
|
||||
"""Extended metadata for content files."""
|
||||
|
||||
__tablename__ = "content_metadata"
|
||||
|
||||
content_id = Column(Integer, ForeignKey('content.id'), unique=True, nullable=False, index=True)
|
||||
|
||||
# Image metadata
|
||||
image_width = Column(Integer, nullable=True)
|
||||
image_height = Column(Integer, nullable=True)
|
||||
image_dpi = Column(Integer, nullable=True)
|
||||
image_color_space = Column(String(50), nullable=True)
|
||||
|
||||
# Video metadata
|
||||
video_duration = Column(Integer, nullable=True) # seconds
|
||||
video_bitrate = Column(Integer, nullable=True)
|
||||
video_fps = Column(Integer, nullable=True)
|
||||
video_resolution = Column(String(20), nullable=True) # e.g., "1920x1080"
|
||||
video_codec = Column(String(50), nullable=True)
|
||||
|
||||
# Audio metadata
|
||||
audio_duration = Column(Integer, nullable=True) # seconds
|
||||
audio_bitrate = Column(Integer, nullable=True)
|
||||
audio_sample_rate = Column(Integer, nullable=True)
|
||||
audio_channels = Column(Integer, nullable=True)
|
||||
audio_codec = Column(String(50), nullable=True)
|
||||
|
||||
# Document metadata
|
||||
document_pages = Column(Integer, nullable=True)
|
||||
document_words = Column(Integer, nullable=True)
|
||||
document_language = Column(String(10), nullable=True)
|
||||
document_author = Column(String(255), nullable=True)
|
||||
|
||||
# EXIF data (JSON)
|
||||
exif_data = Column(Text, nullable=True)
|
||||
|
||||
# GPS coordinates
|
||||
gps_latitude = Column(String(50), nullable=True)
|
||||
gps_longitude = Column(String(50), nullable=True)
|
||||
gps_altitude = Column(String(50), nullable=True)
|
||||
|
||||
# Technical metadata
|
||||
compression_ratio = Column(String(20), nullable=True)
|
||||
quality_score = Column(Integer, nullable=True) # 0-100
|
||||
|
||||
# Relationships
|
||||
content = relationship("Content")
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert metadata to dictionary."""
|
||||
data = super().to_dict(exclude={"content_id"})
|
||||
|
||||
# Parse EXIF data if present
|
||||
if self.exif_data:
|
||||
try:
|
||||
import json
|
||||
data["exif_data"] = json.loads(self.exif_data)
|
||||
except:
|
||||
data["exif_data"] = None
|
||||
|
||||
return data
|
||||
|
||||
def set_exif_data(self, exif_dict: Dict[str, Any]) -> None:
|
||||
"""Set EXIF data from dictionary."""
|
||||
if exif_dict:
|
||||
import json
|
||||
self.exif_data = json.dumps(exif_dict)
|
||||
else:
|
||||
self.exif_data = None
|
||||
|
||||
def get_exif_data(self) -> Optional[Dict[str, Any]]:
|
||||
"""Get EXIF data as dictionary."""
|
||||
if not self.exif_data:
|
||||
return None
|
||||
|
||||
try:
|
||||
import json
|
||||
return json.loads(self.exif_data)
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
class ContentVersion(BaseModel):
|
||||
"""Content version history for tracking changes."""
|
||||
|
||||
__tablename__ = "content_versions"
|
||||
|
||||
content_id = Column(Integer, ForeignKey('content.id'), nullable=False, index=True)
|
||||
user_id = Column(Integer, ForeignKey('users.id'), nullable=False, index=True)
|
||||
|
||||
# Version information
|
||||
version_number = Column(Integer, nullable=False)
|
||||
version_name = Column(String(100), nullable=True)
|
||||
change_description = Column(Text, nullable=True)
|
||||
|
||||
# File information
|
||||
file_path = Column(String(500), nullable=False)
|
||||
file_size = Column(BigInteger, nullable=False)
|
||||
file_hash = Column(String(64), nullable=False)
|
||||
|
||||
# Status
|
||||
is_current = Column(Boolean, default=False, nullable=False)
|
||||
|
||||
# Relationships
|
||||
content = relationship("Content")
|
||||
user = relationship("User")
|
||||
|
||||
__table_args__ = (
|
||||
Index('idx_versions_content_number', 'content_id', 'version_number'),
|
||||
Index('idx_versions_current', 'content_id', 'is_current'),
|
||||
)
|
||||
|
||||
def mark_as_current(self) -> None:
|
||||
"""Mark this version as current."""
|
||||
self.is_current = True
|
||||
|
||||
|
||||
# Add relationship to User model
|
||||
# This would be added to the User model:
|
||||
# content = relationship("Content", back_populates="user")
|
||||
@@ -0,0 +1,420 @@
|
||||
"""
|
||||
User model with async support and enhanced security
|
||||
"""
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional, List, Dict, Any
|
||||
from enum import Enum
|
||||
|
||||
from sqlalchemy import Column, String, BigInteger, Boolean, Integer, Index, text
|
||||
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy.orm import relationship
|
||||
import structlog
|
||||
|
||||
from app.core.models.base import BaseModel
|
||||
from app.core.config import settings
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
class UserRole(str, Enum):
|
||||
"""User role enumeration"""
|
||||
USER = "user"
|
||||
MODERATOR = "moderator"
|
||||
ADMIN = "admin"
|
||||
SUPER_ADMIN = "super_admin"
|
||||
|
||||
|
||||
class UserStatus(str, Enum):
|
||||
"""User status enumeration"""
|
||||
ACTIVE = "active"
|
||||
SUSPENDED = "suspended"
|
||||
BANNED = "banned"
|
||||
PENDING = "pending"
|
||||
|
||||
|
||||
class User(BaseModel):
|
||||
"""Enhanced User model with security and async support"""
|
||||
|
||||
__tablename__ = 'users'
|
||||
|
||||
# Telegram specific fields
|
||||
telegram_id = Column(
|
||||
BigInteger,
|
||||
nullable=False,
|
||||
unique=True,
|
||||
index=True,
|
||||
comment="Telegram user ID"
|
||||
)
|
||||
username = Column(
|
||||
String(512),
|
||||
nullable=True,
|
||||
index=True,
|
||||
comment="Telegram username"
|
||||
)
|
||||
first_name = Column(
|
||||
String(256),
|
||||
nullable=True,
|
||||
comment="User first name"
|
||||
)
|
||||
last_name = Column(
|
||||
String(256),
|
||||
nullable=True,
|
||||
comment="User last name"
|
||||
)
|
||||
|
||||
# Localization
|
||||
language_code = Column(
|
||||
String(8),
|
||||
nullable=False,
|
||||
default="en",
|
||||
comment="User language code"
|
||||
)
|
||||
|
||||
# Security and access control
|
||||
role = Column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default=UserRole.USER.value,
|
||||
index=True,
|
||||
comment="User role"
|
||||
)
|
||||
permissions = Column(
|
||||
ARRAY(String),
|
||||
nullable=False,
|
||||
default=list,
|
||||
comment="User permissions list"
|
||||
)
|
||||
|
||||
# Activity tracking
|
||||
last_activity = Column(
|
||||
"last_use", # Keep old column name for compatibility
|
||||
DateTime,
|
||||
nullable=False,
|
||||
default=datetime.utcnow,
|
||||
index=True,
|
||||
comment="Last user activity timestamp"
|
||||
)
|
||||
login_count = Column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
default=0,
|
||||
comment="Total login count"
|
||||
)
|
||||
|
||||
# Account status
|
||||
is_verified = Column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
comment="Whether user is verified"
|
||||
)
|
||||
is_premium = Column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
comment="Whether user has premium access"
|
||||
)
|
||||
|
||||
# Security settings
|
||||
two_factor_enabled = Column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
comment="Whether 2FA is enabled"
|
||||
)
|
||||
security_settings = Column(
|
||||
JSONB,
|
||||
nullable=False,
|
||||
default=dict,
|
||||
comment="User security settings"
|
||||
)
|
||||
|
||||
# Preferences
|
||||
preferences = Column(
|
||||
JSONB,
|
||||
nullable=False,
|
||||
default=dict,
|
||||
comment="User preferences and settings"
|
||||
)
|
||||
|
||||
# Statistics
|
||||
content_uploaded_count = Column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
default=0,
|
||||
comment="Number of content items uploaded"
|
||||
)
|
||||
content_purchased_count = Column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
default=0,
|
||||
comment="Number of content items purchased"
|
||||
)
|
||||
|
||||
# Rate limiting
|
||||
rate_limit_reset = Column(
|
||||
DateTime,
|
||||
nullable=True,
|
||||
comment="Rate limit reset timestamp"
|
||||
)
|
||||
rate_limit_count = Column(
|
||||
Integer,
|
||||
nullable=False,
|
||||
default=0,
|
||||
comment="Current rate limit count"
|
||||
)
|
||||
|
||||
# Relationships
|
||||
balances = relationship('UserBalance', back_populates='user', cascade="all, delete-orphan")
|
||||
transactions = relationship('InternalTransaction', back_populates='user', cascade="all, delete-orphan")
|
||||
wallet_connections = relationship('WalletConnection', back_populates='user', cascade="all, delete-orphan")
|
||||
content_items = relationship('UserContent', back_populates='user', cascade="all, delete-orphan")
|
||||
actions = relationship('UserAction', back_populates='user', cascade="all, delete-orphan")
|
||||
activities = relationship('UserActivity', back_populates='user', cascade="all, delete-orphan")
|
||||
|
||||
# Indexes for performance
|
||||
__table_args__ = (
|
||||
Index('idx_users_telegram_id', 'telegram_id'),
|
||||
Index('idx_users_username', 'username'),
|
||||
Index('idx_users_role_status', 'role', 'status'),
|
||||
Index('idx_users_last_activity', 'last_activity'),
|
||||
Index('idx_users_created_at', 'created_at'),
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""String representation"""
|
||||
return f"User({self.id}, telegram_id={self.telegram_id}, username={self.username})"
|
||||
|
||||
@property
|
||||
def full_name(self) -> str:
|
||||
"""Get user's full name"""
|
||||
parts = [self.first_name, self.last_name]
|
||||
return " ".join(filter(None, parts)) or self.username or f"User_{self.telegram_id}"
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
"""Get user's display name"""
|
||||
return self.username or self.full_name
|
||||
|
||||
@property
|
||||
def is_admin(self) -> bool:
|
||||
"""Check if user is admin"""
|
||||
return self.role in [UserRole.ADMIN.value, UserRole.SUPER_ADMIN.value]
|
||||
|
||||
@property
|
||||
def is_moderator(self) -> bool:
|
||||
"""Check if user is moderator or higher"""
|
||||
return self.role in [UserRole.MODERATOR.value, UserRole.ADMIN.value, UserRole.SUPER_ADMIN.value]
|
||||
|
||||
@property
|
||||
def cache_key(self) -> str:
|
||||
"""Override cache key to include telegram_id"""
|
||||
return f"user:telegram:{self.telegram_id}"
|
||||
|
||||
def has_permission(self, permission: str) -> bool:
|
||||
"""Check if user has specific permission"""
|
||||
if self.is_admin:
|
||||
return True
|
||||
return permission in (self.permissions or [])
|
||||
|
||||
def add_permission(self, permission: str) -> None:
|
||||
"""Add permission to user"""
|
||||
if not self.permissions:
|
||||
self.permissions = []
|
||||
if permission not in self.permissions:
|
||||
self.permissions.append(permission)
|
||||
|
||||
def remove_permission(self, permission: str) -> None:
|
||||
"""Remove permission from user"""
|
||||
if self.permissions and permission in self.permissions:
|
||||
self.permissions.remove(permission)
|
||||
|
||||
def update_activity(self) -> None:
|
||||
"""Update user activity timestamp"""
|
||||
self.last_activity = datetime.utcnow()
|
||||
self.login_count += 1
|
||||
|
||||
def check_rate_limit(self, limit: int = None, window: int = None) -> bool:
|
||||
"""Check if user is within rate limits"""
|
||||
if self.is_admin:
|
||||
return True
|
||||
|
||||
limit = limit or settings.RATE_LIMIT_REQUESTS
|
||||
window = window or settings.RATE_LIMIT_WINDOW
|
||||
|
||||
now = datetime.utcnow()
|
||||
|
||||
# Reset counter if window has passed
|
||||
if not self.rate_limit_reset or now > self.rate_limit_reset:
|
||||
self.rate_limit_reset = now + timedelta(seconds=window)
|
||||
self.rate_limit_count = 0
|
||||
|
||||
return self.rate_limit_count < limit
|
||||
|
||||
def increment_rate_limit(self) -> None:
|
||||
"""Increment rate limit counter"""
|
||||
if not self.is_admin:
|
||||
self.rate_limit_count += 1
|
||||
|
||||
def set_preference(self, key: str, value: Any) -> None:
|
||||
"""Set user preference"""
|
||||
if not self.preferences:
|
||||
self.preferences = {}
|
||||
self.preferences[key] = value
|
||||
|
||||
def get_preference(self, key: str, default: Any = None) -> Any:
|
||||
"""Get user preference"""
|
||||
if not self.preferences:
|
||||
return default
|
||||
return self.preferences.get(key, default)
|
||||
|
||||
def set_security_setting(self, key: str, value: Any) -> None:
|
||||
"""Set security setting"""
|
||||
if not self.security_settings:
|
||||
self.security_settings = {}
|
||||
self.security_settings[key] = value
|
||||
|
||||
def get_security_setting(self, key: str, default: Any = None) -> Any:
|
||||
"""Get security setting"""
|
||||
if not self.security_settings:
|
||||
return default
|
||||
return self.security_settings.get(key, default)
|
||||
|
||||
def generate_api_token(self) -> str:
|
||||
"""Generate secure API token for user"""
|
||||
token_data = f"{self.id}:{self.telegram_id}:{datetime.utcnow().timestamp()}:{secrets.token_hex(16)}"
|
||||
return hashlib.sha256(token_data.encode()).hexdigest()
|
||||
|
||||
def can_upload_content(self) -> bool:
|
||||
"""Check if user can upload content"""
|
||||
if self.status != UserStatus.ACTIVE.value:
|
||||
return False
|
||||
|
||||
if not self.check_rate_limit(limit=10, window=3600): # 10 uploads per hour
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def can_purchase_content(self) -> bool:
|
||||
"""Check if user can purchase content"""
|
||||
return self.status == UserStatus.ACTIVE.value
|
||||
|
||||
@classmethod
|
||||
async def get_by_telegram_id(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
telegram_id: int
|
||||
) -> Optional['User']:
|
||||
"""Get user by Telegram ID"""
|
||||
try:
|
||||
stmt = select(cls).where(cls.telegram_id == telegram_id)
|
||||
result = await session.execute(stmt)
|
||||
return result.scalar_one_or_none()
|
||||
except Exception as e:
|
||||
logger.error("Error getting user by telegram_id", telegram_id=telegram_id, error=str(e))
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def get_by_username(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
username: str
|
||||
) -> Optional['User']:
|
||||
"""Get user by username"""
|
||||
try:
|
||||
stmt = select(cls).where(cls.username == username)
|
||||
result = await session.execute(stmt)
|
||||
return result.scalar_one_or_none()
|
||||
except Exception as e:
|
||||
logger.error("Error getting user by username", username=username, error=str(e))
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
async def get_active_users(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
days: int = 30,
|
||||
limit: Optional[int] = None
|
||||
) -> List['User']:
|
||||
"""Get active users within specified days"""
|
||||
try:
|
||||
cutoff_date = datetime.utcnow() - timedelta(days=days)
|
||||
stmt = select(cls).where(
|
||||
cls.last_activity >= cutoff_date,
|
||||
cls.status == UserStatus.ACTIVE.value
|
||||
).order_by(cls.last_activity.desc())
|
||||
|
||||
if limit:
|
||||
stmt = stmt.limit(limit)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
return result.scalars().all()
|
||||
except Exception as e:
|
||||
logger.error("Error getting active users", days=days, error=str(e))
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def get_admins(cls, session: AsyncSession) -> List['User']:
|
||||
"""Get all admin users"""
|
||||
try:
|
||||
stmt = select(cls).where(
|
||||
cls.role.in_([UserRole.ADMIN.value, UserRole.SUPER_ADMIN.value])
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
return result.scalars().all()
|
||||
except Exception as e:
|
||||
logger.error("Error getting admin users", error=str(e))
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
async def create_from_telegram(
|
||||
cls,
|
||||
session: AsyncSession,
|
||||
telegram_id: int,
|
||||
username: Optional[str] = None,
|
||||
first_name: Optional[str] = None,
|
||||
last_name: Optional[str] = None,
|
||||
language_code: str = "en"
|
||||
) -> 'User':
|
||||
"""Create user from Telegram data"""
|
||||
user = cls(
|
||||
telegram_id=telegram_id,
|
||||
username=username,
|
||||
first_name=first_name,
|
||||
last_name=last_name,
|
||||
language_code=language_code,
|
||||
status=UserStatus.ACTIVE.value
|
||||
)
|
||||
|
||||
session.add(user)
|
||||
await session.commit()
|
||||
await session.refresh(user)
|
||||
|
||||
logger.info("User created from Telegram", telegram_id=telegram_id, user_id=user.id)
|
||||
return user
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dictionary with safe data"""
|
||||
data = super().to_dict()
|
||||
|
||||
# Remove sensitive fields
|
||||
sensitive_fields = ['security_settings', 'permissions']
|
||||
for field in sensitive_fields:
|
||||
data.pop(field, None)
|
||||
|
||||
return data
|
||||
|
||||
def to_public_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to public dictionary with minimal data"""
|
||||
return {
|
||||
'id': str(self.id),
|
||||
'username': self.username,
|
||||
'display_name': self.display_name,
|
||||
'is_verified': self.is_verified,
|
||||
'is_premium': self.is_premium,
|
||||
'created_at': self.created_at.isoformat() if self.created_at else None
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Compatible user models for MariaDB."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional, List, Dict, Any
|
||||
from sqlalchemy import Column, String, Boolean, Text, Integer, DateTime, Index
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from app.core.models.base_compatible import BaseModel
|
||||
|
||||
|
||||
class User(BaseModel):
|
||||
"""User model compatible with existing MariaDB schema."""
|
||||
|
||||
__tablename__ = "users"
|
||||
|
||||
# Basic user information
|
||||
username = Column(String(50), unique=True, nullable=False, index=True)
|
||||
email = Column(String(100), unique=True, nullable=True, index=True)
|
||||
password_hash = Column(String(255), nullable=False)
|
||||
|
||||
# User status and flags
|
||||
is_active = Column(Boolean, default=True, nullable=False)
|
||||
is_verified = Column(Boolean, default=False, nullable=False)
|
||||
is_admin = Column(Boolean, default=False, nullable=False)
|
||||
|
||||
# Profile information
|
||||
first_name = Column(String(50), nullable=True)
|
||||
last_name = Column(String(50), nullable=True)
|
||||
bio = Column(Text, nullable=True)
|
||||
avatar_url = Column(String(255), nullable=True)
|
||||
|
||||
# System tracking
|
||||
last_login = Column(DateTime, nullable=True)
|
||||
login_count = Column(Integer, default=0, nullable=False)
|
||||
|
||||
# Storage and limits
|
||||
storage_used = Column(Integer, default=0, nullable=False) # bytes
|
||||
storage_limit = Column(Integer, default=100*1024*1024, nullable=False) # 100MB default
|
||||
|
||||
# TON Blockchain integration
|
||||
ton_wallet_address = Column(String(100), nullable=True, index=True)
|
||||
ton_balance = Column(Integer, default=0, nullable=False) # nanotons
|
||||
|
||||
# License and subscription
|
||||
license_key = Column(String(100), nullable=True, index=True)
|
||||
license_expires = Column(DateTime, nullable=True)
|
||||
subscription_level = Column(String(20), default="free", nullable=False)
|
||||
|
||||
# API access
|
||||
api_key = Column(String(100), nullable=True, unique=True, index=True)
|
||||
api_calls_count = Column(Integer, default=0, nullable=False)
|
||||
api_calls_limit = Column(Integer, default=1000, nullable=False)
|
||||
|
||||
# Relationships will be defined when we create content models
|
||||
|
||||
# Table indexes for performance
|
||||
__table_args__ = (
|
||||
Index('idx_users_username_active', 'username', 'is_active'),
|
||||
Index('idx_users_email_verified', 'email', 'is_verified'),
|
||||
Index('idx_users_ton_wallet', 'ton_wallet_address'),
|
||||
Index('idx_users_license', 'license_key', 'license_expires'),
|
||||
)
|
||||
|
||||
def check_storage_limit(self, file_size: int) -> bool:
|
||||
"""Check if user can upload file of given size."""
|
||||
return (self.storage_used + file_size) <= self.storage_limit
|
||||
|
||||
def update_storage_usage(self, size_change: int) -> None:
|
||||
"""Update user's storage usage."""
|
||||
self.storage_used = max(0, self.storage_used + size_change)
|
||||
|
||||
def is_license_valid(self) -> bool:
|
||||
"""Check if user's license is valid."""
|
||||
if not self.license_key or not self.license_expires:
|
||||
return False
|
||||
return self.license_expires > datetime.utcnow()
|
||||
|
||||
def can_make_api_call(self) -> bool:
|
||||
"""Check if user can make API call."""
|
||||
return self.api_calls_count < self.api_calls_limit
|
||||
|
||||
def increment_api_calls(self) -> None:
|
||||
"""Increment API calls counter."""
|
||||
self.api_calls_count += 1
|
||||
|
||||
def reset_api_calls(self) -> None:
|
||||
"""Reset API calls counter (for monthly reset)."""
|
||||
self.api_calls_count = 0
|
||||
|
||||
def get_storage_usage_percent(self) -> float:
|
||||
"""Get storage usage as percentage."""
|
||||
if self.storage_limit == 0:
|
||||
return 0.0
|
||||
return (self.storage_used / self.storage_limit) * 100
|
||||
|
||||
def get_api_usage_percent(self) -> float:
|
||||
"""Get API usage as percentage."""
|
||||
if self.api_calls_limit == 0:
|
||||
return 0.0
|
||||
return (self.api_calls_count / self.api_calls_limit) * 100
|
||||
|
||||
def get_display_name(self) -> str:
|
||||
"""Get user's display name."""
|
||||
if self.first_name and self.last_name:
|
||||
return f"{self.first_name} {self.last_name}"
|
||||
elif self.first_name:
|
||||
return self.first_name
|
||||
return self.username
|
||||
|
||||
def to_dict(self, include_sensitive: bool = False) -> Dict[str, Any]:
|
||||
"""Convert to dictionary with option to exclude sensitive data."""
|
||||
exclude = set()
|
||||
if not include_sensitive:
|
||||
exclude.update({"password_hash", "api_key", "license_key"})
|
||||
|
||||
data = super().to_dict(exclude=exclude)
|
||||
|
||||
# Add computed fields
|
||||
data.update({
|
||||
"display_name": self.get_display_name(),
|
||||
"storage_usage_percent": self.get_storage_usage_percent(),
|
||||
"api_usage_percent": self.get_api_usage_percent(),
|
||||
"license_valid": self.is_license_valid(),
|
||||
})
|
||||
|
||||
return data
|
||||
|
||||
def to_public_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to public dictionary (minimal user info)."""
|
||||
return {
|
||||
"id": self.id,
|
||||
"username": self.username,
|
||||
"display_name": self.get_display_name(),
|
||||
"avatar_url": self.avatar_url,
|
||||
"is_verified": self.is_verified,
|
||||
"subscription_level": self.subscription_level,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
class UserSession(BaseModel):
|
||||
"""User session model for tracking active sessions."""
|
||||
|
||||
__tablename__ = "user_sessions"
|
||||
|
||||
user_id = Column(Integer, nullable=False, index=True)
|
||||
session_token = Column(String(255), unique=True, nullable=False, index=True)
|
||||
refresh_token = Column(String(255), unique=True, nullable=True, index=True)
|
||||
|
||||
# Session metadata
|
||||
ip_address = Column(String(45), nullable=True) # IPv6 support
|
||||
user_agent = Column(Text, nullable=True)
|
||||
device_info = Column(Text, nullable=True)
|
||||
|
||||
# Session status
|
||||
is_active = Column(Boolean, default=True, nullable=False)
|
||||
expires_at = Column(DateTime, nullable=False)
|
||||
last_activity = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
# Security flags
|
||||
is_suspicious = Column(Boolean, default=False, nullable=False)
|
||||
failed_attempts = Column(Integer, default=0, nullable=False)
|
||||
|
||||
__table_args__ = (
|
||||
Index('idx_sessions_user_active', 'user_id', 'is_active'),
|
||||
Index('idx_sessions_token', 'session_token'),
|
||||
Index('idx_sessions_expires', 'expires_at'),
|
||||
)
|
||||
|
||||
def is_expired(self) -> bool:
|
||||
"""Check if session is expired."""
|
||||
return datetime.utcnow() > self.expires_at
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""Check if session is valid."""
|
||||
return self.is_active and not self.is_expired()
|
||||
|
||||
def extend_session(self, hours: int = 24) -> None:
|
||||
"""Extend session expiration."""
|
||||
from datetime import timedelta
|
||||
self.expires_at = datetime.utcnow() + timedelta(hours=hours)
|
||||
self.last_activity = datetime.utcnow()
|
||||
|
||||
def mark_suspicious(self) -> None:
|
||||
"""Mark session as suspicious."""
|
||||
self.is_suspicious = True
|
||||
self.failed_attempts += 1
|
||||
|
||||
def deactivate(self) -> None:
|
||||
"""Deactivate session."""
|
||||
self.is_active = False
|
||||
|
||||
|
||||
class UserPreferences(BaseModel):
|
||||
"""User preferences and settings."""
|
||||
|
||||
__tablename__ = "user_preferences"
|
||||
|
||||
user_id = Column(Integer, unique=True, nullable=False, index=True)
|
||||
|
||||
# UI preferences
|
||||
theme = Column(String(20), default="light", nullable=False)
|
||||
language = Column(String(10), default="en", nullable=False)
|
||||
timezone = Column(String(50), default="UTC", nullable=False)
|
||||
|
||||
# Notification preferences
|
||||
email_notifications = Column(Boolean, default=True, nullable=False)
|
||||
upload_notifications = Column(Boolean, default=True, nullable=False)
|
||||
storage_alerts = Column(Boolean, default=True, nullable=False)
|
||||
|
||||
# Privacy settings
|
||||
public_profile = Column(Boolean, default=False, nullable=False)
|
||||
show_email = Column(Boolean, default=False, nullable=False)
|
||||
allow_indexing = Column(Boolean, default=True, nullable=False)
|
||||
|
||||
# Upload preferences
|
||||
auto_optimize_images = Column(Boolean, default=True, nullable=False)
|
||||
default_privacy = Column(String(20), default="private", nullable=False)
|
||||
max_file_size_mb = Column(Integer, default=10, nullable=False)
|
||||
|
||||
# Cache and performance
|
||||
cache_thumbnails = Column(Boolean, default=True, nullable=False)
|
||||
preload_content = Column(Boolean, default=False, nullable=False)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert preferences to dictionary."""
|
||||
return super().to_dict(exclude={"user_id"})
|
||||
|
||||
@classmethod
|
||||
def get_default_preferences(cls) -> Dict[str, Any]:
|
||||
"""Get default user preferences."""
|
||||
return {
|
||||
"theme": "light",
|
||||
"language": "en",
|
||||
"timezone": "UTC",
|
||||
"email_notifications": True,
|
||||
"upload_notifications": True,
|
||||
"storage_alerts": True,
|
||||
"public_profile": False,
|
||||
"show_email": False,
|
||||
"allow_indexing": True,
|
||||
"auto_optimize_images": True,
|
||||
"default_privacy": "private",
|
||||
"max_file_size_mb": 10,
|
||||
"cache_thumbnails": True,
|
||||
"preload_content": False,
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
"""MY Network - Distributed Content Replication System."""
|
||||
|
||||
from .node_service import MyNetworkNodeService
|
||||
from .sync_manager import ContentSyncManager
|
||||
from .peer_manager import PeerManager
|
||||
from .bootstrap_manager import BootstrapManager
|
||||
|
||||
__all__ = [
|
||||
'MyNetworkNodeService',
|
||||
'ContentSyncManager',
|
||||
'PeerManager',
|
||||
'BootstrapManager'
|
||||
]
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Bootstrap Manager - управление bootstrap нодами и начальной конфигурацией."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BootstrapManager:
|
||||
"""Менеджер для работы с bootstrap конфигурацией."""
|
||||
|
||||
def __init__(self, bootstrap_path: str = "bootstrap.json"):
|
||||
self.bootstrap_path = Path(bootstrap_path)
|
||||
self.config = {}
|
||||
self.nodes_history_path = Path("nodes_history.json")
|
||||
self.nodes_history = {"successful_connections": [], "last_updated": None}
|
||||
|
||||
logger.info(f"Bootstrap Manager initialized with path: {self.bootstrap_path}")
|
||||
|
||||
async def load_bootstrap_config(self) -> Dict[str, Any]:
|
||||
"""Загрузка bootstrap конфигурации."""
|
||||
try:
|
||||
if not self.bootstrap_path.exists():
|
||||
logger.error(f"Bootstrap config not found: {self.bootstrap_path}")
|
||||
raise FileNotFoundError(f"Bootstrap config not found: {self.bootstrap_path}")
|
||||
|
||||
with open(self.bootstrap_path, 'r', encoding='utf-8') as f:
|
||||
self.config = json.load(f)
|
||||
|
||||
logger.info(f"Bootstrap config loaded: {len(self.config.get('bootstrap_nodes', []))} nodes")
|
||||
|
||||
# Загрузить историю нод
|
||||
await self._load_nodes_history()
|
||||
|
||||
return self.config
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading bootstrap config: {e}")
|
||||
raise
|
||||
|
||||
async def _load_nodes_history(self) -> None:
|
||||
"""Загрузка истории подключенных нод."""
|
||||
try:
|
||||
if self.nodes_history_path.exists():
|
||||
with open(self.nodes_history_path, 'r', encoding='utf-8') as f:
|
||||
self.nodes_history = json.load(f)
|
||||
|
||||
logger.info(f"Loaded nodes history: {len(self.nodes_history.get('successful_connections', []))} nodes")
|
||||
else:
|
||||
logger.info("No nodes history found, starting fresh")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading nodes history: {e}")
|
||||
self.nodes_history = {"successful_connections": [], "last_updated": None}
|
||||
|
||||
async def save_nodes_history(self) -> None:
|
||||
"""Сохранение истории нод."""
|
||||
try:
|
||||
self.nodes_history["last_updated"] = datetime.utcnow().isoformat()
|
||||
|
||||
with open(self.nodes_history_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(self.nodes_history, f, indent=2, ensure_ascii=False)
|
||||
|
||||
logger.debug("Nodes history saved")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error saving nodes history: {e}")
|
||||
|
||||
def get_bootstrap_nodes(self) -> List[Dict[str, Any]]:
|
||||
"""Получить список bootstrap нод."""
|
||||
return self.config.get('bootstrap_nodes', [])
|
||||
|
||||
def get_network_settings(self) -> Dict[str, Any]:
|
||||
"""Получить настройки сети."""
|
||||
return self.config.get('network_settings', {})
|
||||
|
||||
def get_sync_settings(self) -> Dict[str, Any]:
|
||||
"""Получить настройки синхронизации."""
|
||||
return self.config.get('sync_settings', {})
|
||||
|
||||
def get_content_settings(self) -> Dict[str, Any]:
|
||||
"""Получить настройки контента."""
|
||||
return self.config.get('content_settings', {})
|
||||
|
||||
def get_security_settings(self) -> Dict[str, Any]:
|
||||
"""Получить настройки безопасности."""
|
||||
return self.config.get('security_settings', {})
|
||||
|
||||
def get_api_settings(self) -> Dict[str, Any]:
|
||||
"""Получить настройки API."""
|
||||
return self.config.get('api_settings', {})
|
||||
|
||||
def get_monitoring_settings(self) -> Dict[str, Any]:
|
||||
"""Получить настройки мониторинга."""
|
||||
return self.config.get('monitoring_settings', {})
|
||||
|
||||
def get_storage_settings(self) -> Dict[str, Any]:
|
||||
"""Получить настройки хранилища."""
|
||||
return self.config.get('storage_settings', {})
|
||||
|
||||
def get_consensus_settings(self) -> Dict[str, Any]:
|
||||
"""Получить настройки консенсуса."""
|
||||
return self.config.get('consensus', {})
|
||||
|
||||
def get_feature_flags(self) -> Dict[str, Any]:
|
||||
"""Получить флаги функций."""
|
||||
return self.config.get('feature_flags', {})
|
||||
|
||||
def is_feature_enabled(self, feature_name: str) -> bool:
|
||||
"""Проверить, включена ли функция."""
|
||||
return self.get_feature_flags().get(feature_name, False)
|
||||
|
||||
def get_regional_settings(self, region: str = None) -> Dict[str, Any]:
|
||||
"""Получить региональные настройки."""
|
||||
regional_settings = self.config.get('regional_settings', {})
|
||||
|
||||
if region and region in regional_settings:
|
||||
return regional_settings[region]
|
||||
|
||||
return regional_settings
|
||||
|
||||
def get_emergency_settings(self) -> Dict[str, Any]:
|
||||
"""Получить настройки экстренных ситуаций."""
|
||||
return self.config.get('emergency_settings', {})
|
||||
|
||||
def is_emergency_mode(self) -> bool:
|
||||
"""Проверить, включен ли режим экстренной ситуации."""
|
||||
return self.get_emergency_settings().get('emergency_mode', False)
|
||||
|
||||
def get_nodes_from_history(self) -> List[Dict[str, Any]]:
|
||||
"""Получить ноды из истории успешных подключений."""
|
||||
return self.nodes_history.get('successful_connections', [])
|
||||
|
||||
def add_successful_connection(self, node_info: Dict[str, Any]) -> None:
|
||||
"""Добавить информацию об успешном подключении."""
|
||||
try:
|
||||
# Обновить существующую запись или добавить новую
|
||||
existing_node = None
|
||||
for i, node in enumerate(self.nodes_history['successful_connections']):
|
||||
if node['node_id'] == node_info['node_id']:
|
||||
existing_node = i
|
||||
break
|
||||
|
||||
connection_info = {
|
||||
"node_id": node_info['node_id'],
|
||||
"address": node_info['address'],
|
||||
"last_seen": datetime.utcnow().isoformat(),
|
||||
"connection_count": node_info.get('connection_count', 1),
|
||||
"performance_score": node_info.get('performance_score', 1.0),
|
||||
"features": node_info.get('features', []),
|
||||
"region": node_info.get('region', 'unknown'),
|
||||
"metadata": node_info.get('metadata', {})
|
||||
}
|
||||
|
||||
if existing_node is not None:
|
||||
# Обновить существующую запись
|
||||
old_info = self.nodes_history['successful_connections'][existing_node]
|
||||
connection_info['connection_count'] = old_info.get('connection_count', 0) + 1
|
||||
connection_info['first_seen'] = old_info.get('first_seen', connection_info['last_seen'])
|
||||
|
||||
self.nodes_history['successful_connections'][existing_node] = connection_info
|
||||
else:
|
||||
# Добавить новую запись
|
||||
connection_info['first_seen'] = connection_info['last_seen']
|
||||
self.nodes_history['successful_connections'].append(connection_info)
|
||||
|
||||
# Ограничить историю (максимум 100 нод)
|
||||
if len(self.nodes_history['successful_connections']) > 100:
|
||||
# Сортировать по последнему подключению и оставить 100 самых свежих
|
||||
self.nodes_history['successful_connections'].sort(
|
||||
key=lambda x: x['last_seen'],
|
||||
reverse=True
|
||||
)
|
||||
self.nodes_history['successful_connections'] = \
|
||||
self.nodes_history['successful_connections'][:100]
|
||||
|
||||
logger.debug(f"Added successful connection to history: {node_info['node_id']}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error adding successful connection: {e}")
|
||||
|
||||
def remove_failed_connection(self, node_id: str) -> None:
|
||||
"""Удалить ноду из истории при неудачном подключении."""
|
||||
try:
|
||||
self.nodes_history['successful_connections'] = [
|
||||
node for node in self.nodes_history['successful_connections']
|
||||
if node['node_id'] != node_id
|
||||
]
|
||||
|
||||
logger.debug(f"Removed failed connection from history: {node_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error removing failed connection: {e}")
|
||||
|
||||
def get_preferred_nodes(self, max_nodes: int = 10) -> List[Dict[str, Any]]:
|
||||
"""Получить предпочтительные ноды для подключения."""
|
||||
try:
|
||||
# Комбинировать bootstrap ноды и ноды из истории
|
||||
all_nodes = []
|
||||
|
||||
# Добавить bootstrap ноды (высокий приоритет)
|
||||
for node in self.get_bootstrap_nodes():
|
||||
all_nodes.append({
|
||||
"node_id": node['id'],
|
||||
"address": node['address'],
|
||||
"priority": 100, # Высокий приоритет для bootstrap
|
||||
"features": node.get('features', []),
|
||||
"region": node.get('region', 'unknown'),
|
||||
"source": "bootstrap"
|
||||
})
|
||||
|
||||
# Добавить ноды из истории
|
||||
for node in self.get_nodes_from_history():
|
||||
# Пропустить, если уже есть в bootstrap
|
||||
if any(n['node_id'] == node['node_id'] for n in all_nodes):
|
||||
continue
|
||||
|
||||
# Рассчитать приоритет на основе performance_score и connection_count
|
||||
priority = min(90, node.get('performance_score', 0.5) * 50 +
|
||||
min(40, node.get('connection_count', 1) * 2))
|
||||
|
||||
all_nodes.append({
|
||||
"node_id": node['node_id'],
|
||||
"address": node['address'],
|
||||
"priority": priority,
|
||||
"features": node.get('features', []),
|
||||
"region": node.get('region', 'unknown'),
|
||||
"source": "history"
|
||||
})
|
||||
|
||||
# Сортировать по приоритету и взять топ
|
||||
all_nodes.sort(key=lambda x: x['priority'], reverse=True)
|
||||
|
||||
return all_nodes[:max_nodes]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting preferred nodes: {e}")
|
||||
return []
|
||||
|
||||
def validate_config(self) -> bool:
|
||||
"""Валидация конфигурации bootstrap."""
|
||||
try:
|
||||
required_fields = ['version', 'network_id', 'bootstrap_nodes']
|
||||
|
||||
for field in required_fields:
|
||||
if field not in self.config:
|
||||
logger.error(f"Missing required field: {field}")
|
||||
return False
|
||||
|
||||
# Проверить bootstrap ноды
|
||||
bootstrap_nodes = self.config.get('bootstrap_nodes', [])
|
||||
if not bootstrap_nodes:
|
||||
logger.error("No bootstrap nodes configured")
|
||||
return False
|
||||
|
||||
for node in bootstrap_nodes:
|
||||
required_node_fields = ['id', 'address']
|
||||
for field in required_node_fields:
|
||||
if field not in node:
|
||||
logger.error(f"Bootstrap node missing field: {field}")
|
||||
return False
|
||||
|
||||
logger.info("Bootstrap configuration validated successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error validating config: {e}")
|
||||
return False
|
||||
|
||||
def get_config_checksum(self) -> str:
|
||||
"""Получить чек-сумму конфигурации."""
|
||||
return self.config.get('checksum', '')
|
||||
|
||||
def verify_config_signature(self) -> bool:
|
||||
"""Проверить подпись конфигурации."""
|
||||
# Заглушка для проверки подписи
|
||||
# В реальной реализации здесь была бы криптографическая проверка
|
||||
signature = self.config.get('signature', '')
|
||||
return bool(signature)
|
||||
|
||||
async def update_bootstrap_config(self, new_config: Dict[str, Any]) -> bool:
|
||||
"""Обновление bootstrap конфигурации."""
|
||||
try:
|
||||
# Сохранить резервную копию
|
||||
backup_path = self.bootstrap_path.with_suffix('.backup')
|
||||
if self.bootstrap_path.exists():
|
||||
self.bootstrap_path.rename(backup_path)
|
||||
|
||||
# Сохранить новую конфигурацию
|
||||
with open(self.bootstrap_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(new_config, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Перезагрузить конфигурацию
|
||||
await self.load_bootstrap_config()
|
||||
|
||||
logger.info("Bootstrap configuration updated successfully")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating bootstrap config: {e}")
|
||||
|
||||
# Восстановить из резервной копии
|
||||
try:
|
||||
if backup_path.exists():
|
||||
backup_path.rename(self.bootstrap_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
return False
|
||||
@@ -0,0 +1,386 @@
|
||||
"""MY Network Node Service - основной сервис ноды."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Set, Any
|
||||
from pathlib import Path
|
||||
|
||||
from app.core.database_compatible import get_async_session
|
||||
from app.core.models.content_compatible import Content
|
||||
from app.core.cache import cache
|
||||
from .bootstrap_manager import BootstrapManager
|
||||
from .peer_manager import PeerManager
|
||||
from .sync_manager import ContentSyncManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MyNetworkNodeService:
|
||||
"""Основной сервис ноды MY Network."""
|
||||
|
||||
def __init__(self, node_id: str = None, storage_path: str = "./storage/my-network"):
|
||||
self.node_id = node_id or self._generate_node_id()
|
||||
self.storage_path = Path(storage_path)
|
||||
self.storage_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Инициализация менеджеров
|
||||
self.bootstrap_manager = BootstrapManager()
|
||||
self.peer_manager = PeerManager(self.node_id)
|
||||
self.sync_manager = ContentSyncManager(self.node_id)
|
||||
|
||||
# Состояние ноды
|
||||
self.is_running = False
|
||||
self.start_time = None
|
||||
self.last_sync_time = None
|
||||
self.node_metrics = {
|
||||
"requests_30min": 0,
|
||||
"total_requests": 0,
|
||||
"content_synced": 0,
|
||||
"active_peers": 0,
|
||||
"storage_used_mb": 0
|
||||
}
|
||||
|
||||
# История запросов для балансировки нагрузки
|
||||
self.request_history = []
|
||||
|
||||
logger.info(f"MY Network Node Service initialized with ID: {self.node_id}")
|
||||
|
||||
def _generate_node_id(self) -> str:
|
||||
"""Генерация уникального ID ноды."""
|
||||
import uuid
|
||||
return f"node-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Запуск ноды MY Network."""
|
||||
try:
|
||||
logger.info(f"Starting MY Network Node: {self.node_id}")
|
||||
|
||||
# Загрузка bootstrap конфигурации
|
||||
await self.bootstrap_manager.load_bootstrap_config()
|
||||
|
||||
# Инициализация peer manager
|
||||
await self.peer_manager.initialize()
|
||||
|
||||
# Подключение к bootstrap нодам
|
||||
await self._connect_to_bootstrap_nodes()
|
||||
|
||||
# Обнаружение других нод в сети
|
||||
await self._discover_network_nodes()
|
||||
|
||||
# Запуск синхронизации контента
|
||||
await self.sync_manager.start_sync_process()
|
||||
|
||||
# Запуск фоновых задач
|
||||
asyncio.create_task(self._background_tasks())
|
||||
|
||||
self.is_running = True
|
||||
self.start_time = datetime.utcnow()
|
||||
|
||||
logger.info(f"MY Network Node {self.node_id} started successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start MY Network Node: {e}")
|
||||
raise
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Остановка ноды MY Network."""
|
||||
try:
|
||||
logger.info(f"Stopping MY Network Node: {self.node_id}")
|
||||
|
||||
self.is_running = False
|
||||
|
||||
# Остановка синхронизации
|
||||
await self.sync_manager.stop_sync_process()
|
||||
|
||||
# Отключение от пиров
|
||||
await self.peer_manager.disconnect_all()
|
||||
|
||||
logger.info(f"MY Network Node {self.node_id} stopped")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error stopping MY Network Node: {e}")
|
||||
|
||||
async def _connect_to_bootstrap_nodes(self) -> None:
|
||||
"""Подключение к bootstrap нодам."""
|
||||
bootstrap_nodes = self.bootstrap_manager.get_bootstrap_nodes()
|
||||
|
||||
for node in bootstrap_nodes:
|
||||
try:
|
||||
# Не подключаться к самому себе
|
||||
if node["id"] == self.node_id:
|
||||
continue
|
||||
|
||||
success = await self.peer_manager.connect_to_peer(
|
||||
node["id"],
|
||||
node["address"]
|
||||
)
|
||||
|
||||
if success:
|
||||
logger.info(f"Connected to bootstrap node: {node['id']}")
|
||||
else:
|
||||
logger.warning(f"Failed to connect to bootstrap node: {node['id']}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error connecting to bootstrap node {node['id']}: {e}")
|
||||
|
||||
async def _discover_network_nodes(self) -> None:
|
||||
"""Обнаружение других нод в сети."""
|
||||
try:
|
||||
# Запрос списка нод у подключенных пиров
|
||||
connected_peers = self.peer_manager.get_connected_peers()
|
||||
|
||||
for peer_id in connected_peers:
|
||||
try:
|
||||
nodes_list = await self.peer_manager.request_nodes_list(peer_id)
|
||||
|
||||
for node_info in nodes_list:
|
||||
# Пропустить себя
|
||||
if node_info["id"] == self.node_id:
|
||||
continue
|
||||
|
||||
# Попытаться подключиться к новой ноде
|
||||
if not self.peer_manager.is_connected(node_info["id"]):
|
||||
await self.peer_manager.connect_to_peer(
|
||||
node_info["id"],
|
||||
node_info["address"]
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error discovering nodes from peer {peer_id}: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in network discovery: {e}")
|
||||
|
||||
async def _background_tasks(self) -> None:
|
||||
"""Фоновые задачи ноды."""
|
||||
while self.is_running:
|
||||
try:
|
||||
# Обновление метрик
|
||||
await self._update_metrics()
|
||||
|
||||
# Очистка истории запросов (оставляем только за последние 30 минут)
|
||||
await self._cleanup_request_history()
|
||||
|
||||
# Проверка состояния пиров
|
||||
await self.peer_manager.check_peers_health()
|
||||
|
||||
# Периодическая синхронизация
|
||||
if self._should_sync():
|
||||
await self.sync_manager.sync_with_network()
|
||||
self.last_sync_time = datetime.utcnow()
|
||||
|
||||
# Обновление кэша статистики
|
||||
await self._update_cache_stats()
|
||||
|
||||
await asyncio.sleep(30) # Проверка каждые 30 секунд
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in background tasks: {e}")
|
||||
await asyncio.sleep(60) # Увеличиваем интервал при ошибке
|
||||
|
||||
async def _update_metrics(self) -> None:
|
||||
"""Обновление метрик ноды."""
|
||||
try:
|
||||
# Подсчет запросов за последние 30 минут
|
||||
cutoff_time = datetime.utcnow() - timedelta(minutes=30)
|
||||
recent_requests = [
|
||||
req for req in self.request_history
|
||||
if req["timestamp"] > cutoff_time
|
||||
]
|
||||
|
||||
self.node_metrics.update({
|
||||
"requests_30min": len(recent_requests),
|
||||
"active_peers": len(self.peer_manager.get_connected_peers()),
|
||||
"storage_used_mb": await self._calculate_storage_usage(),
|
||||
"uptime_hours": self._get_uptime_hours()
|
||||
})
|
||||
|
||||
# Сохранение в кэш для быстрого доступа
|
||||
await cache.set(
|
||||
f"my_network:node:{self.node_id}:metrics",
|
||||
self.node_metrics,
|
||||
ttl=60
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating metrics: {e}")
|
||||
|
||||
async def _cleanup_request_history(self) -> None:
|
||||
"""Очистка истории запросов."""
|
||||
cutoff_time = datetime.utcnow() - timedelta(minutes=30)
|
||||
self.request_history = [
|
||||
req for req in self.request_history
|
||||
if req["timestamp"] > cutoff_time
|
||||
]
|
||||
|
||||
def _should_sync(self) -> bool:
|
||||
"""Проверка, нужно ли запускать синхронизацию."""
|
||||
if not self.last_sync_time:
|
||||
return True
|
||||
|
||||
# Синхронизация каждые 5 минут
|
||||
return datetime.utcnow() - self.last_sync_time > timedelta(minutes=5)
|
||||
|
||||
async def _calculate_storage_usage(self) -> int:
|
||||
"""Подсчет использования хранилища в МБ."""
|
||||
try:
|
||||
total_size = 0
|
||||
if self.storage_path.exists():
|
||||
for file_path in self.storage_path.rglob("*"):
|
||||
if file_path.is_file():
|
||||
total_size += file_path.stat().st_size
|
||||
|
||||
return total_size // (1024 * 1024) # Конвертация в МБ
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating storage usage: {e}")
|
||||
return 0
|
||||
|
||||
def _get_uptime_hours(self) -> float:
|
||||
"""Получение времени работы ноды в часах."""
|
||||
if not self.start_time:
|
||||
return 0.0
|
||||
|
||||
uptime = datetime.utcnow() - self.start_time
|
||||
return uptime.total_seconds() / 3600
|
||||
|
||||
async def _update_cache_stats(self) -> None:
|
||||
"""Обновление статистики в кэше."""
|
||||
try:
|
||||
stats = {
|
||||
"node_id": self.node_id,
|
||||
"is_running": self.is_running,
|
||||
"start_time": self.start_time.isoformat() if self.start_time else None,
|
||||
"last_sync_time": self.last_sync_time.isoformat() if self.last_sync_time else None,
|
||||
"metrics": self.node_metrics,
|
||||
"connected_peers": list(self.peer_manager.get_connected_peers()),
|
||||
"sync_status": await self.sync_manager.get_sync_status()
|
||||
}
|
||||
|
||||
await cache.set(
|
||||
f"my_network:node:{self.node_id}:status",
|
||||
stats,
|
||||
ttl=30
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating cache stats: {e}")
|
||||
|
||||
def record_request(self, request_info: Dict[str, Any]) -> None:
|
||||
"""Записать информацию о запросе для метрик."""
|
||||
self.request_history.append({
|
||||
"timestamp": datetime.utcnow(),
|
||||
"endpoint": request_info.get("endpoint", "unknown"),
|
||||
"method": request_info.get("method", "GET"),
|
||||
"client_ip": request_info.get("client_ip", "unknown")
|
||||
})
|
||||
|
||||
self.node_metrics["total_requests"] += 1
|
||||
|
||||
def get_load_info(self) -> Dict[str, Any]:
|
||||
"""Получить информацию о нагрузке ноды для балансировки."""
|
||||
return {
|
||||
"node_id": self.node_id,
|
||||
"requests_30min": self.node_metrics["requests_30min"],
|
||||
"load_percentage": min(100, (self.node_metrics["requests_30min"] / 1000) * 100),
|
||||
"active_peers": self.node_metrics["active_peers"],
|
||||
"storage_used_mb": self.node_metrics["storage_used_mb"],
|
||||
"uptime_hours": self._get_uptime_hours(),
|
||||
"is_healthy": self.is_running and self.node_metrics["active_peers"] > 0
|
||||
}
|
||||
|
||||
async def replicate_content(self, content_hash: str, target_nodes: List[str] = None) -> Dict[str, Any]:
|
||||
"""Реплицировать контент на другие ноды."""
|
||||
try:
|
||||
logger.info(f"Starting replication of content: {content_hash}")
|
||||
|
||||
# Найти контент в локальной БД
|
||||
async with get_async_session() as session:
|
||||
content = await session.get(Content, {"hash": content_hash})
|
||||
|
||||
if not content:
|
||||
raise ValueError(f"Content not found: {content_hash}")
|
||||
|
||||
# Определить целевые ноды
|
||||
if not target_nodes:
|
||||
target_nodes = self.peer_manager.select_replication_nodes()
|
||||
|
||||
# Запустить репликацию через sync manager
|
||||
result = await self.sync_manager.replicate_content_to_nodes(
|
||||
content_hash,
|
||||
target_nodes
|
||||
)
|
||||
|
||||
logger.info(f"Content replication completed: {content_hash}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error replicating content {content_hash}: {e}")
|
||||
raise
|
||||
|
||||
async def get_network_status(self) -> Dict[str, Any]:
|
||||
"""Получить статус всей сети MY Network."""
|
||||
try:
|
||||
connected_peers = self.peer_manager.get_connected_peers()
|
||||
sync_status = await self.sync_manager.get_sync_status()
|
||||
|
||||
# Получить статус от всех подключенных пиров
|
||||
peer_statuses = {}
|
||||
for peer_id in connected_peers:
|
||||
try:
|
||||
peer_status = await self.peer_manager.request_peer_status(peer_id)
|
||||
peer_statuses[peer_id] = peer_status
|
||||
except Exception as e:
|
||||
peer_statuses[peer_id] = {"error": str(e)}
|
||||
|
||||
return {
|
||||
"local_node": {
|
||||
"id": self.node_id,
|
||||
"status": "running" if self.is_running else "stopped",
|
||||
"metrics": self.node_metrics,
|
||||
"uptime_hours": self._get_uptime_hours()
|
||||
},
|
||||
"network": {
|
||||
"connected_peers": len(connected_peers),
|
||||
"total_discovered_nodes": len(peer_statuses) + 1,
|
||||
"sync_status": sync_status,
|
||||
"last_sync": self.last_sync_time.isoformat() if self.last_sync_time else None
|
||||
},
|
||||
"peers": peer_statuses
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting network status: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
async def get_content_sync_status(self, content_hash: str) -> Dict[str, Any]:
|
||||
"""Получить статус синхронизации конкретного контента."""
|
||||
return await self.sync_manager.get_content_sync_status(content_hash)
|
||||
|
||||
|
||||
# Глобальный экземпляр сервиса ноды
|
||||
_node_service: Optional[MyNetworkNodeService] = None
|
||||
|
||||
|
||||
def get_node_service() -> MyNetworkNodeService:
|
||||
"""Получить глобальный экземпляр сервиса ноды."""
|
||||
global _node_service
|
||||
if _node_service is None:
|
||||
_node_service = MyNetworkNodeService()
|
||||
return _node_service
|
||||
|
||||
|
||||
async def initialize_my_network() -> None:
|
||||
"""Инициализация MY Network."""
|
||||
node_service = get_node_service()
|
||||
await node_service.start()
|
||||
|
||||
|
||||
async def shutdown_my_network() -> None:
|
||||
"""Остановка MY Network."""
|
||||
global _node_service
|
||||
if _node_service:
|
||||
await _node_service.stop()
|
||||
_node_service = None
|
||||
@@ -0,0 +1,477 @@
|
||||
"""Peer Manager - управление подключениями к другим нодам."""
|
||||
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Set, Optional, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PeerConnection:
|
||||
"""Представление подключения к пиру."""
|
||||
|
||||
def __init__(self, peer_id: str, address: str):
|
||||
self.peer_id = peer_id
|
||||
self.address = address
|
||||
self.connected_at = datetime.utcnow()
|
||||
self.last_ping = None
|
||||
self.last_pong = None
|
||||
self.is_healthy = True
|
||||
self.ping_failures = 0
|
||||
self.request_count = 0
|
||||
self.features = []
|
||||
self.metadata = {}
|
||||
|
||||
@property
|
||||
def uptime(self) -> timedelta:
|
||||
"""Время подключения."""
|
||||
return datetime.utcnow() - self.connected_at
|
||||
|
||||
@property
|
||||
def ping_latency(self) -> Optional[float]:
|
||||
"""Задержка пинга в миллисекундах."""
|
||||
if self.last_ping and self.last_pong:
|
||||
return (self.last_pong - self.last_ping).total_seconds() * 1000
|
||||
return None
|
||||
|
||||
def mark_ping_sent(self):
|
||||
"""Отметить отправку пинга."""
|
||||
self.last_ping = datetime.utcnow()
|
||||
|
||||
def mark_pong_received(self):
|
||||
"""Отметить получение понга."""
|
||||
self.last_pong = datetime.utcnow()
|
||||
self.ping_failures = 0
|
||||
self.is_healthy = True
|
||||
|
||||
def mark_ping_failed(self):
|
||||
"""Отметить неудачный пинг."""
|
||||
self.ping_failures += 1
|
||||
if self.ping_failures >= 3:
|
||||
self.is_healthy = False
|
||||
|
||||
|
||||
class PeerManager:
|
||||
"""Менеджер для управления подключениями к пирам."""
|
||||
|
||||
def __init__(self, node_id: str):
|
||||
self.node_id = node_id
|
||||
self.connections: Dict[str, PeerConnection] = {}
|
||||
self.blacklisted_peers: Set[str] = set()
|
||||
self.connection_semaphore = asyncio.Semaphore(25) # Макс 25 исходящих подключений
|
||||
self.session: Optional[aiohttp.ClientSession] = None
|
||||
|
||||
logger.info(f"Peer Manager initialized for node: {node_id}")
|
||||
|
||||
async def initialize(self) -> None:
|
||||
"""Инициализация менеджера пиров."""
|
||||
try:
|
||||
# Создать HTTP сессию для запросов
|
||||
timeout = aiohttp.ClientTimeout(total=30, connect=10)
|
||||
self.session = aiohttp.ClientSession(
|
||||
timeout=timeout,
|
||||
headers={'User-Agent': f'MY-Network-Node/{self.node_id}'}
|
||||
)
|
||||
|
||||
logger.info("Peer Manager initialized successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error initializing Peer Manager: {e}")
|
||||
raise
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
"""Очистка ресурсов."""
|
||||
if self.session:
|
||||
await self.session.close()
|
||||
self.session = None
|
||||
|
||||
self.connections.clear()
|
||||
logger.info("Peer Manager cleaned up")
|
||||
|
||||
async def connect_to_peer(self, peer_id: str, address: str) -> bool:
|
||||
"""Подключение к пиру."""
|
||||
try:
|
||||
# Проверить, что не подключаемся к себе
|
||||
if peer_id == self.node_id:
|
||||
logger.debug(f"Skipping connection to self: {peer_id}")
|
||||
return False
|
||||
|
||||
# Проверить черный список
|
||||
if peer_id in self.blacklisted_peers:
|
||||
logger.debug(f"Peer {peer_id} is blacklisted")
|
||||
return False
|
||||
|
||||
# Проверить, уже подключены ли
|
||||
if peer_id in self.connections:
|
||||
connection = self.connections[peer_id]
|
||||
if connection.is_healthy:
|
||||
logger.debug(f"Already connected to peer: {peer_id}")
|
||||
return True
|
||||
else:
|
||||
# Удалить нездоровое подключение
|
||||
del self.connections[peer_id]
|
||||
|
||||
async with self.connection_semaphore:
|
||||
logger.info(f"Connecting to peer: {peer_id} at {address}")
|
||||
|
||||
# Попытка подключения через handshake
|
||||
success = await self._perform_handshake(peer_id, address)
|
||||
|
||||
if success:
|
||||
# Создать подключение
|
||||
connection = PeerConnection(peer_id, address)
|
||||
self.connections[peer_id] = connection
|
||||
|
||||
logger.info(f"Successfully connected to peer: {peer_id}")
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Failed to connect to peer: {peer_id}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error connecting to peer {peer_id}: {e}")
|
||||
return False
|
||||
|
||||
async def _perform_handshake(self, peer_id: str, address: str) -> bool:
|
||||
"""Выполнить handshake с пиром."""
|
||||
try:
|
||||
if not self.session:
|
||||
return False
|
||||
|
||||
# Парсить адрес
|
||||
parsed_url = self._parse_peer_address(address)
|
||||
if not parsed_url:
|
||||
return False
|
||||
|
||||
handshake_url = f"{parsed_url}/api/my/handshake"
|
||||
|
||||
handshake_data = {
|
||||
"node_id": self.node_id,
|
||||
"protocol_version": "1.0.0",
|
||||
"features": [
|
||||
"content_sync",
|
||||
"consensus",
|
||||
"monitoring"
|
||||
],
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
async with self.session.post(handshake_url, json=handshake_data) as response:
|
||||
if response.status == 200:
|
||||
response_data = await response.json()
|
||||
|
||||
# Проверить ответ
|
||||
if (response_data.get("node_id") == peer_id and
|
||||
response_data.get("status") == "accepted"):
|
||||
|
||||
# Сохранить информацию о пире
|
||||
if peer_id in self.connections:
|
||||
self.connections[peer_id].features = response_data.get("features", [])
|
||||
self.connections[peer_id].metadata = response_data.get("metadata", {})
|
||||
|
||||
return True
|
||||
|
||||
logger.warning(f"Handshake failed with peer {peer_id}: HTTP {response.status}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in handshake with peer {peer_id}: {e}")
|
||||
return False
|
||||
|
||||
def _parse_peer_address(self, address: str) -> Optional[str]:
|
||||
"""Парсинг адреса пира."""
|
||||
try:
|
||||
# Поддержка форматов:
|
||||
# my://host:port
|
||||
# http://host:port
|
||||
# https://host:port
|
||||
# host:port
|
||||
|
||||
if address.startswith("my://"):
|
||||
# Конвертировать MY протокол в HTTP
|
||||
address = address.replace("my://", "http://")
|
||||
elif not address.startswith(("http://", "https://")):
|
||||
# Добавить HTTP префикс
|
||||
address = f"http://{address}"
|
||||
|
||||
parsed = urlparse(address)
|
||||
if parsed.hostname:
|
||||
return f"{parsed.scheme}://{parsed.netloc}"
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing peer address {address}: {e}")
|
||||
return None
|
||||
|
||||
async def disconnect_from_peer(self, peer_id: str) -> None:
|
||||
"""Отключение от пира."""
|
||||
try:
|
||||
if peer_id in self.connections:
|
||||
connection = self.connections[peer_id]
|
||||
|
||||
# Попытаться отправить уведомление об отключении
|
||||
try:
|
||||
await self._send_disconnect_notification(peer_id)
|
||||
except:
|
||||
pass # Игнорировать ошибки при отключении
|
||||
|
||||
# Удалить подключение
|
||||
del self.connections[peer_id]
|
||||
|
||||
logger.info(f"Disconnected from peer: {peer_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error disconnecting from peer {peer_id}: {e}")
|
||||
|
||||
async def _send_disconnect_notification(self, peer_id: str) -> None:
|
||||
"""Отправить уведомление об отключении."""
|
||||
try:
|
||||
if peer_id not in self.connections or not self.session:
|
||||
return
|
||||
|
||||
connection = self.connections[peer_id]
|
||||
parsed_url = self._parse_peer_address(connection.address)
|
||||
|
||||
if parsed_url:
|
||||
disconnect_url = f"{parsed_url}/api/my/disconnect"
|
||||
disconnect_data = {
|
||||
"node_id": self.node_id,
|
||||
"reason": "graceful_shutdown",
|
||||
"timestamp": datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
async with self.session.post(disconnect_url, json=disconnect_data) as response:
|
||||
if response.status == 200:
|
||||
logger.debug(f"Disconnect notification sent to {peer_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error sending disconnect notification to {peer_id}: {e}")
|
||||
|
||||
async def disconnect_all(self) -> None:
|
||||
"""Отключение от всех пиров."""
|
||||
disconnect_tasks = []
|
||||
|
||||
for peer_id in list(self.connections.keys()):
|
||||
disconnect_tasks.append(self.disconnect_from_peer(peer_id))
|
||||
|
||||
if disconnect_tasks:
|
||||
await asyncio.gather(*disconnect_tasks, return_exceptions=True)
|
||||
|
||||
logger.info("Disconnected from all peers")
|
||||
|
||||
async def check_peers_health(self) -> None:
|
||||
"""Проверка здоровья всех подключений."""
|
||||
ping_tasks = []
|
||||
|
||||
for peer_id in list(self.connections.keys()):
|
||||
ping_tasks.append(self._ping_peer(peer_id))
|
||||
|
||||
if ping_tasks:
|
||||
await asyncio.gather(*ping_tasks, return_exceptions=True)
|
||||
|
||||
# Удалить нездоровые подключения
|
||||
unhealthy_peers = [
|
||||
peer_id for peer_id, conn in self.connections.items()
|
||||
if not conn.is_healthy
|
||||
]
|
||||
|
||||
for peer_id in unhealthy_peers:
|
||||
logger.warning(f"Removing unhealthy peer: {peer_id}")
|
||||
await self.disconnect_from_peer(peer_id)
|
||||
|
||||
async def _ping_peer(self, peer_id: str) -> None:
|
||||
"""Пинг пира."""
|
||||
try:
|
||||
if peer_id not in self.connections or not self.session:
|
||||
return
|
||||
|
||||
connection = self.connections[peer_id]
|
||||
parsed_url = self._parse_peer_address(connection.address)
|
||||
|
||||
if not parsed_url:
|
||||
connection.mark_ping_failed()
|
||||
return
|
||||
|
||||
ping_url = f"{parsed_url}/api/my/ping"
|
||||
connection.mark_ping_sent()
|
||||
|
||||
async with self.session.get(ping_url) as response:
|
||||
if response.status == 200:
|
||||
connection.mark_pong_received()
|
||||
logger.debug(f"Ping successful to {peer_id}, latency: {connection.ping_latency:.1f}ms")
|
||||
else:
|
||||
connection.mark_ping_failed()
|
||||
logger.debug(f"Ping failed to {peer_id}: HTTP {response.status}")
|
||||
|
||||
except Exception as e:
|
||||
if peer_id in self.connections:
|
||||
self.connections[peer_id].mark_ping_failed()
|
||||
logger.debug(f"Ping error to {peer_id}: {e}")
|
||||
|
||||
def get_connected_peers(self) -> Set[str]:
|
||||
"""Получить множество подключенных пиров."""
|
||||
return {
|
||||
peer_id for peer_id, conn in self.connections.items()
|
||||
if conn.is_healthy
|
||||
}
|
||||
|
||||
def is_connected(self, peer_id: str) -> bool:
|
||||
"""Проверить, подключены ли к пиру."""
|
||||
return (peer_id in self.connections and
|
||||
self.connections[peer_id].is_healthy)
|
||||
|
||||
def get_peer_info(self, peer_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Получить информацию о пире."""
|
||||
if peer_id not in self.connections:
|
||||
return None
|
||||
|
||||
connection = self.connections[peer_id]
|
||||
return {
|
||||
"peer_id": peer_id,
|
||||
"address": connection.address,
|
||||
"connected_at": connection.connected_at.isoformat(),
|
||||
"uptime_seconds": connection.uptime.total_seconds(),
|
||||
"is_healthy": connection.is_healthy,
|
||||
"ping_latency_ms": connection.ping_latency,
|
||||
"ping_failures": connection.ping_failures,
|
||||
"request_count": connection.request_count,
|
||||
"features": connection.features,
|
||||
"metadata": connection.metadata
|
||||
}
|
||||
|
||||
def get_all_peers_info(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Получить информацию обо всех пирах."""
|
||||
return {
|
||||
peer_id: self.get_peer_info(peer_id)
|
||||
for peer_id in self.connections.keys()
|
||||
}
|
||||
|
||||
def select_replication_nodes(self, count: int = 3) -> List[str]:
|
||||
"""Выбрать ноды для репликации контента."""
|
||||
healthy_peers = [
|
||||
peer_id for peer_id, conn in self.connections.items()
|
||||
if conn.is_healthy
|
||||
]
|
||||
|
||||
if len(healthy_peers) <= count:
|
||||
return healthy_peers
|
||||
|
||||
# Выбрать ноды с лучшими характеристиками
|
||||
peer_scores = []
|
||||
for peer_id in healthy_peers:
|
||||
connection = self.connections[peer_id]
|
||||
|
||||
# Рассчитать оценку на основе различных факторов
|
||||
latency_score = 1.0
|
||||
if connection.ping_latency:
|
||||
latency_score = max(0.1, 1.0 - (connection.ping_latency / 1000))
|
||||
|
||||
uptime_score = min(1.0, connection.uptime.total_seconds() / 3600) # Время работы в часах
|
||||
failure_score = max(0.1, 1.0 - (connection.ping_failures / 10))
|
||||
|
||||
total_score = (latency_score * 0.4 + uptime_score * 0.3 + failure_score * 0.3)
|
||||
peer_scores.append((peer_id, total_score))
|
||||
|
||||
# Сортировать по оценке и взять топ
|
||||
peer_scores.sort(key=lambda x: x[1], reverse=True)
|
||||
return [peer_id for peer_id, _ in peer_scores[:count]]
|
||||
|
||||
async def request_nodes_list(self, peer_id: str) -> List[Dict[str, Any]]:
|
||||
"""Запросить список нод у пира."""
|
||||
try:
|
||||
if peer_id not in self.connections or not self.session:
|
||||
return []
|
||||
|
||||
connection = self.connections[peer_id]
|
||||
parsed_url = self._parse_peer_address(connection.address)
|
||||
|
||||
if not parsed_url:
|
||||
return []
|
||||
|
||||
nodes_url = f"{parsed_url}/api/my/nodes"
|
||||
|
||||
async with self.session.get(nodes_url) as response:
|
||||
if response.status == 200:
|
||||
data = await response.json()
|
||||
return data.get("nodes", [])
|
||||
else:
|
||||
logger.warning(f"Failed to get nodes list from {peer_id}: HTTP {response.status}")
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error requesting nodes list from {peer_id}: {e}")
|
||||
return []
|
||||
|
||||
async def request_peer_status(self, peer_id: str) -> Dict[str, Any]:
|
||||
"""Запросить статус пира."""
|
||||
try:
|
||||
if peer_id not in self.connections or not self.session:
|
||||
return {"error": "Not connected"}
|
||||
|
||||
connection = self.connections[peer_id]
|
||||
parsed_url = self._parse_peer_address(connection.address)
|
||||
|
||||
if not parsed_url:
|
||||
return {"error": "Invalid address"}
|
||||
|
||||
status_url = f"{parsed_url}/api/my/status"
|
||||
|
||||
async with self.session.get(status_url) as response:
|
||||
if response.status == 200:
|
||||
return await response.json()
|
||||
else:
|
||||
return {"error": f"HTTP {response.status}"}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error requesting peer status from {peer_id}: {e}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def add_to_blacklist(self, peer_id: str, duration_hours: int = 24) -> None:
|
||||
"""Добавить пира в черный список."""
|
||||
self.blacklisted_peers.add(peer_id)
|
||||
|
||||
# Запланировать удаление из черного списка
|
||||
async def remove_from_blacklist():
|
||||
await asyncio.sleep(duration_hours * 3600)
|
||||
self.blacklisted_peers.discard(peer_id)
|
||||
logger.info(f"Removed {peer_id} from blacklist")
|
||||
|
||||
asyncio.create_task(remove_from_blacklist())
|
||||
|
||||
logger.info(f"Added {peer_id} to blacklist for {duration_hours} hours")
|
||||
|
||||
def get_connection_stats(self) -> Dict[str, Any]:
|
||||
"""Получить статистику подключений."""
|
||||
healthy_connections = sum(1 for conn in self.connections.values() if conn.is_healthy)
|
||||
|
||||
return {
|
||||
"total_connections": len(self.connections),
|
||||
"healthy_connections": healthy_connections,
|
||||
"blacklisted_peers": len(self.blacklisted_peers),
|
||||
"average_latency_ms": self._calculate_average_latency(),
|
||||
"connection_details": [
|
||||
{
|
||||
"peer_id": peer_id,
|
||||
"uptime_hours": conn.uptime.total_seconds() / 3600,
|
||||
"ping_latency_ms": conn.ping_latency,
|
||||
"is_healthy": conn.is_healthy
|
||||
}
|
||||
for peer_id, conn in self.connections.items()
|
||||
]
|
||||
}
|
||||
|
||||
def _calculate_average_latency(self) -> Optional[float]:
|
||||
"""Рассчитать среднюю задержку."""
|
||||
latencies = [
|
||||
conn.ping_latency for conn in self.connections.values()
|
||||
if conn.ping_latency is not None and conn.is_healthy
|
||||
]
|
||||
|
||||
if latencies:
|
||||
return sum(latencies) / len(latencies)
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,698 @@
|
||||
"""Content Sync Manager - синхронизация контента между нодами."""
|
||||
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any, Set
|
||||
from sqlalchemy import select, and_
|
||||
|
||||
from app.core.database_compatible import get_async_session
|
||||
from app.core.models.content_compatible import Content, ContentMetadata
|
||||
from app.core.cache import cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ContentSyncStatus:
|
||||
"""Статус синхронизации контента."""
|
||||
|
||||
def __init__(self, content_hash: str):
|
||||
self.content_hash = content_hash
|
||||
self.sync_started = datetime.utcnow()
|
||||
self.sync_completed = None
|
||||
self.nodes_synced = set()
|
||||
self.nodes_failed = set()
|
||||
self.total_nodes = 0
|
||||
self.bytes_synced = 0
|
||||
self.status = "syncing" # syncing, completed, failed, partial
|
||||
self.error_message = None
|
||||
|
||||
@property
|
||||
def is_completed(self) -> bool:
|
||||
return self.status in ["completed", "partial"]
|
||||
|
||||
@property
|
||||
def success_rate(self) -> float:
|
||||
if self.total_nodes == 0:
|
||||
return 0.0
|
||||
return len(self.nodes_synced) / self.total_nodes
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"content_hash": self.content_hash,
|
||||
"status": self.status,
|
||||
"sync_started": self.sync_started.isoformat(),
|
||||
"sync_completed": self.sync_completed.isoformat() if self.sync_completed else None,
|
||||
"nodes_synced": list(self.nodes_synced),
|
||||
"nodes_failed": list(self.nodes_failed),
|
||||
"total_nodes": self.total_nodes,
|
||||
"bytes_synced": self.bytes_synced,
|
||||
"success_rate": self.success_rate,
|
||||
"error_message": self.error_message
|
||||
}
|
||||
|
||||
|
||||
class ContentSyncManager:
|
||||
"""Менеджер синхронизации контента между нодами MY Network."""
|
||||
|
||||
def __init__(self, node_id: str):
|
||||
self.node_id = node_id
|
||||
self.sync_queue: asyncio.Queue = asyncio.Queue()
|
||||
self.active_syncs: Dict[str, ContentSyncStatus] = {}
|
||||
self.sync_history: List[ContentSyncStatus] = []
|
||||
self.is_running = False
|
||||
self.sync_workers: List[asyncio.Task] = []
|
||||
self.session: Optional[aiohttp.ClientSession] = None
|
||||
|
||||
# Настройки синхронизации
|
||||
self.max_concurrent_syncs = 5
|
||||
self.chunk_size = 1024 * 1024 # 1MB chunks
|
||||
self.sync_timeout = 300 # 5 minutes per content
|
||||
self.retry_attempts = 3
|
||||
|
||||
logger.info(f"Content Sync Manager initialized for node: {node_id}")
|
||||
|
||||
async def start_sync_process(self) -> None:
|
||||
"""Запуск процесса синхронизации."""
|
||||
try:
|
||||
# Создать HTTP сессию
|
||||
timeout = aiohttp.ClientTimeout(total=self.sync_timeout)
|
||||
self.session = aiohttp.ClientSession(timeout=timeout)
|
||||
|
||||
# Запустить worker'ы для синхронизации
|
||||
self.is_running = True
|
||||
for i in range(self.max_concurrent_syncs):
|
||||
worker = asyncio.create_task(self._sync_worker(f"worker-{i}"))
|
||||
self.sync_workers.append(worker)
|
||||
|
||||
logger.info(f"Started {len(self.sync_workers)} sync workers")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting sync process: {e}")
|
||||
raise
|
||||
|
||||
async def stop_sync_process(self) -> None:
|
||||
"""Остановка процесса синхронизации."""
|
||||
try:
|
||||
self.is_running = False
|
||||
|
||||
# Остановить worker'ы
|
||||
for worker in self.sync_workers:
|
||||
worker.cancel()
|
||||
|
||||
if self.sync_workers:
|
||||
await asyncio.gather(*self.sync_workers, return_exceptions=True)
|
||||
|
||||
# Закрыть HTTP сессию
|
||||
if self.session:
|
||||
await self.session.close()
|
||||
self.session = None
|
||||
|
||||
self.sync_workers.clear()
|
||||
logger.info("Sync process stopped")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error stopping sync process: {e}")
|
||||
|
||||
async def _sync_worker(self, worker_name: str) -> None:
|
||||
"""Worker для обработки очереди синхронизации."""
|
||||
logger.info(f"Sync worker {worker_name} started")
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
# Получить задачу из очереди
|
||||
sync_task = await asyncio.wait_for(
|
||||
self.sync_queue.get(),
|
||||
timeout=1.0
|
||||
)
|
||||
|
||||
# Обработать задачу синхронизации
|
||||
await self._process_sync_task(sync_task)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
continue # Продолжить ожидание
|
||||
except Exception as e:
|
||||
logger.error(f"Error in sync worker {worker_name}: {e}")
|
||||
await asyncio.sleep(5) # Пауза при ошибке
|
||||
|
||||
logger.info(f"Sync worker {worker_name} stopped")
|
||||
|
||||
async def _process_sync_task(self, sync_task: Dict[str, Any]) -> None:
|
||||
"""Обработка задачи синхронизации."""
|
||||
try:
|
||||
task_type = sync_task.get("type")
|
||||
content_hash = sync_task.get("content_hash")
|
||||
target_nodes = sync_task.get("target_nodes", [])
|
||||
|
||||
if task_type == "replicate":
|
||||
await self._replicate_content(content_hash, target_nodes)
|
||||
elif task_type == "download":
|
||||
source_node = sync_task.get("source_node")
|
||||
await self._download_content(content_hash, source_node)
|
||||
elif task_type == "verify":
|
||||
await self._verify_content_integrity(content_hash)
|
||||
else:
|
||||
logger.warning(f"Unknown sync task type: {task_type}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing sync task: {e}")
|
||||
|
||||
async def replicate_content_to_nodes(self, content_hash: str, target_nodes: List[str]) -> Dict[str, Any]:
|
||||
"""Реплицировать контент на указанные ноды."""
|
||||
try:
|
||||
# Создать статус синхронизации
|
||||
sync_status = ContentSyncStatus(content_hash)
|
||||
sync_status.total_nodes = len(target_nodes)
|
||||
self.active_syncs[content_hash] = sync_status
|
||||
|
||||
# Добавить задачу в очередь
|
||||
sync_task = {
|
||||
"type": "replicate",
|
||||
"content_hash": content_hash,
|
||||
"target_nodes": target_nodes
|
||||
}
|
||||
|
||||
await self.sync_queue.put(sync_task)
|
||||
|
||||
logger.info(f"Queued replication of {content_hash} to {len(target_nodes)} nodes")
|
||||
|
||||
return {
|
||||
"status": "queued",
|
||||
"content_hash": content_hash,
|
||||
"target_nodes": target_nodes,
|
||||
"sync_id": content_hash
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error queuing content replication: {e}")
|
||||
raise
|
||||
|
||||
async def _replicate_content(self, content_hash: str, target_nodes: List[str]) -> None:
|
||||
"""Реплицировать контент на целевые ноды."""
|
||||
try:
|
||||
if content_hash not in self.active_syncs:
|
||||
logger.warning(f"No sync status found for content: {content_hash}")
|
||||
return
|
||||
|
||||
sync_status = self.active_syncs[content_hash]
|
||||
|
||||
# Получить контент из локальной БД
|
||||
content_info = await self._get_local_content_info(content_hash)
|
||||
if not content_info:
|
||||
sync_status.status = "failed"
|
||||
sync_status.error_message = "Content not found locally"
|
||||
return
|
||||
|
||||
# Реплицировать на каждую ноду
|
||||
replication_tasks = []
|
||||
for node_id in target_nodes:
|
||||
task = self._replicate_to_single_node(content_hash, node_id, content_info)
|
||||
replication_tasks.append(task)
|
||||
|
||||
# Ждать завершения всех репликаций
|
||||
results = await asyncio.gather(*replication_tasks, return_exceptions=True)
|
||||
|
||||
# Обработать результаты
|
||||
for i, result in enumerate(results):
|
||||
node_id = target_nodes[i]
|
||||
|
||||
if isinstance(result, Exception):
|
||||
sync_status.nodes_failed.add(node_id)
|
||||
logger.error(f"Replication to {node_id} failed: {result}")
|
||||
elif result:
|
||||
sync_status.nodes_synced.add(node_id)
|
||||
sync_status.bytes_synced += content_info.get("file_size", 0)
|
||||
logger.info(f"Successfully replicated to {node_id}")
|
||||
else:
|
||||
sync_status.nodes_failed.add(node_id)
|
||||
|
||||
# Завершить синхронизацию
|
||||
self._complete_sync(sync_status)
|
||||
|
||||
except Exception as e:
|
||||
if content_hash in self.active_syncs:
|
||||
self.active_syncs[content_hash].status = "failed"
|
||||
self.active_syncs[content_hash].error_message = str(e)
|
||||
logger.error(f"Error replicating content {content_hash}: {e}")
|
||||
|
||||
async def _replicate_to_single_node(self, content_hash: str, node_id: str, content_info: Dict[str, Any]) -> bool:
|
||||
"""Реплицировать контент на одну ноду."""
|
||||
try:
|
||||
if not self.session:
|
||||
return False
|
||||
|
||||
# Получить адрес ноды (через peer manager)
|
||||
from .node_service import get_node_service
|
||||
node_service = get_node_service()
|
||||
peer_info = node_service.peer_manager.get_peer_info(node_id)
|
||||
|
||||
if not peer_info:
|
||||
logger.warning(f"No peer info for node: {node_id}")
|
||||
return False
|
||||
|
||||
# Парсить адрес
|
||||
peer_address = node_service.peer_manager._parse_peer_address(peer_info["address"])
|
||||
if not peer_address:
|
||||
return False
|
||||
|
||||
# Проверить, нужна ли репликация
|
||||
check_url = f"{peer_address}/api/my/content/{content_hash}/exists"
|
||||
async with self.session.get(check_url) as response:
|
||||
if response.status == 200:
|
||||
exists_data = await response.json()
|
||||
if exists_data.get("exists", False):
|
||||
logger.debug(f"Content {content_hash} already exists on {node_id}")
|
||||
return True
|
||||
|
||||
# Начать репликацию
|
||||
replicate_url = f"{peer_address}/api/my/content/replicate"
|
||||
|
||||
# Подготовить данные для репликации
|
||||
replication_data = {
|
||||
"content_hash": content_hash,
|
||||
"metadata": content_info,
|
||||
"source_node": self.node_id
|
||||
}
|
||||
|
||||
async with self.session.post(replicate_url, json=replication_data) as response:
|
||||
if response.status == 200:
|
||||
# Передать сам файл
|
||||
success = await self._upload_content_to_node(
|
||||
content_hash,
|
||||
peer_address,
|
||||
content_info
|
||||
)
|
||||
return success
|
||||
else:
|
||||
logger.warning(f"Replication request failed to {node_id}: HTTP {response.status}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error replicating to node {node_id}: {e}")
|
||||
return False
|
||||
|
||||
async def _upload_content_to_node(self, content_hash: str, peer_address: str, content_info: Dict[str, Any]) -> bool:
|
||||
"""Загрузить файл контента на ноду."""
|
||||
try:
|
||||
if not self.session:
|
||||
return False
|
||||
|
||||
# Найти файл локально
|
||||
file_path = Path(content_info.get("file_path", ""))
|
||||
if not file_path.exists():
|
||||
logger.error(f"Local file not found: {file_path}")
|
||||
return False
|
||||
|
||||
upload_url = f"{peer_address}/api/my/content/{content_hash}/upload"
|
||||
|
||||
# Создать multipart upload
|
||||
with open(file_path, 'rb') as file:
|
||||
data = aiohttp.FormData()
|
||||
data.add_field('file', file, filename=content_info.get("filename", "unknown"))
|
||||
|
||||
async with self.session.post(upload_url, data=data) as response:
|
||||
if response.status == 200:
|
||||
result = await response.json()
|
||||
return result.get("success", False)
|
||||
else:
|
||||
logger.error(f"File upload failed: HTTP {response.status}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error uploading content to node: {e}")
|
||||
return False
|
||||
|
||||
async def _get_local_content_info(self, content_hash: str) -> Optional[Dict[str, Any]]:
|
||||
"""Получить информацию о локальном контенте."""
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
# Найти контент по хешу
|
||||
stmt = select(Content).where(Content.md5_hash == content_hash or Content.sha256_hash == content_hash)
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content:
|
||||
return None
|
||||
|
||||
# Получить метаданные
|
||||
metadata_stmt = select(ContentMetadata).where(ContentMetadata.content_id == content.id)
|
||||
metadata_result = await session.execute(metadata_stmt)
|
||||
metadata = metadata_result.scalar_one_or_none()
|
||||
|
||||
return {
|
||||
"id": content.id,
|
||||
"hash": content_hash,
|
||||
"filename": content.filename,
|
||||
"original_filename": content.original_filename,
|
||||
"file_path": content.file_path,
|
||||
"file_size": content.file_size,
|
||||
"file_type": content.file_type,
|
||||
"mime_type": content.mime_type,
|
||||
"encrypted": content.encrypted if hasattr(content, 'encrypted') else False,
|
||||
"metadata": metadata.to_dict() if metadata else {}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting local content info: {e}")
|
||||
return None
|
||||
|
||||
async def download_content_from_network(self, content_hash: str, source_nodes: List[str] = None) -> bool:
|
||||
"""Скачать контент из сети."""
|
||||
try:
|
||||
# Добавить задачу загрузки в очередь
|
||||
for source_node in (source_nodes or []):
|
||||
sync_task = {
|
||||
"type": "download",
|
||||
"content_hash": content_hash,
|
||||
"source_node": source_node
|
||||
}
|
||||
|
||||
await self.sync_queue.put(sync_task)
|
||||
|
||||
logger.info(f"Queued download of {content_hash} from {len(source_nodes or [])} nodes")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error queuing content download: {e}")
|
||||
return False
|
||||
|
||||
async def _download_content(self, content_hash: str, source_node: str) -> bool:
|
||||
"""Скачать контент с конкретной ноды."""
|
||||
try:
|
||||
if not self.session:
|
||||
return False
|
||||
|
||||
# Получить адрес исходной ноды
|
||||
from .node_service import get_node_service
|
||||
node_service = get_node_service()
|
||||
peer_info = node_service.peer_manager.get_peer_info(source_node)
|
||||
|
||||
if not peer_info:
|
||||
logger.warning(f"No peer info for source node: {source_node}")
|
||||
return False
|
||||
|
||||
peer_address = node_service.peer_manager._parse_peer_address(peer_info["address"])
|
||||
if not peer_address:
|
||||
return False
|
||||
|
||||
# Получить метаданные контента
|
||||
metadata_url = f"{peer_address}/api/my/content/{content_hash}/metadata"
|
||||
async with self.session.get(metadata_url) as response:
|
||||
if response.status != 200:
|
||||
logger.error(f"Failed to get content metadata: HTTP {response.status}")
|
||||
return False
|
||||
|
||||
content_metadata = await response.json()
|
||||
|
||||
# Скачать файл
|
||||
download_url = f"{peer_address}/api/my/content/{content_hash}/download"
|
||||
async with self.session.get(download_url) as response:
|
||||
if response.status != 200:
|
||||
logger.error(f"Failed to download content: HTTP {response.status}")
|
||||
return False
|
||||
|
||||
# Сохранить файл локально
|
||||
local_path = await self._save_downloaded_content(
|
||||
content_hash,
|
||||
response,
|
||||
content_metadata
|
||||
)
|
||||
|
||||
if local_path:
|
||||
# Сохранить в базу данных
|
||||
await self._save_content_to_db(content_hash, local_path, content_metadata)
|
||||
logger.info(f"Successfully downloaded content {content_hash} from {source_node}")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading content from {source_node}: {e}")
|
||||
return False
|
||||
|
||||
async def _save_downloaded_content(self, content_hash: str, response: aiohttp.ClientResponse, metadata: Dict[str, Any]) -> Optional[Path]:
|
||||
"""Сохранить скачанный контент."""
|
||||
try:
|
||||
# Создать путь для сохранения
|
||||
storage_path = Path("./storage/my-network/downloaded")
|
||||
storage_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
filename = metadata.get("filename", f"{content_hash}")
|
||||
file_path = storage_path / filename
|
||||
|
||||
# Сохранить файл
|
||||
with open(file_path, 'wb') as f:
|
||||
async for chunk in response.content.iter_chunked(self.chunk_size):
|
||||
f.write(chunk)
|
||||
|
||||
# Проверить целостность
|
||||
if await self._verify_file_integrity(file_path, content_hash):
|
||||
return file_path
|
||||
else:
|
||||
file_path.unlink() # Удалить поврежденный файл
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error saving downloaded content: {e}")
|
||||
return None
|
||||
|
||||
async def _verify_file_integrity(self, file_path: Path, expected_hash: str) -> bool:
|
||||
"""Проверить целостность файла."""
|
||||
try:
|
||||
# Вычислить хеш файла
|
||||
hash_md5 = hashlib.md5()
|
||||
hash_sha256 = hashlib.sha256()
|
||||
|
||||
with open(file_path, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(self.chunk_size), b""):
|
||||
hash_md5.update(chunk)
|
||||
hash_sha256.update(chunk)
|
||||
|
||||
file_md5 = hash_md5.hexdigest()
|
||||
file_sha256 = hash_sha256.hexdigest()
|
||||
|
||||
# Проверить соответствие
|
||||
return expected_hash in [file_md5, file_sha256]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error verifying file integrity: {e}")
|
||||
return False
|
||||
|
||||
async def _save_content_to_db(self, content_hash: str, file_path: Path, metadata: Dict[str, Any]) -> None:
|
||||
"""Сохранить информацию о контенте в базу данных."""
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
# Создать запись контента
|
||||
content = Content(
|
||||
filename=metadata.get("filename", file_path.name),
|
||||
original_filename=metadata.get("original_filename", file_path.name),
|
||||
file_path=str(file_path),
|
||||
file_size=file_path.stat().st_size,
|
||||
file_type=metadata.get("file_type", "unknown"),
|
||||
mime_type=metadata.get("mime_type", "application/octet-stream"),
|
||||
md5_hash=content_hash if len(content_hash) == 32 else None,
|
||||
sha256_hash=content_hash if len(content_hash) == 64 else None,
|
||||
is_active=True,
|
||||
processing_status="completed"
|
||||
)
|
||||
|
||||
session.add(content)
|
||||
await session.flush()
|
||||
|
||||
# Сохранить метаданные если есть
|
||||
if metadata.get("metadata"):
|
||||
content_metadata = ContentMetadata(
|
||||
content_id=content.id,
|
||||
**metadata["metadata"]
|
||||
)
|
||||
session.add(content_metadata)
|
||||
|
||||
await session.commit()
|
||||
logger.info(f"Saved content {content_hash} to database")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error saving content to database: {e}")
|
||||
|
||||
def _complete_sync(self, sync_status: ContentSyncStatus) -> None:
|
||||
"""Завершить синхронизацию."""
|
||||
sync_status.sync_completed = datetime.utcnow()
|
||||
|
||||
# Определить итоговый статус
|
||||
if len(sync_status.nodes_synced) == sync_status.total_nodes:
|
||||
sync_status.status = "completed"
|
||||
elif len(sync_status.nodes_synced) > 0:
|
||||
sync_status.status = "partial"
|
||||
else:
|
||||
sync_status.status = "failed"
|
||||
|
||||
# Переместить в историю
|
||||
self.sync_history.append(sync_status)
|
||||
del self.active_syncs[sync_status.content_hash]
|
||||
|
||||
# Ограничить историю
|
||||
if len(self.sync_history) > 100:
|
||||
self.sync_history = self.sync_history[-100:]
|
||||
|
||||
logger.info(f"Sync completed for {sync_status.content_hash}: {sync_status.status}")
|
||||
|
||||
async def sync_with_network(self) -> Dict[str, Any]:
|
||||
"""Синхронизация с сетью - обнаружение и загрузка нового контента."""
|
||||
try:
|
||||
from .node_service import get_node_service
|
||||
node_service = get_node_service()
|
||||
|
||||
connected_peers = node_service.peer_manager.get_connected_peers()
|
||||
if not connected_peers:
|
||||
return {"status": "no_peers", "message": "No connected peers for sync"}
|
||||
|
||||
# Получить списки контента от всех пиров
|
||||
network_content = {}
|
||||
for peer_id in connected_peers:
|
||||
try:
|
||||
peer_content = await self._get_peer_content_list(peer_id)
|
||||
network_content[peer_id] = peer_content
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting content list from {peer_id}: {e}")
|
||||
|
||||
# Найти новый контент для загрузки
|
||||
new_content = await self._identify_new_content(network_content)
|
||||
|
||||
# Запустить загрузку нового контента
|
||||
download_tasks = []
|
||||
for content_hash, source_nodes in new_content.items():
|
||||
download_tasks.append(
|
||||
self.download_content_from_network(content_hash, source_nodes)
|
||||
)
|
||||
|
||||
if download_tasks:
|
||||
results = await asyncio.gather(*download_tasks, return_exceptions=True)
|
||||
successful_downloads = sum(1 for r in results if r is True)
|
||||
|
||||
return {
|
||||
"status": "sync_completed",
|
||||
"new_content_found": len(new_content),
|
||||
"downloads_queued": len(download_tasks),
|
||||
"immediate_successes": successful_downloads
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "up_to_date",
|
||||
"message": "No new content found"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in network sync: {e}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
async def _get_peer_content_list(self, peer_id: str) -> List[Dict[str, Any]]:
|
||||
"""Получить список контента от пира."""
|
||||
try:
|
||||
if not self.session:
|
||||
return []
|
||||
|
||||
from .node_service import get_node_service
|
||||
node_service = get_node_service()
|
||||
peer_info = node_service.peer_manager.get_peer_info(peer_id)
|
||||
|
||||
if not peer_info:
|
||||
return []
|
||||
|
||||
peer_address = node_service.peer_manager._parse_peer_address(peer_info["address"])
|
||||
if not peer_address:
|
||||
return []
|
||||
|
||||
content_list_url = f"{peer_address}/api/my/content/list"
|
||||
async with self.session.get(content_list_url) as response:
|
||||
if response.status == 200:
|
||||
data = await response.json()
|
||||
return data.get("content", [])
|
||||
else:
|
||||
logger.warning(f"Failed to get content list from {peer_id}: HTTP {response.status}")
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting content list from {peer_id}: {e}")
|
||||
return []
|
||||
|
||||
async def _identify_new_content(self, network_content: Dict[str, List[Dict[str, Any]]]) -> Dict[str, List[str]]:
|
||||
"""Определить новый контент для загрузки."""
|
||||
try:
|
||||
# Получить список локального контента
|
||||
local_hashes = await self._get_local_content_hashes()
|
||||
|
||||
# Найти новый контент
|
||||
new_content = {}
|
||||
|
||||
for peer_id, content_list in network_content.items():
|
||||
for content_info in content_list:
|
||||
content_hash = content_info.get("hash")
|
||||
if not content_hash:
|
||||
continue
|
||||
|
||||
# Проверить, есть ли у нас этот контент
|
||||
if content_hash not in local_hashes:
|
||||
if content_hash not in new_content:
|
||||
new_content[content_hash] = []
|
||||
new_content[content_hash].append(peer_id)
|
||||
|
||||
return new_content
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error identifying new content: {e}")
|
||||
return {}
|
||||
|
||||
async def _get_local_content_hashes(self) -> Set[str]:
|
||||
"""Получить множество хешей локального контента."""
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
stmt = select(Content.md5_hash, Content.sha256_hash).where(Content.is_active == True)
|
||||
result = await session.execute(stmt)
|
||||
|
||||
hashes = set()
|
||||
for row in result:
|
||||
if row.md5_hash:
|
||||
hashes.add(row.md5_hash)
|
||||
if row.sha256_hash:
|
||||
hashes.add(row.sha256_hash)
|
||||
|
||||
return hashes
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting local content hashes: {e}")
|
||||
return set()
|
||||
|
||||
async def get_sync_status(self) -> Dict[str, Any]:
|
||||
"""Получить статус синхронизации."""
|
||||
return {
|
||||
"is_running": self.is_running,
|
||||
"active_syncs": len(self.active_syncs),
|
||||
"queue_size": self.sync_queue.qsize(),
|
||||
"workers_count": len(self.sync_workers),
|
||||
"recent_syncs": [
|
||||
sync.to_dict() for sync in self.sync_history[-10:]
|
||||
],
|
||||
"current_syncs": {
|
||||
content_hash: sync.to_dict()
|
||||
for content_hash, sync in self.active_syncs.items()
|
||||
}
|
||||
}
|
||||
|
||||
async def get_content_sync_status(self, content_hash: str) -> Dict[str, Any]:
|
||||
"""Получить статус синхронизации конкретного контента."""
|
||||
# Проверить активные синхронизации
|
||||
if content_hash in self.active_syncs:
|
||||
return self.active_syncs[content_hash].to_dict()
|
||||
|
||||
# Проверить историю
|
||||
for sync in reversed(self.sync_history):
|
||||
if sync.content_hash == content_hash:
|
||||
return sync.to_dict()
|
||||
|
||||
return {
|
||||
"content_hash": content_hash,
|
||||
"status": "not_found",
|
||||
"message": "No sync information found for this content"
|
||||
}
|
||||
@@ -0,0 +1,571 @@
|
||||
"""
|
||||
Comprehensive security module with encryption, JWT tokens, password hashing, and access control.
|
||||
Provides secure file encryption, token management, and authentication utilities.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Any, Union
|
||||
from uuid import UUID
|
||||
|
||||
import bcrypt
|
||||
import jwt
|
||||
from cryptography.fernet import Fernet
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
|
||||
import base64
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
class SecurityManager:
|
||||
"""Main security manager for encryption, tokens, and authentication."""
|
||||
|
||||
def __init__(self):
|
||||
self.fernet_key = self._get_or_create_fernet_key()
|
||||
self.fernet = Fernet(self.fernet_key)
|
||||
|
||||
def _get_or_create_fernet_key(self) -> bytes:
|
||||
"""Get or create Fernet encryption key from settings."""
|
||||
if hasattr(settings, 'ENCRYPTION_KEY') and settings.ENCRYPTION_KEY:
|
||||
# Derive key from settings
|
||||
kdf = PBKDF2HMAC(
|
||||
algorithm=hashes.SHA256(),
|
||||
length=32,
|
||||
salt=settings.SECRET_KEY.encode()[:16],
|
||||
iterations=100000,
|
||||
)
|
||||
key = base64.urlsafe_b64encode(kdf.derive(settings.ENCRYPTION_KEY.encode()))
|
||||
return key
|
||||
else:
|
||||
# Generate random key (for development only)
|
||||
return Fernet.generate_key()
|
||||
|
||||
# Global security manager instance
|
||||
_security_manager = SecurityManager()
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""
|
||||
Hash password using bcrypt with salt.
|
||||
|
||||
Args:
|
||||
password: Plain text password
|
||||
|
||||
Returns:
|
||||
str: Hashed password
|
||||
"""
|
||||
try:
|
||||
salt = bcrypt.gensalt(rounds=12)
|
||||
hashed = bcrypt.hashpw(password.encode('utf-8'), salt)
|
||||
return hashed.decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error("Failed to hash password", error=str(e))
|
||||
raise
|
||||
|
||||
def verify_password(password: str, hashed_password: str) -> bool:
|
||||
"""
|
||||
Verify password against hash.
|
||||
|
||||
Args:
|
||||
password: Plain text password
|
||||
hashed_password: Bcrypt hashed password
|
||||
|
||||
Returns:
|
||||
bool: True if password matches
|
||||
"""
|
||||
try:
|
||||
return bcrypt.checkpw(password.encode('utf-8'), hashed_password.encode('utf-8'))
|
||||
except Exception as e:
|
||||
logger.error("Failed to verify password", error=str(e))
|
||||
return False
|
||||
|
||||
def generate_access_token(
|
||||
payload: Dict[str, Any],
|
||||
expires_in: int = 3600,
|
||||
token_type: str = "access"
|
||||
) -> str:
|
||||
"""
|
||||
Generate JWT access token.
|
||||
|
||||
Args:
|
||||
payload: Token payload data
|
||||
expires_in: Token expiration time in seconds
|
||||
token_type: Type of token (access, refresh, api)
|
||||
|
||||
Returns:
|
||||
str: JWT token
|
||||
"""
|
||||
try:
|
||||
now = datetime.utcnow()
|
||||
token_payload = {
|
||||
"iat": now,
|
||||
"exp": now + timedelta(seconds=expires_in),
|
||||
"type": token_type,
|
||||
"jti": secrets.token_urlsafe(16), # Unique token ID
|
||||
**payload
|
||||
}
|
||||
|
||||
token = jwt.encode(
|
||||
token_payload,
|
||||
settings.SECRET_KEY,
|
||||
algorithm="HS256"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Access token generated",
|
||||
token_type=token_type,
|
||||
expires_in=expires_in,
|
||||
user_id=payload.get("user_id")
|
||||
)
|
||||
|
||||
return token
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to generate access token", error=str(e))
|
||||
raise
|
||||
|
||||
def verify_access_token(token: str, token_type: str = "access") -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Verify and decode JWT token.
|
||||
|
||||
Args:
|
||||
token: JWT token string
|
||||
token_type: Expected token type
|
||||
|
||||
Returns:
|
||||
Optional[Dict]: Decoded payload or None if invalid
|
||||
"""
|
||||
try:
|
||||
payload = jwt.decode(
|
||||
token,
|
||||
settings.SECRET_KEY,
|
||||
algorithms=["HS256"]
|
||||
)
|
||||
|
||||
# Verify token type
|
||||
if payload.get("type") != token_type:
|
||||
logger.warning("Token type mismatch", expected=token_type, actual=payload.get("type"))
|
||||
return None
|
||||
|
||||
# Check expiration
|
||||
if datetime.utcnow() > datetime.fromtimestamp(payload["exp"]):
|
||||
logger.warning("Token expired", exp=payload["exp"])
|
||||
return None
|
||||
|
||||
return payload
|
||||
|
||||
except jwt.ExpiredSignatureError:
|
||||
logger.warning("Token expired")
|
||||
return None
|
||||
except jwt.InvalidTokenError as e:
|
||||
logger.warning("Invalid token", error=str(e))
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error("Failed to verify token", error=str(e))
|
||||
return None
|
||||
|
||||
def generate_refresh_token(user_id: UUID, device_id: Optional[str] = None) -> str:
|
||||
"""
|
||||
Generate long-lived refresh token.
|
||||
|
||||
Args:
|
||||
user_id: User UUID
|
||||
device_id: Optional device identifier
|
||||
|
||||
Returns:
|
||||
str: Refresh token
|
||||
"""
|
||||
payload = {
|
||||
"user_id": str(user_id),
|
||||
"device_id": device_id,
|
||||
"token_family": secrets.token_urlsafe(16) # For token rotation
|
||||
}
|
||||
|
||||
return generate_access_token(
|
||||
payload,
|
||||
expires_in=settings.REFRESH_TOKEN_EXPIRE_DAYS * 24 * 3600,
|
||||
token_type="refresh"
|
||||
)
|
||||
|
||||
def generate_api_key(
|
||||
user_id: UUID,
|
||||
permissions: List[str],
|
||||
name: str,
|
||||
expires_in: Optional[int] = None
|
||||
) -> str:
|
||||
"""
|
||||
Generate API key with specific permissions.
|
||||
|
||||
Args:
|
||||
user_id: User UUID
|
||||
permissions: List of permissions
|
||||
name: API key name
|
||||
expires_in: Optional expiration time in seconds
|
||||
|
||||
Returns:
|
||||
str: API key token
|
||||
"""
|
||||
payload = {
|
||||
"user_id": str(user_id),
|
||||
"permissions": permissions,
|
||||
"name": name,
|
||||
"key_id": secrets.token_urlsafe(16)
|
||||
}
|
||||
|
||||
expires = expires_in or (365 * 24 * 3600) # Default 1 year
|
||||
|
||||
return generate_access_token(payload, expires_in=expires, token_type="api")
|
||||
|
||||
def encrypt_data(data: Union[str, bytes], context: str = "") -> str:
|
||||
"""
|
||||
Encrypt data using Fernet symmetric encryption.
|
||||
|
||||
Args:
|
||||
data: Data to encrypt
|
||||
context: Optional context for additional security
|
||||
|
||||
Returns:
|
||||
str: Base64 encoded encrypted data
|
||||
"""
|
||||
try:
|
||||
if isinstance(data, str):
|
||||
data = data.encode('utf-8')
|
||||
|
||||
# Add context to data for additional security
|
||||
if context:
|
||||
data = f"{context}:{len(data)}:".encode('utf-8') + data
|
||||
|
||||
encrypted = _security_manager.fernet.encrypt(data)
|
||||
return base64.urlsafe_b64encode(encrypted).decode('utf-8')
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to encrypt data", error=str(e))
|
||||
raise
|
||||
|
||||
def decrypt_data(encrypted_data: str, context: str = "") -> Union[str, bytes]:
|
||||
"""
|
||||
Decrypt data using Fernet symmetric encryption.
|
||||
|
||||
Args:
|
||||
encrypted_data: Base64 encoded encrypted data
|
||||
context: Optional context for verification
|
||||
|
||||
Returns:
|
||||
Union[str, bytes]: Decrypted data
|
||||
"""
|
||||
try:
|
||||
encrypted_bytes = base64.urlsafe_b64decode(encrypted_data.encode('utf-8'))
|
||||
decrypted = _security_manager.fernet.decrypt(encrypted_bytes)
|
||||
|
||||
# Verify and remove context if provided
|
||||
if context:
|
||||
context_prefix = f"{context}:".encode('utf-8')
|
||||
if not decrypted.startswith(context_prefix):
|
||||
raise ValueError("Context mismatch during decryption")
|
||||
|
||||
# Extract length and data
|
||||
remaining = decrypted[len(context_prefix):]
|
||||
length_end = remaining.find(b':')
|
||||
if length_end == -1:
|
||||
raise ValueError("Invalid encrypted data format")
|
||||
|
||||
expected_length = int(remaining[:length_end].decode('utf-8'))
|
||||
data = remaining[length_end + 1:]
|
||||
|
||||
if len(data) != expected_length:
|
||||
raise ValueError("Data length mismatch")
|
||||
|
||||
return data
|
||||
|
||||
return decrypted
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to decrypt data", error=str(e))
|
||||
raise
|
||||
|
||||
def encrypt_file(file_data: bytes, file_id: str) -> bytes:
|
||||
"""
|
||||
Encrypt file data with file-specific context.
|
||||
|
||||
Args:
|
||||
file_data: File bytes to encrypt
|
||||
file_id: Unique file identifier
|
||||
|
||||
Returns:
|
||||
bytes: Encrypted file data
|
||||
"""
|
||||
try:
|
||||
encrypted_str = encrypt_data(file_data, context=f"file:{file_id}")
|
||||
return base64.urlsafe_b64decode(encrypted_str.encode('utf-8'))
|
||||
except Exception as e:
|
||||
logger.error("Failed to encrypt file", file_id=file_id, error=str(e))
|
||||
raise
|
||||
|
||||
def decrypt_file(encrypted_data: bytes, file_id: str) -> bytes:
|
||||
"""
|
||||
Decrypt file data with file-specific context.
|
||||
|
||||
Args:
|
||||
encrypted_data: Encrypted file bytes
|
||||
file_id: Unique file identifier
|
||||
|
||||
Returns:
|
||||
bytes: Decrypted file data
|
||||
"""
|
||||
try:
|
||||
encrypted_str = base64.urlsafe_b64encode(encrypted_data).decode('utf-8')
|
||||
decrypted = decrypt_data(encrypted_str, context=f"file:{file_id}")
|
||||
return decrypted if isinstance(decrypted, bytes) else decrypted.encode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error("Failed to decrypt file", file_id=file_id, error=str(e))
|
||||
raise
|
||||
|
||||
def generate_secure_filename(original_filename: str, user_id: UUID) -> str:
|
||||
"""
|
||||
Generate secure filename to prevent path traversal and collisions.
|
||||
|
||||
Args:
|
||||
original_filename: Original filename
|
||||
user_id: User UUID
|
||||
|
||||
Returns:
|
||||
str: Secure filename
|
||||
"""
|
||||
# Extract extension
|
||||
parts = original_filename.rsplit('.', 1)
|
||||
extension = parts[1] if len(parts) > 1 else ''
|
||||
|
||||
# Generate secure base name
|
||||
timestamp = datetime.utcnow().strftime('%Y%m%d_%H%M%S')
|
||||
random_part = secrets.token_urlsafe(8)
|
||||
user_hash = hashlib.sha256(str(user_id).encode()).hexdigest()[:8]
|
||||
|
||||
secure_name = f"{timestamp}_{user_hash}_{random_part}"
|
||||
|
||||
if extension:
|
||||
# Validate extension
|
||||
allowed_extensions = {
|
||||
'txt', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx',
|
||||
'jpg', 'jpeg', 'png', 'gif', 'bmp', 'webp', 'svg',
|
||||
'mp3', 'wav', 'flac', 'ogg', 'mp4', 'avi', 'mkv', 'webm',
|
||||
'zip', 'rar', '7z', 'tar', 'gz', 'json', 'xml', 'csv'
|
||||
}
|
||||
|
||||
clean_extension = extension.lower().strip()
|
||||
if clean_extension in allowed_extensions:
|
||||
secure_name += f".{clean_extension}"
|
||||
|
||||
return secure_name
|
||||
|
||||
def validate_file_signature(file_data: bytes, claimed_type: str) -> bool:
|
||||
"""
|
||||
Validate file signature against claimed MIME type.
|
||||
|
||||
Args:
|
||||
file_data: File bytes to validate
|
||||
claimed_type: Claimed MIME type
|
||||
|
||||
Returns:
|
||||
bool: True if signature matches type
|
||||
"""
|
||||
if len(file_data) < 8:
|
||||
return False
|
||||
|
||||
# File signatures (magic numbers)
|
||||
signatures = {
|
||||
'image/jpeg': [b'\xFF\xD8\xFF'],
|
||||
'image/png': [b'\x89PNG\r\n\x1a\n'],
|
||||
'image/gif': [b'GIF87a', b'GIF89a'],
|
||||
'image/webp': [b'RIFF', b'WEBP'],
|
||||
'application/pdf': [b'%PDF-'],
|
||||
'application/zip': [b'PK\x03\x04', b'PK\x05\x06', b'PK\x07\x08'],
|
||||
'audio/mpeg': [b'ID3', b'\xFF\xFB', b'\xFF\xF3', b'\xFF\xF2'],
|
||||
'video/mp4': [b'\x00\x00\x00\x18ftypmp4', b'\x00\x00\x00\x20ftypmp4'],
|
||||
'text/plain': [], # Text files don't have reliable signatures
|
||||
}
|
||||
|
||||
expected_sigs = signatures.get(claimed_type, [])
|
||||
|
||||
# If no signatures defined, allow (like text files)
|
||||
if not expected_sigs:
|
||||
return True
|
||||
|
||||
# Check if file starts with any expected signature
|
||||
file_start = file_data[:32] # Check first 32 bytes
|
||||
|
||||
for sig in expected_sigs:
|
||||
if file_start.startswith(sig):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def generate_csrf_token(user_id: UUID, session_id: str) -> str:
|
||||
"""
|
||||
Generate CSRF token for form protection.
|
||||
|
||||
Args:
|
||||
user_id: User UUID
|
||||
session_id: Session identifier
|
||||
|
||||
Returns:
|
||||
str: CSRF token
|
||||
"""
|
||||
timestamp = str(int(datetime.utcnow().timestamp()))
|
||||
data = f"{user_id}:{session_id}:{timestamp}"
|
||||
|
||||
signature = hmac.new(
|
||||
settings.SECRET_KEY.encode(),
|
||||
data.encode(),
|
||||
hashlib.sha256
|
||||
).hexdigest()
|
||||
|
||||
token_data = f"{data}:{signature}"
|
||||
return base64.urlsafe_b64encode(token_data.encode()).decode()
|
||||
|
||||
def verify_csrf_token(token: str, user_id: UUID, session_id: str, max_age: int = 3600) -> bool:
|
||||
"""
|
||||
Verify CSRF token.
|
||||
|
||||
Args:
|
||||
token: CSRF token to verify
|
||||
user_id: User UUID
|
||||
session_id: Session identifier
|
||||
max_age: Maximum token age in seconds
|
||||
|
||||
Returns:
|
||||
bool: True if token is valid
|
||||
"""
|
||||
try:
|
||||
token_data = base64.urlsafe_b64decode(token.encode()).decode()
|
||||
parts = token_data.split(':')
|
||||
|
||||
if len(parts) != 4:
|
||||
return False
|
||||
|
||||
token_user_id, token_session_id, timestamp, signature = parts
|
||||
|
||||
# Verify components
|
||||
if token_user_id != str(user_id) or token_session_id != session_id:
|
||||
return False
|
||||
|
||||
# Check age
|
||||
token_time = int(timestamp)
|
||||
current_time = int(datetime.utcnow().timestamp())
|
||||
|
||||
if current_time - token_time > max_age:
|
||||
return False
|
||||
|
||||
# Verify signature
|
||||
data = f"{token_user_id}:{token_session_id}:{timestamp}"
|
||||
expected_signature = hmac.new(
|
||||
settings.SECRET_KEY.encode(),
|
||||
data.encode(),
|
||||
hashlib.sha256
|
||||
).hexdigest()
|
||||
|
||||
return hmac.compare_digest(signature, expected_signature)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Failed to verify CSRF token", error=str(e))
|
||||
return False
|
||||
|
||||
def sanitize_input(input_data: str, max_length: int = 1000) -> str:
|
||||
"""
|
||||
Sanitize user input to prevent XSS and injection attacks.
|
||||
|
||||
Args:
|
||||
input_data: Input string to sanitize
|
||||
max_length: Maximum allowed length
|
||||
|
||||
Returns:
|
||||
str: Sanitized input
|
||||
"""
|
||||
if not input_data:
|
||||
return ""
|
||||
|
||||
# Truncate if too long
|
||||
if len(input_data) > max_length:
|
||||
input_data = input_data[:max_length]
|
||||
|
||||
# Remove/escape dangerous characters
|
||||
dangerous_chars = ['<', '>', '"', "'", '&', '\x00', '\r', '\n']
|
||||
|
||||
for char in dangerous_chars:
|
||||
if char in input_data:
|
||||
input_data = input_data.replace(char, '')
|
||||
|
||||
# Strip whitespace
|
||||
return input_data.strip()
|
||||
|
||||
def check_permission(user_permissions: List[str], required_permission: str) -> bool:
|
||||
"""
|
||||
Check if user has required permission.
|
||||
|
||||
Args:
|
||||
user_permissions: List of user permissions
|
||||
required_permission: Required permission string
|
||||
|
||||
Returns:
|
||||
bool: True if user has permission
|
||||
"""
|
||||
# Admin has all permissions
|
||||
if 'admin' in user_permissions:
|
||||
return True
|
||||
|
||||
# Check exact permission
|
||||
if required_permission in user_permissions:
|
||||
return True
|
||||
|
||||
# Check wildcard permissions
|
||||
permission_parts = required_permission.split('.')
|
||||
for i in range(len(permission_parts)):
|
||||
wildcard_perm = '.'.join(permission_parts[:i+1]) + '.*'
|
||||
if wildcard_perm in user_permissions:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def rate_limit_key(identifier: str, action: str, window: str = "default") -> str:
|
||||
"""
|
||||
Generate rate limiting key.
|
||||
|
||||
Args:
|
||||
identifier: User/IP identifier
|
||||
action: Action being rate limited
|
||||
window: Time window identifier
|
||||
|
||||
Returns:
|
||||
str: Rate limit cache key
|
||||
"""
|
||||
key_data = f"rate_limit:{action}:{window}:{identifier}"
|
||||
return hashlib.sha256(key_data.encode()).hexdigest()
|
||||
|
||||
def generate_otp(length: int = 6) -> str:
|
||||
"""
|
||||
Generate one-time password.
|
||||
|
||||
Args:
|
||||
length: Length of OTP
|
||||
|
||||
Returns:
|
||||
str: Numeric OTP
|
||||
"""
|
||||
return ''.join(secrets.choice('0123456789') for _ in range(length))
|
||||
|
||||
def constant_time_compare(a: str, b: str) -> bool:
|
||||
"""
|
||||
Constant time string comparison to prevent timing attacks.
|
||||
|
||||
Args:
|
||||
a: First string
|
||||
b: Second string
|
||||
|
||||
Returns:
|
||||
bool: True if strings are equal
|
||||
"""
|
||||
return hmac.compare_digest(a.encode('utf-8'), b.encode('utf-8'))
|
||||
+565
-36
@@ -1,45 +1,574 @@
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
"""
|
||||
Comprehensive storage management with chunked uploads, multiple backends, and security.
|
||||
Supports local storage, S3-compatible storage, and async operations with Redis caching.
|
||||
"""
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.sql import text
|
||||
import asyncio
|
||||
import hashlib
|
||||
import mimetypes
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, AsyncGenerator, Any, Tuple
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from app.core._config import MYSQL_URI, MYSQL_DATABASE
|
||||
from app.core.logger import make_log
|
||||
from sqlalchemy.pool import NullPool
|
||||
import aiofiles
|
||||
import aiofiles.os
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
engine = create_engine(MYSQL_URI, poolclass=NullPool) #, echo=True)
|
||||
Session = sessionmaker(bind=engine)
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session, get_cache_manager
|
||||
from app.core.logging import get_logger
|
||||
from app.core.models.content import Content, ContentChunk
|
||||
from app.core.security import encrypt_file, decrypt_file, generate_access_token
|
||||
|
||||
logger = get_logger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
database_initialized = False
|
||||
while not database_initialized:
|
||||
try:
|
||||
with Session() as session:
|
||||
databases_list = session.execute(text("SHOW DATABASES;"))
|
||||
databases_list = [row[0] for row in databases_list]
|
||||
make_log("SQL", 'Database list: ' + str(databases_list), level='debug')
|
||||
assert MYSQL_DATABASE in databases_list, 'Database not found'
|
||||
database_initialized = True
|
||||
except Exception as e:
|
||||
make_log("SQL", 'MariaDB is not ready yet: ' + str(e), level='debug')
|
||||
time.sleep(1)
|
||||
class StorageBackend:
|
||||
"""Abstract base class for storage backends."""
|
||||
|
||||
async def store_chunk(self, upload_id: UUID, chunk_index: int, data: bytes) -> str:
|
||||
"""Store a file chunk and return its identifier."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def retrieve_chunk(self, chunk_id: str) -> bytes:
|
||||
"""Retrieve a file chunk by its identifier."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def delete_chunk(self, chunk_id: str) -> bool:
|
||||
"""Delete a file chunk."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def assemble_file(self, upload_id: UUID, chunks: List[str]) -> str:
|
||||
"""Assemble chunks into final file and return file path."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def delete_file(self, file_path: str) -> bool:
|
||||
"""Delete a complete file."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_file_stream(self, file_path: str) -> AsyncGenerator[bytes, None]:
|
||||
"""Get async file stream for download."""
|
||||
raise NotImplementedError
|
||||
|
||||
engine = create_engine(f"{MYSQL_URI}/{MYSQL_DATABASE}", poolclass=NullPool)
|
||||
Session = sessionmaker(bind=engine)
|
||||
class LocalStorageBackend(StorageBackend):
|
||||
"""Local filesystem storage backend with encryption support."""
|
||||
|
||||
def __init__(self):
|
||||
self.base_path = Path(settings.STORAGE_PATH)
|
||||
self.chunks_path = self.base_path / "chunks"
|
||||
self.files_path = self.base_path / "files"
|
||||
|
||||
# Create directories if they don't exist
|
||||
self.chunks_path.mkdir(parents=True, exist_ok=True)
|
||||
self.files_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async def store_chunk(self, upload_id: UUID, chunk_index: int, data: bytes) -> str:
|
||||
"""Store chunk to local filesystem with optional encryption."""
|
||||
try:
|
||||
chunk_id = f"{upload_id}_{chunk_index:06d}"
|
||||
chunk_path = self.chunks_path / f"{chunk_id}.chunk"
|
||||
|
||||
# Encrypt chunk if encryption is enabled
|
||||
if settings.ENCRYPT_FILES:
|
||||
data = encrypt_file(data, str(upload_id))
|
||||
|
||||
async with aiofiles.open(chunk_path, 'wb') as f:
|
||||
await f.write(data)
|
||||
|
||||
await logger.adebug(
|
||||
"Chunk stored successfully",
|
||||
upload_id=str(upload_id),
|
||||
chunk_index=chunk_index,
|
||||
chunk_size=len(data)
|
||||
)
|
||||
|
||||
return chunk_id
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to store chunk",
|
||||
upload_id=str(upload_id),
|
||||
chunk_index=chunk_index,
|
||||
error=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
async def retrieve_chunk(self, chunk_id: str) -> bytes:
|
||||
"""Retrieve and optionally decrypt chunk from local filesystem."""
|
||||
try:
|
||||
chunk_path = self.chunks_path / f"{chunk_id}.chunk"
|
||||
|
||||
if not chunk_path.exists():
|
||||
raise FileNotFoundError(f"Chunk {chunk_id} not found")
|
||||
|
||||
async with aiofiles.open(chunk_path, 'rb') as f:
|
||||
data = await f.read()
|
||||
|
||||
# Decrypt chunk if encryption is enabled
|
||||
if settings.ENCRYPT_FILES:
|
||||
upload_id = chunk_id.split('_')[0]
|
||||
data = decrypt_file(data, upload_id)
|
||||
|
||||
return data
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Failed to retrieve chunk", chunk_id=chunk_id, error=str(e))
|
||||
raise
|
||||
|
||||
async def delete_chunk(self, chunk_id: str) -> bool:
|
||||
"""Delete chunk file from local filesystem."""
|
||||
try:
|
||||
chunk_path = self.chunks_path / f"{chunk_id}.chunk"
|
||||
|
||||
if chunk_path.exists():
|
||||
await aiofiles.os.remove(chunk_path)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Failed to delete chunk", chunk_id=chunk_id, error=str(e))
|
||||
return False
|
||||
|
||||
async def assemble_file(self, upload_id: UUID, chunks: List[str]) -> str:
|
||||
"""Assemble chunks into final file."""
|
||||
try:
|
||||
file_id = str(uuid4())
|
||||
file_path = self.files_path / f"{file_id}"
|
||||
|
||||
async with aiofiles.open(file_path, 'wb') as output_file:
|
||||
for chunk_id in chunks:
|
||||
chunk_data = await self.retrieve_chunk(chunk_id)
|
||||
await output_file.write(chunk_data)
|
||||
|
||||
# Clean up chunks after assembly
|
||||
for chunk_id in chunks:
|
||||
await self.delete_chunk(chunk_id)
|
||||
|
||||
await logger.ainfo(
|
||||
"File assembled successfully",
|
||||
upload_id=str(upload_id),
|
||||
file_path=str(file_path),
|
||||
chunks_count=len(chunks)
|
||||
)
|
||||
|
||||
return str(file_path)
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to assemble file",
|
||||
upload_id=str(upload_id),
|
||||
error=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
async def delete_file(self, file_path: str) -> bool:
|
||||
"""Delete file from local filesystem."""
|
||||
try:
|
||||
path = Path(file_path)
|
||||
|
||||
if path.exists() and path.is_file():
|
||||
await aiofiles.os.remove(path)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Failed to delete file", file_path=file_path, error=str(e))
|
||||
return False
|
||||
|
||||
async def get_file_stream(self, file_path: str) -> AsyncGenerator[bytes, None]:
|
||||
"""Stream file content for download."""
|
||||
try:
|
||||
path = Path(file_path)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"File {file_path} not found")
|
||||
|
||||
async with aiofiles.open(path, 'rb') as f:
|
||||
while True:
|
||||
chunk = await f.read(65536) # 64KB chunks
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Failed to stream file", file_path=file_path, error=str(e))
|
||||
raise
|
||||
|
||||
class StorageManager:
|
||||
"""Main storage manager with upload session management and caching."""
|
||||
|
||||
def __init__(self):
|
||||
self.backend = LocalStorageBackend() # Can be extended to support S3, etc.
|
||||
self.cache_manager = get_cache_manager()
|
||||
|
||||
async def create_upload_session(self, content_id: UUID, total_size: int) -> Dict[str, Any]:
|
||||
"""Create new upload session with chunked upload support."""
|
||||
try:
|
||||
upload_id = uuid4()
|
||||
session_data = {
|
||||
"upload_id": str(upload_id),
|
||||
"content_id": str(content_id),
|
||||
"total_size": total_size,
|
||||
"chunk_size": settings.CHUNK_SIZE,
|
||||
"total_chunks": (total_size + settings.CHUNK_SIZE - 1) // settings.CHUNK_SIZE,
|
||||
"uploaded_chunks": [],
|
||||
"created_at": datetime.utcnow().isoformat(),
|
||||
"expires_at": (datetime.utcnow() + timedelta(hours=24)).isoformat(),
|
||||
"status": "active"
|
||||
}
|
||||
|
||||
# Store session in cache
|
||||
session_key = f"upload_session:{upload_id}"
|
||||
await self.cache_manager.set(session_key, session_data, ttl=86400) # 24 hours
|
||||
|
||||
# Store in database for persistence
|
||||
async with get_async_session() as session:
|
||||
upload_session = ContentUploadSession(
|
||||
id=upload_id,
|
||||
content_id=content_id,
|
||||
total_size=total_size,
|
||||
chunk_size=settings.CHUNK_SIZE,
|
||||
total_chunks=session_data["total_chunks"],
|
||||
expires_at=datetime.fromisoformat(session_data["expires_at"])
|
||||
)
|
||||
session.add(upload_session)
|
||||
await session.commit()
|
||||
|
||||
await logger.ainfo(
|
||||
"Upload session created",
|
||||
upload_id=str(upload_id),
|
||||
content_id=str(content_id),
|
||||
total_size=total_size
|
||||
)
|
||||
|
||||
return {
|
||||
"upload_id": str(upload_id),
|
||||
"chunk_size": settings.CHUNK_SIZE,
|
||||
"total_chunks": session_data["total_chunks"],
|
||||
"upload_url": f"/api/v1/storage/upload/{upload_id}",
|
||||
"expires_at": session_data["expires_at"]
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to create upload session",
|
||||
content_id=str(content_id),
|
||||
error=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
async def upload_chunk(
|
||||
self,
|
||||
upload_id: UUID,
|
||||
chunk_index: int,
|
||||
chunk_data: bytes,
|
||||
chunk_hash: str
|
||||
) -> Dict[str, Any]:
|
||||
"""Upload and validate a file chunk."""
|
||||
try:
|
||||
# Verify chunk hash
|
||||
calculated_hash = hashlib.sha256(chunk_data).hexdigest()
|
||||
if calculated_hash != chunk_hash:
|
||||
raise ValueError("Chunk hash mismatch")
|
||||
|
||||
# Get upload session
|
||||
session_data = await self._get_upload_session(upload_id)
|
||||
if not session_data:
|
||||
raise ValueError("Upload session not found or expired")
|
||||
|
||||
# Check if chunk already uploaded
|
||||
if chunk_index in session_data.get("uploaded_chunks", []):
|
||||
return {"status": "already_uploaded", "chunk_index": chunk_index}
|
||||
|
||||
# Store chunk
|
||||
chunk_id = await self.backend.store_chunk(upload_id, chunk_index, chunk_data)
|
||||
|
||||
# Update session data
|
||||
session_data["uploaded_chunks"].append(chunk_index)
|
||||
session_data["uploaded_chunks"].sort()
|
||||
|
||||
session_key = f"upload_session:{upload_id}"
|
||||
await self.cache_manager.set(session_key, session_data, ttl=86400)
|
||||
|
||||
# Store chunk info in database
|
||||
async with get_async_session() as session:
|
||||
chunk_record = ContentChunk(
|
||||
upload_id=upload_id,
|
||||
chunk_index=chunk_index,
|
||||
chunk_id=chunk_id,
|
||||
chunk_hash=chunk_hash,
|
||||
chunk_size=len(chunk_data)
|
||||
)
|
||||
session.add(chunk_record)
|
||||
await session.commit()
|
||||
|
||||
await logger.adebug(
|
||||
"Chunk uploaded successfully",
|
||||
upload_id=str(upload_id),
|
||||
chunk_index=chunk_index,
|
||||
chunk_size=len(chunk_data)
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "uploaded",
|
||||
"chunk_index": chunk_index,
|
||||
"uploaded_chunks": len(session_data["uploaded_chunks"]),
|
||||
"total_chunks": session_data["total_chunks"]
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to upload chunk",
|
||||
upload_id=str(upload_id),
|
||||
chunk_index=chunk_index,
|
||||
error=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
async def finalize_upload(self, upload_id: UUID) -> Dict[str, Any]:
|
||||
"""Finalize upload by assembling chunks into final file."""
|
||||
try:
|
||||
# Get upload session
|
||||
session_data = await self._get_upload_session(upload_id)
|
||||
if not session_data:
|
||||
raise ValueError("Upload session not found")
|
||||
|
||||
# Verify all chunks are uploaded
|
||||
uploaded_chunks = session_data.get("uploaded_chunks", [])
|
||||
total_chunks = session_data["total_chunks"]
|
||||
|
||||
if len(uploaded_chunks) != total_chunks:
|
||||
missing_chunks = set(range(total_chunks)) - set(uploaded_chunks)
|
||||
raise ValueError(f"Missing chunks: {missing_chunks}")
|
||||
|
||||
# Get chunk IDs in order
|
||||
async with get_async_session() as session:
|
||||
stmt = (
|
||||
select(ContentChunk)
|
||||
.where(ContentChunk.upload_id == upload_id)
|
||||
.order_by(ContentChunk.chunk_index)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
chunks = result.scalars().all()
|
||||
|
||||
chunk_ids = [chunk.chunk_id for chunk in chunks]
|
||||
|
||||
# Assemble file
|
||||
file_path = await self.backend.assemble_file(upload_id, chunk_ids)
|
||||
|
||||
# Update content record
|
||||
async with get_async_session() as session:
|
||||
stmt = (
|
||||
update(Content)
|
||||
.where(Content.id == UUID(session_data["content_id"]))
|
||||
.values(
|
||||
file_path=file_path,
|
||||
status="completed",
|
||||
updated_at=datetime.utcnow()
|
||||
)
|
||||
)
|
||||
await session.execute(stmt)
|
||||
await session.commit()
|
||||
|
||||
# Clean up session
|
||||
session_key = f"upload_session:{upload_id}"
|
||||
await self.cache_manager.delete(session_key)
|
||||
|
||||
await logger.ainfo(
|
||||
"Upload finalized successfully",
|
||||
upload_id=str(upload_id),
|
||||
file_path=file_path,
|
||||
total_chunks=total_chunks
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"file_path": file_path,
|
||||
"content_id": session_data["content_id"]
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to finalize upload",
|
||||
upload_id=str(upload_id),
|
||||
error=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
async def get_file_stream(self, file_path: str) -> AsyncGenerator[bytes, None]:
|
||||
"""Get file stream for download with caching support."""
|
||||
try:
|
||||
# Check if file is cached
|
||||
cache_key = f"file_stream:{hashlib.md5(file_path.encode()).hexdigest()}"
|
||||
|
||||
async for chunk in self.backend.get_file_stream(file_path):
|
||||
yield chunk
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Failed to get file stream", file_path=file_path, error=str(e))
|
||||
raise
|
||||
|
||||
async def delete_content_files(self, content_id: UUID) -> bool:
|
||||
"""Delete all files associated with content."""
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
# Get content
|
||||
stmt = select(Content).where(Content.id == content_id)
|
||||
result = await session.execute(stmt)
|
||||
content = result.scalar_one_or_none()
|
||||
|
||||
if not content or not content.file_path:
|
||||
return True
|
||||
|
||||
# Delete main file
|
||||
await self.backend.delete_file(content.file_path)
|
||||
|
||||
# Delete any remaining chunks
|
||||
chunk_stmt = select(ContentChunk).where(
|
||||
ContentChunk.upload_id == content_id
|
||||
)
|
||||
chunk_result = await session.execute(chunk_stmt)
|
||||
chunks = chunk_result.scalars().all()
|
||||
|
||||
for chunk in chunks:
|
||||
await self.backend.delete_chunk(chunk.chunk_id)
|
||||
|
||||
# Update content record
|
||||
update_stmt = (
|
||||
update(Content)
|
||||
.where(Content.id == content_id)
|
||||
.values(file_path=None, status="deleted")
|
||||
)
|
||||
await session.execute(update_stmt)
|
||||
await session.commit()
|
||||
|
||||
await logger.ainfo(
|
||||
"Content files deleted",
|
||||
content_id=str(content_id)
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to delete content files",
|
||||
content_id=str(content_id),
|
||||
error=str(e)
|
||||
)
|
||||
return False
|
||||
|
||||
async def get_storage_stats(self) -> Dict[str, Any]:
|
||||
"""Get storage usage statistics."""
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
# Get total files and size
|
||||
from sqlalchemy import func
|
||||
stmt = select(
|
||||
func.count(Content.id).label('total_files'),
|
||||
func.sum(Content.file_size).label('total_size')
|
||||
).where(Content.status == 'completed')
|
||||
|
||||
result = await session.execute(stmt)
|
||||
stats = result.first()
|
||||
|
||||
# Get storage by type
|
||||
type_stmt = select(
|
||||
Content.content_type,
|
||||
func.count(Content.id).label('count'),
|
||||
func.sum(Content.file_size).label('size')
|
||||
).where(Content.status == 'completed').group_by(Content.content_type)
|
||||
|
||||
type_result = await session.execute(type_stmt)
|
||||
type_stats = {
|
||||
row.content_type: {
|
||||
'count': row.count,
|
||||
'size': row.size or 0
|
||||
}
|
||||
for row in type_result
|
||||
}
|
||||
|
||||
return {
|
||||
'total_files': stats.total_files or 0,
|
||||
'total_size': stats.total_size or 0,
|
||||
'by_type': type_stats,
|
||||
'updated_at': datetime.utcnow().isoformat()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror("Failed to get storage stats", error=str(e))
|
||||
return {}
|
||||
|
||||
async def _get_upload_session(self, upload_id: UUID) -> Optional[Dict[str, Any]]:
|
||||
"""Get upload session from cache or database."""
|
||||
# Try cache first
|
||||
session_key = f"upload_session:{upload_id}"
|
||||
session_data = await self.cache_manager.get(session_key)
|
||||
|
||||
if session_data:
|
||||
# Check if session is expired
|
||||
expires_at = datetime.fromisoformat(session_data["expires_at"])
|
||||
if expires_at > datetime.utcnow():
|
||||
return session_data
|
||||
|
||||
# Fallback to database
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
stmt = (
|
||||
select(ContentUploadSession)
|
||||
.where(ContentUploadSession.id == upload_id)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
upload_session = result.scalar_one_or_none()
|
||||
|
||||
if upload_session and upload_session.expires_at > datetime.utcnow():
|
||||
# Rebuild session data
|
||||
chunk_stmt = select(ContentChunk).where(
|
||||
ContentChunk.upload_id == upload_id
|
||||
)
|
||||
chunk_result = await session.execute(chunk_stmt)
|
||||
chunks = chunk_result.scalars().all()
|
||||
|
||||
session_data = {
|
||||
"upload_id": str(upload_session.id),
|
||||
"content_id": str(upload_session.content_id),
|
||||
"total_size": upload_session.total_size,
|
||||
"chunk_size": upload_session.chunk_size,
|
||||
"total_chunks": upload_session.total_chunks,
|
||||
"uploaded_chunks": [chunk.chunk_index for chunk in chunks],
|
||||
"created_at": upload_session.created_at.isoformat(),
|
||||
"expires_at": upload_session.expires_at.isoformat(),
|
||||
"status": "active"
|
||||
}
|
||||
|
||||
# Update cache
|
||||
await self.cache_manager.set(session_key, session_data, ttl=86400)
|
||||
return session_data
|
||||
|
||||
except Exception as e:
|
||||
await logger.aerror(
|
||||
"Failed to get upload session from database",
|
||||
upload_id=str(upload_id),
|
||||
error=str(e)
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@contextmanager
|
||||
def db_session(auto_commit=False):
|
||||
_session = Session()
|
||||
try:
|
||||
yield _session
|
||||
if auto_commit is True:
|
||||
_session.commit()
|
||||
except BaseException as e:
|
||||
_session.rollback()
|
||||
raise e
|
||||
finally:
|
||||
_session.close()
|
||||
# Additional model for upload sessions
|
||||
from app.core.models.base import Base
|
||||
from sqlalchemy import Column, Integer, DateTime
|
||||
|
||||
class ContentUploadSession(Base):
|
||||
"""Model for tracking upload sessions."""
|
||||
__tablename__ = "content_upload_sessions"
|
||||
|
||||
content_id = Column("content_id", sa.UUID(as_uuid=True), nullable=False)
|
||||
total_size = Column(Integer, nullable=False)
|
||||
chunk_size = Column(Integer, nullable=False, default=1048576) # 1MB
|
||||
total_chunks = Column(Integer, nullable=False)
|
||||
expires_at = Column(DateTime, nullable=False)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
@@ -0,0 +1,371 @@
|
||||
"""
|
||||
Comprehensive validation schemas using Pydantic for request/response validation.
|
||||
Provides type safety, data validation, and automatic documentation generation.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any, Union
|
||||
from uuid import UUID
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, Field, validator, root_validator
|
||||
from pydantic.networks import EmailStr, HttpUrl
|
||||
|
||||
class ContentTypeEnum(str, Enum):
|
||||
"""Supported content types."""
|
||||
AUDIO = "audio"
|
||||
VIDEO = "video"
|
||||
IMAGE = "image"
|
||||
DOCUMENT = "document"
|
||||
ARCHIVE = "archive"
|
||||
OTHER = "other"
|
||||
|
||||
class VisibilityEnum(str, Enum):
|
||||
"""Content visibility levels."""
|
||||
PUBLIC = "public"
|
||||
PRIVATE = "private"
|
||||
UNLISTED = "unlisted"
|
||||
RESTRICTED = "restricted"
|
||||
|
||||
class StatusEnum(str, Enum):
|
||||
"""Content processing status."""
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
DELETED = "deleted"
|
||||
|
||||
class PermissionEnum(str, Enum):
|
||||
"""User permissions."""
|
||||
READ = "read"
|
||||
WRITE = "write"
|
||||
DELETE = "delete"
|
||||
ADMIN = "admin"
|
||||
|
||||
class BaseSchema(BaseModel):
|
||||
"""Base schema with common configuration."""
|
||||
|
||||
class Config:
|
||||
use_enum_values = True
|
||||
validate_assignment = True
|
||||
allow_population_by_field_name = True
|
||||
json_encoders = {
|
||||
datetime: lambda v: v.isoformat(),
|
||||
UUID: lambda v: str(v)
|
||||
}
|
||||
|
||||
class ContentSchema(BaseSchema):
|
||||
"""Schema for content creation."""
|
||||
title: str = Field(..., min_length=1, max_length=255, description="Content title")
|
||||
description: Optional[str] = Field(None, max_length=2000, description="Content description")
|
||||
content_type: ContentTypeEnum = Field(..., description="Type of content")
|
||||
file_size: Optional[int] = Field(None, ge=0, le=10737418240, description="File size in bytes (max 10GB)")
|
||||
visibility: VisibilityEnum = Field(VisibilityEnum.PRIVATE, description="Content visibility")
|
||||
tags: List[str] = Field(default_factory=list, max_items=20, description="Content tags")
|
||||
license_id: Optional[UUID] = Field(None, description="License ID if applicable")
|
||||
metadata: Optional[Dict[str, Any]] = Field(None, description="Additional metadata")
|
||||
|
||||
@validator('tags')
|
||||
def validate_tags(cls, v):
|
||||
"""Validate tags format and content."""
|
||||
if not v:
|
||||
return v
|
||||
|
||||
# Check each tag
|
||||
for tag in v:
|
||||
if not isinstance(tag, str):
|
||||
raise ValueError("Tags must be strings")
|
||||
if len(tag) < 1 or len(tag) > 50:
|
||||
raise ValueError("Tag length must be between 1 and 50 characters")
|
||||
if not tag.replace('-', '').replace('_', '').isalnum():
|
||||
raise ValueError("Tags can only contain alphanumeric characters, hyphens, and underscores")
|
||||
|
||||
# Remove duplicates while preserving order
|
||||
seen = set()
|
||||
unique_tags = []
|
||||
for tag in v:
|
||||
tag_lower = tag.lower()
|
||||
if tag_lower not in seen:
|
||||
seen.add(tag_lower)
|
||||
unique_tags.append(tag)
|
||||
|
||||
return unique_tags
|
||||
|
||||
@validator('metadata')
|
||||
def validate_metadata(cls, v):
|
||||
"""Validate metadata structure."""
|
||||
if not v:
|
||||
return v
|
||||
|
||||
# Check metadata size (JSON serialized)
|
||||
import json
|
||||
try:
|
||||
serialized = json.dumps(v)
|
||||
if len(serialized) > 10000: # Max 10KB of metadata
|
||||
raise ValueError("Metadata too large (max 10KB)")
|
||||
except (TypeError, ValueError) as e:
|
||||
raise ValueError(f"Invalid metadata format: {e}")
|
||||
|
||||
return v
|
||||
|
||||
class ContentUpdateSchema(BaseSchema):
|
||||
"""Schema for content updates."""
|
||||
title: Optional[str] = Field(None, min_length=1, max_length=255)
|
||||
description: Optional[str] = Field(None, max_length=2000)
|
||||
visibility: Optional[VisibilityEnum] = None
|
||||
tags: Optional[List[str]] = Field(None, max_items=20)
|
||||
license_id: Optional[UUID] = None
|
||||
status: Optional[StatusEnum] = None
|
||||
|
||||
@validator('tags')
|
||||
def validate_tags(cls, v):
|
||||
"""Validate tags if provided."""
|
||||
if v is None:
|
||||
return v
|
||||
return ContentSchema.validate_tags(v)
|
||||
|
||||
class ContentSearchSchema(BaseSchema):
|
||||
"""Schema for content search requests."""
|
||||
query: Optional[str] = Field(None, min_length=1, max_length=200, description="Search query")
|
||||
content_type: Optional[ContentTypeEnum] = None
|
||||
status: Optional[StatusEnum] = None
|
||||
tags: Optional[List[str]] = Field(None, max_items=10)
|
||||
visibility: Optional[VisibilityEnum] = None
|
||||
date_from: Optional[datetime] = None
|
||||
date_to: Optional[datetime] = None
|
||||
sort_by: Optional[str] = Field("updated_at", regex="^(created_at|updated_at|title|file_size)$")
|
||||
sort_order: Optional[str] = Field("desc", regex="^(asc|desc)$")
|
||||
page: int = Field(1, ge=1, le=1000)
|
||||
per_page: int = Field(20, ge=1, le=100)
|
||||
|
||||
@root_validator
|
||||
def validate_date_range(cls, values):
|
||||
"""Validate date range."""
|
||||
date_from = values.get('date_from')
|
||||
date_to = values.get('date_to')
|
||||
|
||||
if date_from and date_to and date_from >= date_to:
|
||||
raise ValueError("date_from must be before date_to")
|
||||
|
||||
return values
|
||||
|
||||
class UserRegistrationSchema(BaseSchema):
|
||||
"""Schema for user registration."""
|
||||
username: str = Field(..., min_length=3, max_length=50, regex="^[a-zA-Z0-9_.-]+$")
|
||||
email: EmailStr = Field(..., description="Valid email address")
|
||||
password: str = Field(..., min_length=8, max_length=128, description="Password (min 8 characters)")
|
||||
full_name: Optional[str] = Field(None, max_length=100)
|
||||
|
||||
@validator('password')
|
||||
def validate_password(cls, v):
|
||||
"""Validate password strength."""
|
||||
if len(v) < 8:
|
||||
raise ValueError("Password must be at least 8 characters long")
|
||||
|
||||
# Check for required character types
|
||||
has_upper = any(c.isupper() for c in v)
|
||||
has_lower = any(c.islower() for c in v)
|
||||
has_digit = any(c.isdigit() for c in v)
|
||||
has_special = any(c in "!@#$%^&*()_+-=[]{}|;:,.<>?" for c in v)
|
||||
|
||||
if not (has_upper and has_lower and has_digit and has_special):
|
||||
raise ValueError(
|
||||
"Password must contain at least one uppercase letter, "
|
||||
"one lowercase letter, one digit, and one special character"
|
||||
)
|
||||
|
||||
return v
|
||||
|
||||
class UserLoginSchema(BaseSchema):
|
||||
"""Schema for user login."""
|
||||
username: str = Field(..., min_length=1, max_length=50)
|
||||
password: str = Field(..., min_length=1, max_length=128)
|
||||
remember_me: bool = Field(False, description="Keep session longer")
|
||||
|
||||
class UserUpdateSchema(BaseSchema):
|
||||
"""Schema for user profile updates."""
|
||||
full_name: Optional[str] = Field(None, max_length=100)
|
||||
email: Optional[EmailStr] = None
|
||||
bio: Optional[str] = Field(None, max_length=500)
|
||||
avatar_url: Optional[HttpUrl] = None
|
||||
settings: Optional[Dict[str, Any]] = None
|
||||
|
||||
@validator('settings')
|
||||
def validate_settings(cls, v):
|
||||
"""Validate user settings."""
|
||||
if not v:
|
||||
return v
|
||||
|
||||
# Allowed settings keys
|
||||
allowed_keys = {
|
||||
'notifications', 'privacy', 'theme', 'language',
|
||||
'timezone', 'auto_save', 'quality_preference'
|
||||
}
|
||||
|
||||
for key in v.keys():
|
||||
if key not in allowed_keys:
|
||||
raise ValueError(f"Invalid settings key: {key}")
|
||||
|
||||
return v
|
||||
|
||||
class StorageUploadSchema(BaseSchema):
|
||||
"""Schema for file upload initiation."""
|
||||
filename: str = Field(..., min_length=1, max_length=255)
|
||||
file_size: int = Field(..., ge=1, le=10737418240) # Max 10GB
|
||||
content_type: str = Field(..., min_length=1, max_length=100)
|
||||
chunk_size: Optional[int] = Field(1048576, ge=65536, le=10485760) # 64KB to 10MB
|
||||
|
||||
@validator('filename')
|
||||
def validate_filename(cls, v):
|
||||
"""Validate filename format."""
|
||||
import re
|
||||
|
||||
# Check for dangerous characters
|
||||
if re.search(r'[<>:"/\\|?*\x00-\x1f]', v):
|
||||
raise ValueError("Filename contains invalid characters")
|
||||
|
||||
# Check for reserved names (Windows)
|
||||
reserved_names = {
|
||||
'CON', 'PRN', 'AUX', 'NUL',
|
||||
'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9',
|
||||
'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9'
|
||||
}
|
||||
|
||||
name_part = v.split('.')[0].upper()
|
||||
if name_part in reserved_names:
|
||||
raise ValueError("Filename uses reserved name")
|
||||
|
||||
return v
|
||||
|
||||
class ChunkUploadSchema(BaseSchema):
|
||||
"""Schema for chunk upload."""
|
||||
upload_id: UUID = Field(..., description="Upload session ID")
|
||||
chunk_index: int = Field(..., ge=0, description="Chunk sequence number")
|
||||
chunk_hash: str = Field(..., min_length=64, max_length=64, description="SHA256 hash of chunk")
|
||||
is_final: bool = Field(False, description="Is this the final chunk")
|
||||
|
||||
class BlockchainTransactionSchema(BaseSchema):
|
||||
"""Schema for blockchain transactions."""
|
||||
transaction_type: str = Field(..., regex="^(transfer|mint|burn|stake|unstake)$")
|
||||
amount: Optional[int] = Field(None, ge=0, description="Amount in nanotons")
|
||||
recipient_address: Optional[str] = Field(None, min_length=48, max_length=48)
|
||||
message: Optional[str] = Field(None, max_length=500)
|
||||
|
||||
@validator('recipient_address')
|
||||
def validate_ton_address(cls, v):
|
||||
"""Validate TON address format."""
|
||||
if not v:
|
||||
return v
|
||||
|
||||
# Basic TON address validation
|
||||
import re
|
||||
if not re.match(r'^[a-zA-Z0-9_-]{48}$', v):
|
||||
raise ValueError("Invalid TON address format")
|
||||
|
||||
return v
|
||||
|
||||
class LicenseSchema(BaseSchema):
|
||||
"""Schema for license information."""
|
||||
name: str = Field(..., min_length=1, max_length=100)
|
||||
description: Optional[str] = Field(None, max_length=1000)
|
||||
url: Optional[HttpUrl] = None
|
||||
commercial_use: bool = Field(False, description="Allows commercial use")
|
||||
attribution_required: bool = Field(True, description="Requires attribution")
|
||||
share_alike: bool = Field(False, description="Requires share-alike")
|
||||
|
||||
class AccessControlSchema(BaseSchema):
|
||||
"""Schema for content access control."""
|
||||
user_id: UUID = Field(..., description="User to grant access to")
|
||||
permission: str = Field(..., regex="^(read|write|delete|admin)$")
|
||||
expires_at: Optional[datetime] = Field(None, description="Access expiration time")
|
||||
|
||||
@root_validator
|
||||
def validate_expiration(cls, values):
|
||||
"""Validate access expiration."""
|
||||
expires_at = values.get('expires_at')
|
||||
|
||||
if expires_at and expires_at <= datetime.utcnow():
|
||||
raise ValueError("Expiration time must be in the future")
|
||||
|
||||
return values
|
||||
|
||||
class ApiKeySchema(BaseSchema):
|
||||
"""Schema for API key creation."""
|
||||
name: str = Field(..., min_length=1, max_length=100, description="API key name")
|
||||
permissions: List[str] = Field(..., min_items=1, description="List of permissions")
|
||||
expires_at: Optional[datetime] = Field(None, description="Key expiration time")
|
||||
|
||||
@validator('permissions')
|
||||
def validate_permissions(cls, v):
|
||||
"""Validate permission format."""
|
||||
valid_permissions = {
|
||||
'content.read', 'content.create', 'content.update', 'content.delete',
|
||||
'storage.upload', 'storage.download', 'storage.delete',
|
||||
'user.read', 'user.update', 'admin.read', 'admin.write'
|
||||
}
|
||||
|
||||
for perm in v:
|
||||
if perm not in valid_permissions:
|
||||
raise ValueError(f"Invalid permission: {perm}")
|
||||
|
||||
return list(set(v)) # Remove duplicates
|
||||
|
||||
class WebhookSchema(BaseSchema):
|
||||
"""Schema for webhook configuration."""
|
||||
url: HttpUrl = Field(..., description="Webhook endpoint URL")
|
||||
events: List[str] = Field(..., min_items=1, description="Events to subscribe to")
|
||||
secret: Optional[str] = Field(None, min_length=16, max_length=64, description="Webhook secret")
|
||||
active: bool = Field(True, description="Whether webhook is active")
|
||||
|
||||
@validator('events')
|
||||
def validate_events(cls, v):
|
||||
"""Validate webhook events."""
|
||||
valid_events = {
|
||||
'content.created', 'content.updated', 'content.deleted',
|
||||
'user.registered', 'user.updated', 'upload.completed',
|
||||
'blockchain.transaction', 'system.error'
|
||||
}
|
||||
|
||||
for event in v:
|
||||
if event not in valid_events:
|
||||
raise ValueError(f"Invalid event: {event}")
|
||||
|
||||
return list(set(v))
|
||||
|
||||
# Response schemas
|
||||
class ContentResponseSchema(BaseSchema):
|
||||
"""Schema for content response."""
|
||||
id: UUID
|
||||
title: str
|
||||
description: Optional[str]
|
||||
content_type: ContentTypeEnum
|
||||
file_size: int
|
||||
status: StatusEnum
|
||||
visibility: VisibilityEnum
|
||||
tags: List[str]
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
user_id: UUID
|
||||
|
||||
class UserResponseSchema(BaseSchema):
|
||||
"""Schema for user response."""
|
||||
id: UUID
|
||||
username: str
|
||||
email: EmailStr
|
||||
full_name: Optional[str]
|
||||
created_at: datetime
|
||||
is_active: bool
|
||||
permissions: List[str]
|
||||
|
||||
class ErrorResponseSchema(BaseSchema):
|
||||
"""Schema for error responses."""
|
||||
error: str = Field(..., description="Error message")
|
||||
code: str = Field(..., description="Error code")
|
||||
details: Optional[Dict[str, Any]] = Field(None, description="Additional error details")
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
class SuccessResponseSchema(BaseSchema):
|
||||
"""Schema for success responses."""
|
||||
message: str = Field(..., description="Success message")
|
||||
data: Optional[Dict[str, Any]] = Field(None, description="Response data")
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
"""
|
||||
MY Network - Main Application Entry Point
|
||||
Точка входа для приложения с поддержкой MY Network
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
# Настройка логирования
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Определить режим запуска
|
||||
def get_app_mode():
|
||||
"""Определить режим запуска приложения."""
|
||||
import os
|
||||
|
||||
# Проверить переменные окружения
|
||||
if os.getenv('USE_FASTAPI', '').lower() == 'true':
|
||||
return 'fastapi'
|
||||
|
||||
# Проверить наличие FastAPI зависимостей
|
||||
try:
|
||||
import fastapi
|
||||
import uvicorn
|
||||
return 'fastapi'
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Проверить наличие Sanic
|
||||
try:
|
||||
import sanic
|
||||
return 'sanic'
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Fallback к минимальному серверу
|
||||
return 'minimal'
|
||||
|
||||
|
||||
def create_fastapi_app():
|
||||
"""Создать FastAPI приложение с MY Network."""
|
||||
try:
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
app = FastAPI(
|
||||
title="MY Network",
|
||||
description="Distributed Content Protocol v2.0",
|
||||
version="2.0.0"
|
||||
)
|
||||
|
||||
# CORS middleware
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Добавить MY Network маршруты
|
||||
try:
|
||||
from app.api.routes.my_network_routes import router as my_network_router
|
||||
from app.api.routes.my_monitoring import router as monitoring_router
|
||||
|
||||
app.include_router(my_network_router)
|
||||
app.include_router(monitoring_router)
|
||||
|
||||
logger.info("MY Network routes added to FastAPI")
|
||||
|
||||
except ImportError as e:
|
||||
logger.warning(f"Could not import MY Network routes: {e}")
|
||||
|
||||
# Базовые маршруты
|
||||
@app.get("/")
|
||||
async def root():
|
||||
return {"message": "MY Network v2.0 - Distributed Content Protocol"}
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "healthy", "service": "MY Network"}
|
||||
|
||||
return app
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create FastAPI app: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def create_sanic_app():
|
||||
"""Создать Sanic приложение с MY Network."""
|
||||
try:
|
||||
# Импортировать существующее Sanic приложение
|
||||
from app.api import create_app
|
||||
return create_app()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create Sanic app: {e}")
|
||||
raise
|
||||
|
||||
|
||||
async def start_my_network_service():
|
||||
"""Запустить MY Network сервис."""
|
||||
try:
|
||||
from app.core.my_network.node_service import NodeService
|
||||
|
||||
logger.info("Starting MY Network service...")
|
||||
|
||||
# Создать и запустить сервис
|
||||
node_service = NodeService()
|
||||
await node_service.start()
|
||||
|
||||
logger.info("MY Network service started successfully")
|
||||
|
||||
# Вернуть задачу для поддержания сервиса
|
||||
async def keep_service_running():
|
||||
try:
|
||||
while node_service.is_running:
|
||||
await asyncio.sleep(30) # Проверять каждые 30 секунд
|
||||
except asyncio.CancelledError:
|
||||
logger.info("MY Network service shutdown requested")
|
||||
await node_service.stop()
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"MY Network service error: {e}")
|
||||
await node_service.stop()
|
||||
raise
|
||||
|
||||
return keep_service_running()
|
||||
|
||||
except ImportError as e:
|
||||
logger.info(f"MY Network service not available: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start MY Network service: {e}")
|
||||
return None
|
||||
|
||||
|
||||
async def run_fastapi_server():
|
||||
"""Запустить FastAPI сервер."""
|
||||
try:
|
||||
import uvicorn
|
||||
|
||||
# Создать приложение
|
||||
app = create_fastapi_app()
|
||||
|
||||
# Запустить MY Network сервис в фоне
|
||||
my_network_task = await start_my_network_service()
|
||||
|
||||
# Конфигурация сервера
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host="0.0.0.0",
|
||||
port=8000,
|
||||
log_level="info"
|
||||
)
|
||||
|
||||
server = uvicorn.Server(config)
|
||||
|
||||
# Запустить сервер и MY Network параллельно
|
||||
if my_network_task:
|
||||
await asyncio.gather(
|
||||
server.serve(),
|
||||
my_network_task,
|
||||
return_exceptions=True
|
||||
)
|
||||
else:
|
||||
await server.serve()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"FastAPI server error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
async def run_sanic_server():
|
||||
"""Запустить Sanic сервер."""
|
||||
try:
|
||||
# Создать приложение
|
||||
app = create_sanic_app()
|
||||
|
||||
# Запустить MY Network сервис в фоне
|
||||
my_network_task = await start_my_network_service()
|
||||
if my_network_task:
|
||||
app.add_background_task(my_network_task)
|
||||
|
||||
# Запустить сервер
|
||||
await app.create_server(
|
||||
host="0.0.0.0",
|
||||
port=8000,
|
||||
debug=False,
|
||||
access_log=True
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Sanic server error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
async def run_minimal_server():
|
||||
"""Запустить минимальный HTTP сервер."""
|
||||
try:
|
||||
from start_my_network import main as start_minimal
|
||||
|
||||
logger.info("Starting minimal MY Network server...")
|
||||
start_minimal()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Minimal server error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
async def main():
|
||||
"""Главная функция запуска."""
|
||||
|
||||
print("""
|
||||
╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
║ MY NETWORK v2.0 ║
|
||||
║ Distributed Content Protocol ║
|
||||
║ ║
|
||||
║ Starting application with MY Network integration... ║
|
||||
╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
""")
|
||||
|
||||
# Определить режим запуска
|
||||
app_mode = get_app_mode()
|
||||
logger.info(f"Application mode: {app_mode}")
|
||||
|
||||
try:
|
||||
if app_mode == 'fastapi':
|
||||
logger.info("Starting FastAPI server with MY Network...")
|
||||
await run_fastapi_server()
|
||||
|
||||
elif app_mode == 'sanic':
|
||||
logger.info("Starting Sanic server with MY Network...")
|
||||
await run_sanic_server()
|
||||
|
||||
else:
|
||||
logger.info("Starting minimal MY Network server...")
|
||||
await run_minimal_server()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Received keyboard interrupt, shutting down...")
|
||||
except Exception as e:
|
||||
logger.error(f"Application error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
# FastAPI app для ASGI серверов (uvicorn, gunicorn)
|
||||
try:
|
||||
app = create_fastapi_app()
|
||||
|
||||
# Добавить startup event для MY Network
|
||||
@app.on_event("startup")
|
||||
async def startup_event():
|
||||
"""Startup event для MY Network."""
|
||||
my_network_task = await start_my_network_service()
|
||||
if my_network_task:
|
||||
# Запустить как фоновую задачу
|
||||
import asyncio
|
||||
asyncio.create_task(my_network_task)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not create FastAPI app instance: {e}")
|
||||
app = None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Запуск через python app/main.py
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Script to create an admin user for the application."""
|
||||
|
||||
import asyncio
|
||||
import getpass
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.database import get_async_session
|
||||
from app.core.models.user import User
|
||||
from app.core.security import hash_password
|
||||
|
||||
|
||||
async def create_admin_user():
|
||||
"""Create an admin user interactively."""
|
||||
print("🔧 My Uploader Bot - Admin User Creation")
|
||||
print("=" * 50)
|
||||
|
||||
# Get user input
|
||||
username = input("Enter admin username: ").strip()
|
||||
if not username:
|
||||
print("❌ Username is required")
|
||||
sys.exit(1)
|
||||
|
||||
email = input("Enter admin email: ").strip()
|
||||
if not email:
|
||||
print("❌ Email is required")
|
||||
sys.exit(1)
|
||||
|
||||
password = getpass.getpass("Enter admin password: ").strip()
|
||||
if not password:
|
||||
print("❌ Password is required")
|
||||
sys.exit(1)
|
||||
|
||||
confirm_password = getpass.getpass("Confirm admin password: ").strip()
|
||||
if password != confirm_password:
|
||||
print("❌ Passwords do not match")
|
||||
sys.exit(1)
|
||||
|
||||
first_name = input("Enter first name (optional): ").strip() or None
|
||||
last_name = input("Enter last name (optional): ").strip() or None
|
||||
|
||||
try:
|
||||
async with get_async_session() as session:
|
||||
# Check if user already exists
|
||||
from sqlalchemy import select
|
||||
|
||||
existing_user = await session.execute(
|
||||
select(User).where(
|
||||
(User.username == username) | (User.email == email)
|
||||
)
|
||||
)
|
||||
if existing_user.scalar_one_or_none():
|
||||
print("❌ User with this username or email already exists")
|
||||
sys.exit(1)
|
||||
|
||||
# Create admin user
|
||||
admin_user = User(
|
||||
id=uuid4(),
|
||||
username=username,
|
||||
email=email,
|
||||
password_hash=hash_password(password),
|
||||
first_name=first_name,
|
||||
last_name=last_name,
|
||||
is_active=True,
|
||||
is_verified=True,
|
||||
is_superuser=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
session.add(admin_user)
|
||||
await session.commit()
|
||||
|
||||
print("✅ Admin user created successfully!")
|
||||
print(f" Username: {username}")
|
||||
print(f" Email: {email}")
|
||||
print(f" User ID: {admin_user.id}")
|
||||
print(" Status: Active, Verified, Superuser")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error creating admin user: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(create_admin_user())
|
||||
@@ -0,0 +1,620 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>MY Network Monitor - Distributed Protocol v2.0</title>
|
||||
<style>
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
background: linear-gradient(135deg, #0a0a0a 0%, #1a1a2e 50%, #16213e 100%);
|
||||
color: #00ff41;
|
||||
font-family: 'Courier New', 'Lucida Console', monospace;
|
||||
min-height: 100vh;
|
||||
overflow-x: auto;
|
||||
animation: backgroundPulse 10s ease-in-out infinite alternate;
|
||||
}
|
||||
|
||||
@keyframes backgroundPulse {
|
||||
0% { background: linear-gradient(135deg, #0a0a0a 0%, #1a1a2e 50%, #16213e 100%); }
|
||||
100% { background: linear-gradient(135deg, #0f0f0f 0%, #1f1f3e 50%, #1b274e 100%); }
|
||||
}
|
||||
|
||||
.container {
|
||||
max-width: 1400px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.header {
|
||||
text-align: center;
|
||||
margin-bottom: 30px;
|
||||
padding: 20px;
|
||||
border: 2px solid #00ff41;
|
||||
border-radius: 10px;
|
||||
background: rgba(0, 255, 65, 0.05);
|
||||
box-shadow: 0 0 20px rgba(0, 255, 65, 0.3);
|
||||
}
|
||||
|
||||
.header h1 {
|
||||
font-size: 2.5em;
|
||||
text-shadow: 0 0 10px #00ff41;
|
||||
animation: glow 2s ease-in-out infinite alternate;
|
||||
}
|
||||
|
||||
@keyframes glow {
|
||||
from { text-shadow: 0 0 10px #00ff41, 0 0 20px #00ff41; }
|
||||
to { text-shadow: 0 0 20px #00ff41, 0 0 30px #00ff41, 0 0 40px #00ff41; }
|
||||
}
|
||||
|
||||
.status-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(350px, 1fr));
|
||||
gap: 20px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
|
||||
.status-card {
|
||||
background: rgba(0, 0, 0, 0.7);
|
||||
border: 1px solid #00ff41;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
box-shadow: 0 0 15px rgba(0, 255, 65, 0.2);
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
|
||||
.status-card:hover {
|
||||
transform: translateY(-5px);
|
||||
box-shadow: 0 5px 25px rgba(0, 255, 65, 0.4);
|
||||
}
|
||||
|
||||
.status-card h3 {
|
||||
color: #00ff41;
|
||||
margin-bottom: 15px;
|
||||
font-size: 1.2em;
|
||||
text-transform: uppercase;
|
||||
border-bottom: 1px solid #00ff41;
|
||||
padding-bottom: 5px;
|
||||
}
|
||||
|
||||
.status-item {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
margin: 10px 0;
|
||||
padding: 5px 0;
|
||||
}
|
||||
|
||||
.status-value {
|
||||
color: #ffffff;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.status-online { color: #00ff00; }
|
||||
.status-offline { color: #ff0000; }
|
||||
.status-warning { color: #ffff00; }
|
||||
|
||||
.ascii-display {
|
||||
background: rgba(0, 0, 0, 0.9);
|
||||
border: 2px solid #00ff41;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin: 20px 0;
|
||||
font-family: 'Courier New', monospace;
|
||||
font-size: 11px;
|
||||
line-height: 1.1;
|
||||
white-space: pre;
|
||||
overflow-x: auto;
|
||||
box-shadow: inset 0 0 20px rgba(0, 255, 65, 0.1);
|
||||
}
|
||||
|
||||
.control-panel {
|
||||
display: flex;
|
||||
gap: 15px;
|
||||
margin: 20px 0;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.btn {
|
||||
background: linear-gradient(45deg, #00ff41, #00cc33);
|
||||
color: #000;
|
||||
border: none;
|
||||
padding: 12px 24px;
|
||||
font-family: inherit;
|
||||
font-weight: bold;
|
||||
cursor: pointer;
|
||||
border-radius: 5px;
|
||||
text-transform: uppercase;
|
||||
transition: all 0.3s ease;
|
||||
box-shadow: 0 0 10px rgba(0, 255, 65, 0.3);
|
||||
}
|
||||
|
||||
.btn:hover {
|
||||
background: linear-gradient(45deg, #00cc33, #00ff41);
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 5px 15px rgba(0, 255, 65, 0.5);
|
||||
}
|
||||
|
||||
.btn:active {
|
||||
transform: translateY(1px);
|
||||
}
|
||||
|
||||
.network-topology {
|
||||
background: rgba(0, 0, 0, 0.8);
|
||||
border: 1px solid #00ff41;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin: 20px 0;
|
||||
min-height: 300px;
|
||||
}
|
||||
|
||||
.node {
|
||||
display: inline-block;
|
||||
background: rgba(0, 255, 65, 0.1);
|
||||
border: 2px solid #00ff41;
|
||||
border-radius: 50%;
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
line-height: 76px;
|
||||
text-align: center;
|
||||
margin: 10px;
|
||||
position: relative;
|
||||
animation: pulse 3s ease-in-out infinite;
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0%, 100% { transform: scale(1); }
|
||||
50% { transform: scale(1.05); }
|
||||
}
|
||||
|
||||
.node.this-node {
|
||||
background: rgba(0, 255, 65, 0.3);
|
||||
animation: strongPulse 2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
@keyframes strongPulse {
|
||||
0%, 100% { box-shadow: 0 0 10px #00ff41; }
|
||||
50% { box-shadow: 0 0 30px #00ff41, 0 0 40px #00ff41; }
|
||||
}
|
||||
|
||||
.logs-panel {
|
||||
background: rgba(0, 0, 0, 0.9);
|
||||
border: 1px solid #00ff41;
|
||||
border-radius: 8px;
|
||||
padding: 15px;
|
||||
margin: 20px 0;
|
||||
height: 200px;
|
||||
overflow-y: auto;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.log-entry {
|
||||
margin: 5px 0;
|
||||
padding: 2px 0;
|
||||
}
|
||||
|
||||
.log-timestamp {
|
||||
color: #888;
|
||||
margin-right: 10px;
|
||||
}
|
||||
|
||||
.log-info { color: #00ff41; }
|
||||
.log-warning { color: #ffff00; }
|
||||
.log-error { color: #ff0000; }
|
||||
|
||||
.footer {
|
||||
text-align: center;
|
||||
margin-top: 40px;
|
||||
padding: 20px;
|
||||
border-top: 1px solid #00ff41;
|
||||
color: #888;
|
||||
}
|
||||
|
||||
.loading {
|
||||
display: inline-block;
|
||||
animation: spin 1s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
0% { transform: rotate(0deg); }
|
||||
100% { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
.metrics-bar {
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
height: 20px;
|
||||
border-radius: 10px;
|
||||
margin: 10px 0;
|
||||
overflow: hidden;
|
||||
border: 1px solid #00ff41;
|
||||
}
|
||||
|
||||
.metrics-fill {
|
||||
height: 100%;
|
||||
background: linear-gradient(90deg, #00ff41, #00cc33);
|
||||
transition: width 0.5s ease;
|
||||
}
|
||||
|
||||
/* Responsive design */
|
||||
@media (max-width: 768px) {
|
||||
.status-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.header h1 {
|
||||
font-size: 1.8em;
|
||||
}
|
||||
|
||||
.ascii-display {
|
||||
font-size: 9px;
|
||||
}
|
||||
|
||||
.control-panel {
|
||||
justify-content: center;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<h1>MY NETWORK MONITOR</h1>
|
||||
<p>Distributed Content Protocol v2.0</p>
|
||||
<p id="last-update">Last Update: <span id="timestamp">Loading...</span></p>
|
||||
</div>
|
||||
|
||||
<div class="control-panel">
|
||||
<button class="btn" onclick="refreshData()">
|
||||
<span id="refresh-icon">🔄</span> REFRESH
|
||||
</button>
|
||||
<button class="btn" onclick="toggleASCII()">
|
||||
📊 ASCII VIEW
|
||||
</button>
|
||||
<button class="btn" onclick="startSync()">
|
||||
⚡ START SYNC
|
||||
</button>
|
||||
<button class="btn" onclick="showLogs()">
|
||||
📋 LOGS
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="status-grid">
|
||||
<div class="status-card">
|
||||
<h3>🖥️ Node Status</h3>
|
||||
<div class="status-item">
|
||||
<span>Node ID:</span>
|
||||
<span class="status-value" id="node-id">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Status:</span>
|
||||
<span class="status-value" id="node-status">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Uptime:</span>
|
||||
<span class="status-value" id="node-uptime">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Version:</span>
|
||||
<span class="status-value" id="node-version">Loading...</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="status-card">
|
||||
<h3>🌐 Network Status</h3>
|
||||
<div class="status-item">
|
||||
<span>Connected Peers:</span>
|
||||
<span class="status-value" id="peer-count">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Known Nodes:</span>
|
||||
<span class="status-value" id="known-nodes">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Network Health:</span>
|
||||
<span class="status-value" id="network-health">Loading...</span>
|
||||
</div>
|
||||
<div class="metrics-bar">
|
||||
<div class="metrics-fill" id="network-bar" style="width: 0%"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="status-card">
|
||||
<h3>⚡ Sync Engine</h3>
|
||||
<div class="status-item">
|
||||
<span>Sync Status:</span>
|
||||
<span class="status-value" id="sync-status">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Active Syncs:</span>
|
||||
<span class="status-value" id="active-syncs">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Queue Size:</span>
|
||||
<span class="status-value" id="queue-size">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Workers:</span>
|
||||
<span class="status-value" id="workers-count">Loading...</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="status-card">
|
||||
<h3>📊 Content Stats</h3>
|
||||
<div class="status-item">
|
||||
<span>Total Items:</span>
|
||||
<span class="status-value" id="content-items">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Total Size:</span>
|
||||
<span class="status-value" id="content-size">Loading...</span>
|
||||
</div>
|
||||
<div class="status-item">
|
||||
<span>Replicated:</span>
|
||||
<span class="status-value" id="replicated-count">Loading...</span>
|
||||
</div>
|
||||
<div class="metrics-bar">
|
||||
<div class="metrics-fill" id="storage-bar" style="width: 0%"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="ascii-container" class="ascii-display" style="display: none;">
|
||||
<div id="ascii-content">Loading ASCII status...</div>
|
||||
</div>
|
||||
|
||||
<div class="network-topology" id="topology-view">
|
||||
<h3>🔗 Network Topology</h3>
|
||||
<div id="topology-nodes">
|
||||
<div class="node this-node">THIS<br>NODE</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="logs-panel" id="logs-panel" style="display: none;">
|
||||
<h3>📋 System Logs</h3>
|
||||
<div id="logs-content">
|
||||
<div class="log-entry log-info">
|
||||
<span class="log-timestamp">{{ monitoring_data.timestamp[:19] if monitoring_data.timestamp else 'N/A' }}</span>
|
||||
<span>[INFO] MY Network Monitor initialized</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="footer">
|
||||
<p>MY Network Protocol - Decentralized Content Distribution System</p>
|
||||
<p>Real-time monitoring dashboard with live updates every 15 seconds</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
let asciiVisible = false;
|
||||
let logsVisible = false;
|
||||
let updateInterval;
|
||||
|
||||
// Инициализация данных
|
||||
let monitoringData = {};
|
||||
|
||||
function setMonitoringData(data) {
|
||||
monitoringData = data;
|
||||
}
|
||||
|
||||
function initializeData() {
|
||||
if (monitoringData.status === 'online') {
|
||||
updateNodeInfo(monitoringData.node_info);
|
||||
updatePeersInfo(monitoringData.peers_info);
|
||||
updateSyncStatus(monitoringData.sync_status);
|
||||
} else {
|
||||
showOfflineStatus();
|
||||
}
|
||||
updateTimestamp();
|
||||
}
|
||||
|
||||
function updateNodeInfo(nodeInfo) {
|
||||
document.getElementById('node-id').textContent =
|
||||
nodeInfo.node_id ? nodeInfo.node_id.substring(0, 16) + '...' : 'Unknown';
|
||||
|
||||
const statusElement = document.getElementById('node-status');
|
||||
statusElement.textContent = nodeInfo.status ? nodeInfo.status.toUpperCase() : 'UNKNOWN';
|
||||
statusElement.className = 'status-value ' +
|
||||
(nodeInfo.status === 'running' ? 'status-online' : 'status-offline');
|
||||
|
||||
const uptime = nodeInfo.uptime || 0;
|
||||
const hours = Math.floor(uptime / 3600);
|
||||
const minutes = Math.floor((uptime % 3600) / 60);
|
||||
document.getElementById('node-uptime').textContent = `${hours}h ${minutes}m`;
|
||||
|
||||
document.getElementById('node-version').textContent = nodeInfo.version || 'MY Network 2.0';
|
||||
}
|
||||
|
||||
function updatePeersInfo(peersInfo) {
|
||||
const peerCount = peersInfo.peer_count || 0;
|
||||
document.getElementById('peer-count').textContent = peerCount;
|
||||
document.getElementById('known-nodes').textContent = peersInfo.peers ? peersInfo.peers.length : 0;
|
||||
|
||||
const healthElement = document.getElementById('network-health');
|
||||
const health = peerCount > 0 ? 'CONNECTED' : 'ISOLATED';
|
||||
healthElement.textContent = health;
|
||||
healthElement.className = 'status-value ' +
|
||||
(peerCount > 0 ? 'status-online' : 'status-warning');
|
||||
|
||||
// Обновить индикатор сети
|
||||
const networkBar = document.getElementById('network-bar');
|
||||
const healthPercent = Math.min(peerCount * 20, 100);
|
||||
networkBar.style.width = healthPercent + '%';
|
||||
|
||||
// Обновить топологию
|
||||
updateTopology(peersInfo.peers || []);
|
||||
}
|
||||
|
||||
function updateSyncStatus(syncStatus) {
|
||||
const isRunning = syncStatus.is_running || false;
|
||||
const statusElement = document.getElementById('sync-status');
|
||||
statusElement.textContent = isRunning ? 'RUNNING' : 'STOPPED';
|
||||
statusElement.className = 'status-value ' +
|
||||
(isRunning ? 'status-online' : 'status-offline');
|
||||
|
||||
document.getElementById('active-syncs').textContent = syncStatus.active_syncs || 0;
|
||||
document.getElementById('queue-size').textContent = syncStatus.queue_size || 0;
|
||||
document.getElementById('workers-count').textContent = syncStatus.workers_count || 0;
|
||||
}
|
||||
|
||||
function updateTopology(peers) {
|
||||
const topologyNodes = document.getElementById('topology-nodes');
|
||||
|
||||
// Очистить существующие ноды (кроме центральной)
|
||||
const existingNodes = topologyNodes.querySelectorAll('.node:not(.this-node)');
|
||||
existingNodes.forEach(node => node.remove());
|
||||
|
||||
// Добавить ноды пиров
|
||||
peers.slice(0, 8).forEach((peer, index) => {
|
||||
const node = document.createElement('div');
|
||||
node.className = 'node';
|
||||
node.innerHTML = peer.node_id ? peer.node_id.substring(0, 4).toUpperCase() : 'PEER';
|
||||
node.style.animationDelay = (index * 0.2) + 's';
|
||||
topologyNodes.appendChild(node);
|
||||
});
|
||||
}
|
||||
|
||||
function showOfflineStatus() {
|
||||
document.getElementById('node-status').textContent = 'OFFLINE';
|
||||
document.getElementById('node-status').className = 'status-value status-offline';
|
||||
document.getElementById('network-health').textContent = 'DISCONNECTED';
|
||||
document.getElementById('network-health').className = 'status-value status-offline';
|
||||
document.getElementById('sync-status').textContent = 'STOPPED';
|
||||
document.getElementById('sync-status').className = 'status-value status-offline';
|
||||
}
|
||||
|
||||
function updateTimestamp() {
|
||||
const now = new Date();
|
||||
document.getElementById('timestamp').textContent =
|
||||
now.toISOString().substring(0, 19).replace('T', ' ') + ' UTC';
|
||||
}
|
||||
|
||||
async function refreshData() {
|
||||
const refreshIcon = document.getElementById('refresh-icon');
|
||||
refreshIcon.className = 'loading';
|
||||
refreshIcon.textContent = '⏳';
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/my/monitor/live');
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
updateNodeInfo(data.data.node_info);
|
||||
updatePeersInfo({
|
||||
peer_count: data.data.network_stats.connected_peers,
|
||||
peers: data.data.peers
|
||||
});
|
||||
updateSyncStatus(data.data.sync_status);
|
||||
updateTimestamp();
|
||||
|
||||
addLogEntry('info', 'Data refreshed successfully');
|
||||
}
|
||||
} catch (error) {
|
||||
addLogEntry('error', 'Failed to refresh data: ' + error.message);
|
||||
} finally {
|
||||
refreshIcon.className = '';
|
||||
refreshIcon.textContent = '🔄';
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleASCII() {
|
||||
asciiVisible = !asciiVisible;
|
||||
const container = document.getElementById('ascii-container');
|
||||
|
||||
if (asciiVisible) {
|
||||
container.style.display = 'block';
|
||||
try {
|
||||
const response = await fetch('/api/my/monitor/ascii');
|
||||
const data = await response.json();
|
||||
document.getElementById('ascii-content').textContent = data.ascii;
|
||||
} catch (error) {
|
||||
document.getElementById('ascii-content').textContent = 'Error loading ASCII view';
|
||||
}
|
||||
} else {
|
||||
container.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
async function startSync() {
|
||||
try {
|
||||
const response = await fetch('/api/my/sync/start', { method: 'POST' });
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
addLogEntry('info', 'Network sync started');
|
||||
setTimeout(refreshData, 1000); // Обновить через секунду
|
||||
} else {
|
||||
addLogEntry('error', 'Failed to start sync');
|
||||
}
|
||||
} catch (error) {
|
||||
addLogEntry('error', 'Sync request failed: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
function showLogs() {
|
||||
logsVisible = !logsVisible;
|
||||
const logsPanel = document.getElementById('logs-panel');
|
||||
logsPanel.style.display = logsVisible ? 'block' : 'none';
|
||||
}
|
||||
|
||||
function addLogEntry(level, message) {
|
||||
const logsContent = document.getElementById('logs-content');
|
||||
const logEntry = document.createElement('div');
|
||||
logEntry.className = `log-entry log-${level}`;
|
||||
|
||||
const timestamp = new Date().toISOString().substring(11, 19);
|
||||
logEntry.innerHTML = `
|
||||
<span class="log-timestamp">${timestamp}</span>
|
||||
<span>[${level.toUpperCase()}] ${message}</span>
|
||||
`;
|
||||
|
||||
logsContent.appendChild(logEntry);
|
||||
logsContent.scrollTop = logsContent.scrollHeight;
|
||||
|
||||
// Ограничить количество логов
|
||||
if (logsContent.children.length > 50) {
|
||||
logsContent.removeChild(logsContent.firstChild);
|
||||
}
|
||||
}
|
||||
|
||||
// Автоматическое обновление каждые 15 секунд
|
||||
function startAutoUpdate() {
|
||||
updateInterval = setInterval(refreshData, 15000);
|
||||
}
|
||||
|
||||
function stopAutoUpdate() {
|
||||
if (updateInterval) {
|
||||
clearInterval(updateInterval);
|
||||
}
|
||||
}
|
||||
|
||||
// Инициализация при загрузке страницы
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
// Получить данные из data-атрибута
|
||||
const dataElement = document.getElementById('monitoring-data');
|
||||
if (dataElement) {
|
||||
try {
|
||||
const data = JSON.parse(dataElement.textContent);
|
||||
setMonitoringData(data);
|
||||
} catch (e) {
|
||||
console.error('Error parsing monitoring data:', e);
|
||||
setMonitoringData({status: 'offline', error: 'Data parsing failed'});
|
||||
}
|
||||
}
|
||||
initializeData();
|
||||
startAutoUpdate();
|
||||
addLogEntry('info', 'MY Network Monitor loaded');
|
||||
});
|
||||
|
||||
// Остановить обновления при уходе со страницы
|
||||
window.addEventListener('beforeunload', stopAutoUpdate);
|
||||
</script>
|
||||
|
||||
<!-- Данные мониторинга -->
|
||||
<script type="application/json" id="monitoring-data">{{ monitoring_data | tojson }}</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in new issue
Block a user