This commit is contained in:
user committed 2025-07-02 19:25:20 +03:00
1 parent 21964fa986
commit 797f379648
68 files changed
+23871 -1271

No files matched your search

+870
View File
@@ -0,0 +1,870 @@
"""
Authentication and authorization routes with JWT tokens, user management, and security features.
Provides user registration, login, token refresh, and account management with comprehensive validation.
"""
import asyncio
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any
from uuid import UUID, uuid4
from sanic import Blueprint, Request, response
from sanic.response import JSONResponse
from sqlalchemy import select, update, and_
from sqlalchemy.orm import selectinload
from app.core.config import get_settings
from app.core.database import get_async_session, get_cache_manager
from app.core.logging import get_logger
from app.core.models.user import User, UserSession, UserRole
from app.core.security import (
hash_password, verify_password, generate_access_token,
verify_access_token, generate_refresh_token, generate_api_key,
sanitize_input, generate_csrf_token
)
from app.api.middleware import require_auth, validate_request, rate_limit
from app.core.validation import (
UserRegistrationSchema, UserLoginSchema, UserUpdateSchema,
ApiKeySchema
)
# Initialize blueprint
auth_bp = Blueprint("auth", url_prefix="/api/v1/auth")
logger = get_logger(__name__)
settings = get_settings()
@auth_bp.route("/register", methods=["POST"])
@rate_limit(limit=5, window=3600) # 5 registrations per hour
@validate_request(UserRegistrationSchema)
async def register_user(request: Request) -> JSONResponse:
"""
Register new user with comprehensive validation and security checks.
Args:
request: Sanic request with user registration data
Returns:
JSONResponse: Registration result with access tokens
"""
try:
data = request.json
client_ip = request.headers.get("X-Forwarded-For", request.remote_addr)
# Sanitize input data
username = sanitize_input(data["username"])
email = sanitize_input(data["email"])
full_name = sanitize_input(data.get("full_name", ""))
async with get_async_session() as session:
# Check if username already exists
username_stmt = select(User).where(User.username == username)
username_result = await session.execute(username_stmt)
if username_result.scalar_one_or_none():
return response.json(
{"error": "Username already exists", "code": "USERNAME_EXISTS"},
status=400
)
# Check if email already exists
email_stmt = select(User).where(User.email == email)
email_result = await session.execute(email_stmt)
if email_result.scalar_one_or_none():
return response.json(
{"error": "Email already registered", "code": "EMAIL_EXISTS"},
status=400
)
# Check registration rate limiting by IP
cache_manager = get_cache_manager()
ip_reg_key = f"registration_ip:{client_ip}"
ip_registrations = await cache_manager.get(ip_reg_key, default=0)
if ip_registrations >= 3: # Max 3 registrations per IP per day
return response.json(
{"error": "Too many registrations from this IP", "code": "IP_LIMIT_EXCEEDED"},
status=429
)
# Hash password
password_hash = hash_password(data["password"])
# Create user
new_user = User(
id=uuid4(),
username=username,
email=email,
password_hash=password_hash,
full_name=full_name,
is_active=True,
email_verified=False, # Require email verification
registration_ip=client_ip,
last_login_ip=client_ip,
settings={"theme": "light", "notifications": True}
)
session.add(new_user)
await session.commit()
await session.refresh(new_user)
# Assign default role
default_role_stmt = select(UserRole).where(UserRole.name == "user")
role_result = await session.execute(default_role_stmt)
default_role = role_result.scalar_one_or_none()
if default_role:
new_user.roles.append(default_role)
await session.commit()
# Update IP registration counter
await cache_manager.increment(ip_reg_key, ttl=86400) # 24 hours
# Generate tokens
access_token = generate_access_token(
{"user_id": str(new_user.id), "username": username},
expires_in=settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
)
refresh_token = generate_refresh_token(new_user.id)
# Create user session
session_id = str(uuid4())
csrf_token = generate_csrf_token(new_user.id, session_id)
async with get_async_session() as session:
user_session = UserSession(
id=UUID(session_id),
user_id=new_user.id,
refresh_token_hash=hash_password(refresh_token[-32:]), # Hash last 32 chars
ip_address=client_ip,
user_agent=request.headers.get("User-Agent", ""),
expires_at=datetime.utcnow() + timedelta(days=settings.REFRESH_TOKEN_EXPIRE_DAYS)
)
session.add(user_session)
await session.commit()
await logger.ainfo(
"User registered successfully",
user_id=str(new_user.id),
username=username,
email=email,
ip=client_ip
)
return response.json({
"message": "Registration successful",
"user": {
"id": str(new_user.id),
"username": username,
"email": email,
"full_name": full_name,
"created_at": new_user.created_at.isoformat()
},
"tokens": {
"access_token": access_token,
"refresh_token": refresh_token,
"token_type": "Bearer",
"expires_in": settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
},
"session": {
"session_id": session_id,
"csrf_token": csrf_token
}
}, status=201)
except Exception as e:
await logger.aerror(
"User registration failed",
username=data.get("username"),
email=data.get("email"),
error=str(e)
)
return response.json(
{"error": "Registration failed", "code": "REGISTRATION_FAILED"},
status=500
)
@auth_bp.route("/login", methods=["POST"])
@rate_limit(limit=10, window=900) # 10 login attempts per 15 minutes
@validate_request(UserLoginSchema)
async def login_user(request: Request) -> JSONResponse:
"""
Authenticate user and generate access tokens with security logging.
Args:
request: Sanic request with login credentials
Returns:
JSONResponse: Authentication result with tokens
"""
try:
data = request.json
username_or_email = sanitize_input(data["username"])
password = data["password"]
remember_me = data.get("remember_me", False)
client_ip = request.headers.get("X-Forwarded-For", request.remote_addr)
# Check login rate limiting
cache_manager = get_cache_manager()
login_key = f"login_attempts:{username_or_email}:{client_ip}"
attempts = await cache_manager.get(login_key, default=0)
if attempts >= 5: # Max 5 failed attempts
return response.json(
{"error": "Too many login attempts", "code": "LOGIN_BLOCKED"},
status=429
)
async with get_async_session() as session:
# Find user by username or email
user_stmt = select(User).where(
or_(User.username == username_or_email, User.email == username_or_email)
).options(selectinload(User.roles))
user_result = await session.execute(user_stmt)
user = user_result.scalar_one_or_none()
if not user or not verify_password(password, user.password_hash):
# Increment failed attempts
await cache_manager.increment(login_key, ttl=900) # 15 minutes
await logger.awarning(
"Failed login attempt",
username=username_or_email,
ip=client_ip,
attempts=attempts + 1
)
return response.json(
{"error": "Invalid credentials", "code": "INVALID_CREDENTIALS"},
status=401
)
if not user.is_active:
return response.json(
{"error": "Account deactivated", "code": "ACCOUNT_DEACTIVATED"},
status=403
)
# Successful login - clear failed attempts
await cache_manager.delete(login_key)
# Update user login info
user.last_login_at = datetime.utcnow()
user.last_login_ip = client_ip
user.login_count = (user.login_count or 0) + 1
await session.commit()
# Generate tokens
user_permissions = []
for role in user.roles:
user_permissions.extend(role.permissions)
token_payload = {
"user_id": str(user.id),
"username": user.username,
"permissions": list(set(user_permissions)) # Remove duplicates
}
expires_in = settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
if remember_me:
expires_in *= 24 # 24x longer for remember me
access_token = generate_access_token(token_payload, expires_in=expires_in)
refresh_token = generate_refresh_token(user.id)
# Create user session
session_id = str(uuid4())
csrf_token = generate_csrf_token(user.id, session_id)
refresh_expires = timedelta(days=settings.REFRESH_TOKEN_EXPIRE_DAYS)
if remember_me:
refresh_expires *= 2 # Longer refresh for remember me
async with get_async_session() as session:
user_session = UserSession(
id=UUID(session_id),
user_id=user.id,
refresh_token_hash=hash_password(refresh_token[-32:]),
ip_address=client_ip,
user_agent=request.headers.get("User-Agent", ""),
expires_at=datetime.utcnow() + refresh_expires,
remember_me=remember_me
)
session.add(user_session)
await session.commit()
await logger.ainfo(
"User logged in successfully",
user_id=str(user.id),
username=user.username,
ip=client_ip,
remember_me=remember_me
)
return response.json({
"message": "Login successful",
"user": {
"id": str(user.id),
"username": user.username,
"email": user.email,
"full_name": user.full_name,
"last_login": user.last_login_at.isoformat() if user.last_login_at else None,
"permissions": user_permissions
},
"tokens": {
"access_token": access_token,
"refresh_token": refresh_token,
"token_type": "Bearer",
"expires_in": expires_in
},
"session": {
"session_id": session_id,
"csrf_token": csrf_token
}
})
except Exception as e:
await logger.aerror(
"Login failed",
username=data.get("username"),
error=str(e)
)
return response.json(
{"error": "Login failed", "code": "LOGIN_FAILED"},
status=500
)
@auth_bp.route("/refresh", methods=["POST"])
@rate_limit(limit=50, window=3600) # 50 refresh attempts per hour
async def refresh_tokens(request: Request) -> JSONResponse:
"""
Refresh access token using refresh token with rotation.
Args:
request: Sanic request with refresh token
Returns:
JSONResponse: New access and refresh tokens
"""
try:
refresh_token = request.json.get("refresh_token")
if not refresh_token:
return response.json(
{"error": "Refresh token required", "code": "TOKEN_REQUIRED"},
status=400
)
# Verify refresh token
payload = verify_access_token(refresh_token, token_type="refresh")
if not payload:
return response.json(
{"error": "Invalid refresh token", "code": "INVALID_TOKEN"},
status=401
)
user_id = UUID(payload["user_id"])
async with get_async_session() as session:
# Verify session exists and is valid
session_stmt = select(UserSession).where(
and_(
UserSession.user_id == user_id,
UserSession.refresh_token_hash == hash_password(refresh_token[-32:]),
UserSession.expires_at > datetime.utcnow(),
UserSession.is_active == True
)
)
session_result = await session.execute(session_stmt)
user_session = session_result.scalar_one_or_none()
if not user_session:
return response.json(
{"error": "Session expired or invalid", "code": "SESSION_INVALID"},
status=401
)
# Get user with permissions
user_stmt = select(User).where(User.id == user_id).options(selectinload(User.roles))
user_result = await session.execute(user_stmt)
user = user_result.scalar_one_or_none()
if not user or not user.is_active:
return response.json(
{"error": "User not found or inactive", "code": "USER_INACTIVE"},
status=401
)
# Generate new tokens (token rotation)
user_permissions = []
for role in user.roles:
user_permissions.extend(role.permissions)
new_access_token = generate_access_token(
{
"user_id": str(user.id),
"username": user.username,
"permissions": list(set(user_permissions))
},
expires_in=settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
)
new_refresh_token = generate_refresh_token(user.id)
# Update session with new refresh token
user_session.refresh_token_hash = hash_password(new_refresh_token[-32:])
user_session.last_used_at = datetime.utcnow()
await session.commit()
await logger.adebug(
"Tokens refreshed",
user_id=str(user_id),
session_id=str(user_session.id)
)
return response.json({
"tokens": {
"access_token": new_access_token,
"refresh_token": new_refresh_token,
"token_type": "Bearer",
"expires_in": settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
}
})
except Exception as e:
await logger.aerror("Token refresh failed", error=str(e))
return response.json(
{"error": "Token refresh failed", "code": "REFRESH_FAILED"},
status=500
)
@auth_bp.route("/logout", methods=["POST"])
@require_auth()
async def logout_user(request: Request) -> JSONResponse:
"""
Logout user and invalidate session.
Args:
request: Sanic request object
Returns:
JSONResponse: Logout confirmation
"""
try:
user_id = request.ctx.user.id
session_id = request.headers.get("X-Session-ID")
if session_id:
async with get_async_session() as session:
# Invalidate specific session
session_stmt = select(UserSession).where(
and_(
UserSession.id == UUID(session_id),
UserSession.user_id == user_id
)
)
session_result = await session.execute(session_stmt)
user_session = session_result.scalar_one_or_none()
if user_session:
user_session.is_active = False
user_session.logged_out_at = datetime.utcnow()
await session.commit()
await logger.ainfo(
"User logged out",
user_id=str(user_id),
session_id=session_id
)
return response.json({
"message": "Logout successful",
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
await logger.aerror(
"Logout failed",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Logout failed", "code": "LOGOUT_FAILED"},
status=500
)
@auth_bp.route("/me", methods=["GET"])
@require_auth()
async def get_current_user(request: Request) -> JSONResponse:
"""
Get current user information and permissions.
Args:
request: Sanic request object
Returns:
JSONResponse: Current user data
"""
try:
user = request.ctx.user
async with get_async_session() as session:
# Get user with full details
user_stmt = select(User).where(User.id == user.id).options(
selectinload(User.roles),
selectinload(User.api_keys)
)
user_result = await session.execute(user_stmt)
full_user = user_result.scalar_one_or_none()
if not full_user:
return response.json(
{"error": "User not found", "code": "USER_NOT_FOUND"},
status=404
)
# Get user permissions
permissions = []
roles = []
for role in full_user.roles:
roles.append({
"name": role.name,
"description": role.description
})
permissions.extend(role.permissions)
# Get active sessions
sessions_stmt = select(UserSession).where(
and_(
UserSession.user_id == user.id,
UserSession.is_active == True,
UserSession.expires_at > datetime.utcnow()
)
)
sessions_result = await session.execute(sessions_stmt)
active_sessions = sessions_result.scalars().all()
return response.json({
"user": {
"id": str(full_user.id),
"username": full_user.username,
"email": full_user.email,
"full_name": full_user.full_name,
"bio": full_user.bio,
"avatar_url": full_user.avatar_url,
"is_active": full_user.is_active,
"email_verified": full_user.email_verified,
"created_at": full_user.created_at.isoformat(),
"last_login_at": full_user.last_login_at.isoformat() if full_user.last_login_at else None,
"login_count": full_user.login_count,
"settings": full_user.settings
},
"roles": roles,
"permissions": list(set(permissions)),
"active_sessions": len(active_sessions),
"api_keys": [
{
"id": str(key.id),
"name": key.name,
"created_at": key.created_at.isoformat(),
"last_used_at": key.last_used_at.isoformat() if key.last_used_at else None,
"expires_at": key.expires_at.isoformat() if key.expires_at else None
}
for key in full_user.api_keys
if key.is_active
]
})
except Exception as e:
await logger.aerror(
"Failed to get current user",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Failed to get user information", "code": "USER_INFO_FAILED"},
status=500
)
@auth_bp.route("/me", methods=["PUT"])
@require_auth()
@validate_request(UserUpdateSchema)
async def update_current_user(request: Request) -> JSONResponse:
"""
Update current user profile information.
Args:
request: Sanic request with update data
Returns:
JSONResponse: Updated user information
"""
try:
user_id = request.ctx.user.id
data = request.json
async with get_async_session() as session:
# Get current user
user_stmt = select(User).where(User.id == user_id)
user_result = await session.execute(user_stmt)
user = user_result.scalar_one_or_none()
if not user:
return response.json(
{"error": "User not found", "code": "USER_NOT_FOUND"},
status=404
)
# Update allowed fields
updatable_fields = ["full_name", "bio", "avatar_url", "settings"]
for field in updatable_fields:
if field in data:
if field == "full_name":
setattr(user, field, sanitize_input(data[field]))
elif field == "bio":
setattr(user, field, sanitize_input(data[field], max_length=500))
else:
setattr(user, field, data[field])
# Handle email change (requires verification)
if "email" in data and data["email"] != user.email:
new_email = sanitize_input(data["email"])
# Check if email is already taken
email_stmt = select(User).where(
and_(User.email == new_email, User.id != user_id)
)
email_result = await session.execute(email_stmt)
if email_result.scalar_one_or_none():
return response.json(
{"error": "Email already in use", "code": "EMAIL_IN_USE"},
status=400
)
user.email = new_email
user.email_verified = False # Require re-verification
user.updated_at = datetime.utcnow()
await session.commit()
await logger.ainfo(
"User profile updated",
user_id=str(user_id),
updated_fields=list(data.keys())
)
return response.json({
"message": "Profile updated successfully",
"user": {
"id": str(user.id),
"username": user.username,
"email": user.email,
"full_name": user.full_name,
"bio": user.bio,
"avatar_url": user.avatar_url,
"updated_at": user.updated_at.isoformat()
}
})
except Exception as e:
await logger.aerror(
"Failed to update user profile",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Failed to update profile", "code": "UPDATE_FAILED"},
status=500
)
@auth_bp.route("/api-keys", methods=["POST"])
@rate_limit(limit=5, window=3600) # 5 API keys per hour
@require_auth(permissions=["api.create"])
@validate_request(ApiKeySchema)
async def create_api_key(request: Request) -> JSONResponse:
"""
Create new API key for programmatic access.
Args:
request: Sanic request with API key data
Returns:
JSONResponse: Created API key information
"""
try:
user_id = request.ctx.user.id
data = request.json
# Generate API key
api_key = generate_api_key(
user_id=user_id,
permissions=data["permissions"],
name=data["name"],
expires_in=None if not data.get("expires_at") else
int((datetime.fromisoformat(data["expires_at"]) - datetime.utcnow()).total_seconds())
)
async with get_async_session() as session:
from app.core.models.user import ApiKey
# Create API key record
new_api_key = ApiKey(
id=uuid4(),
user_id=user_id,
name=sanitize_input(data["name"]),
key_hash=hash_password(api_key[-32:]), # Hash last 32 chars
permissions=data["permissions"],
expires_at=datetime.fromisoformat(data["expires_at"]) if data.get("expires_at") else None
)
session.add(new_api_key)
await session.commit()
await session.refresh(new_api_key)
await logger.ainfo(
"API key created",
user_id=str(user_id),
api_key_id=str(new_api_key.id),
name=data["name"],
permissions=data["permissions"]
)
return response.json({
"message": "API key created successfully",
"api_key": {
"id": str(new_api_key.id),
"name": new_api_key.name,
"key": api_key, # Only returned once
"permissions": new_api_key.permissions,
"created_at": new_api_key.created_at.isoformat(),
"expires_at": new_api_key.expires_at.isoformat() if new_api_key.expires_at else None
},
"warning": "Save this API key securely. It will not be shown again."
}, status=201)
except Exception as e:
await logger.aerror(
"Failed to create API key",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Failed to create API key", "code": "API_KEY_FAILED"},
status=500
)
@auth_bp.route("/sessions", methods=["GET"])
@require_auth()
async def get_user_sessions(request: Request) -> JSONResponse:
"""
Get all active user sessions.
Args:
request: Sanic request object
Returns:
JSONResponse: List of active sessions
"""
try:
user_id = request.ctx.user.id
async with get_async_session() as session:
sessions_stmt = select(UserSession).where(
and_(
UserSession.user_id == user_id,
UserSession.is_active == True,
UserSession.expires_at > datetime.utcnow()
)
).order_by(UserSession.created_at.desc())
sessions_result = await session.execute(sessions_stmt)
sessions = sessions_result.scalars().all()
sessions_data = []
for sess in sessions:
sessions_data.append({
"id": str(sess.id),
"ip_address": sess.ip_address,
"user_agent": sess.user_agent,
"created_at": sess.created_at.isoformat(),
"last_used_at": sess.last_used_at.isoformat() if sess.last_used_at else None,
"expires_at": sess.expires_at.isoformat(),
"remember_me": sess.remember_me,
"is_current": str(sess.id) == request.headers.get("X-Session-ID")
})
return response.json({
"sessions": sessions_data,
"total": len(sessions_data)
})
except Exception as e:
await logger.aerror(
"Failed to get user sessions",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Failed to get sessions", "code": "SESSIONS_FAILED"},
status=500
)
@auth_bp.route("/sessions/<session_id:uuid>", methods=["DELETE"])
@require_auth()
async def revoke_session(request: Request, session_id: UUID) -> JSONResponse:
"""
Revoke specific user session.
Args:
request: Sanic request object
session_id: Session UUID to revoke
Returns:
JSONResponse: Revocation status
"""
try:
user_id = request.ctx.user.id
async with get_async_session() as session:
session_stmt = select(UserSession).where(
and_(
UserSession.id == session_id,
UserSession.user_id == user_id
)
)
session_result = await session.execute(session_stmt)
user_session = session_result.scalar_one_or_none()
if not user_session:
return response.json(
{"error": "Session not found", "code": "SESSION_NOT_FOUND"},
status=404
)
user_session.is_active = False
user_session.logged_out_at = datetime.utcnow()
await session.commit()
await logger.ainfo(
"Session revoked",
user_id=str(user_id),
session_id=str(session_id)
)
return response.json({
"message": "Session revoked successfully",
"session_id": str(session_id)
})
except Exception as e:
await logger.aerror(
"Failed to revoke session",
user_id=str(request.ctx.user.id),
session_id=str(session_id),
error=str(e)
)
return response.json(
{"error": "Failed to revoke session", "code": "REVOKE_FAILED"},
status=500
)
+634
View File
@@ -0,0 +1,634 @@
"""
Blockchain operations routes for TON integration with async wallet management.
Provides secure transaction handling, balance queries, and smart contract interactions.
"""
import asyncio
from datetime import datetime, timedelta
from decimal import Decimal
from typing import Dict, List, Optional, Any
from uuid import UUID, uuid4
from sanic import Blueprint, Request, response
from sanic.response import JSONResponse
from sqlalchemy import select, update, and_
from app.core.config import get_settings
from app.core.database import get_async_session, get_cache_manager
from app.core.logging import get_logger
from app.core.models.user import User
from app.api.middleware import require_auth, validate_request, rate_limit
from app.core.validation import BlockchainTransactionSchema
from app.core.background.ton_service import TONService
# Initialize blueprint
blockchain_bp = Blueprint("blockchain", url_prefix="/api/v1/blockchain")
logger = get_logger(__name__)
settings = get_settings()
@blockchain_bp.route("/wallet/balance", methods=["GET"])
@rate_limit(limit=100, window=3600) # 100 balance checks per hour
@require_auth(permissions=["blockchain.read"])
async def get_wallet_balance(request: Request) -> JSONResponse:
"""
Get user wallet balance with caching for performance.
Args:
request: Sanic request object
Returns:
JSONResponse: Wallet balance information
"""
try:
user_id = request.ctx.user.id
cache_manager = get_cache_manager()
# Try cache first
balance_key = f"wallet_balance:{user_id}"
cached_balance = await cache_manager.get(balance_key)
if cached_balance:
return response.json({
"balance": cached_balance,
"cached": True,
"updated_at": cached_balance.get("updated_at")
})
async with get_async_session() as session:
# Get user wallet address
user_stmt = select(User).where(User.id == user_id)
user_result = await session.execute(user_stmt)
user = user_result.scalar_one_or_none()
if not user or not user.wallet_address:
return response.json(
{"error": "Wallet not configured", "code": "WALLET_NOT_CONFIGURED"},
status=400
)
# Get balance from TON service
ton_service = TONService()
balance_data = await ton_service.get_wallet_balance(user.wallet_address)
if balance_data.get("error"):
return response.json(
{"error": balance_data["error"], "code": "BALANCE_FETCH_FAILED"},
status=500
)
# Cache balance for 5 minutes
balance_response = {
"address": user.wallet_address,
"balance_nanotons": balance_data["balance"],
"balance_tons": str(Decimal(balance_data["balance"]) / Decimal("1000000000")),
"last_transaction_lt": balance_data.get("last_transaction_lt"),
"updated_at": datetime.utcnow().isoformat()
}
await cache_manager.set(balance_key, balance_response, ttl=300)
await logger.ainfo(
"Wallet balance retrieved",
user_id=str(user_id),
address=user.wallet_address,
balance=balance_data["balance"]
)
return response.json({
"balance": balance_response,
"cached": False
})
except Exception as e:
await logger.aerror(
"Failed to get wallet balance",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Failed to get balance", "code": "BALANCE_FAILED"},
status=500
)
@blockchain_bp.route("/wallet/transactions", methods=["GET"])
@rate_limit(limit=50, window=3600) # 50 transaction history requests per hour
@require_auth(permissions=["blockchain.read"])
async def get_wallet_transactions(request: Request) -> JSONResponse:
"""
Get wallet transaction history with pagination.
Args:
request: Sanic request object
Returns:
JSONResponse: Transaction history
"""
try:
user_id = request.ctx.user.id
# Parse query parameters
limit = min(int(request.args.get("limit", 20)), 100) # Max 100 transactions
offset = max(int(request.args.get("offset", 0)), 0)
async with get_async_session() as session:
# Get user wallet address
user_stmt = select(User).where(User.id == user_id)
user_result = await session.execute(user_stmt)
user = user_result.scalar_one_or_none()
if not user or not user.wallet_address:
return response.json(
{"error": "Wallet not configured", "code": "WALLET_NOT_CONFIGURED"},
status=400
)
# Check cache for recent transactions
cache_manager = get_cache_manager()
cache_key = f"wallet_transactions:{user_id}:{limit}:{offset}"
cached_transactions = await cache_manager.get(cache_key)
if cached_transactions:
return response.json({
"transactions": cached_transactions,
"cached": True
})
# Get transactions from TON service
ton_service = TONService()
transactions_data = await ton_service.get_wallet_transactions(
user.wallet_address,
limit=limit,
offset=offset
)
if transactions_data.get("error"):
return response.json(
{"error": transactions_data["error"], "code": "TRANSACTIONS_FETCH_FAILED"},
status=500
)
# Process and format transactions
formatted_transactions = []
for tx in transactions_data.get("transactions", []):
formatted_tx = {
"hash": tx.get("hash"),
"lt": tx.get("lt"),
"timestamp": tx.get("utime"),
"value": tx.get("value", "0"),
"value_tons": str(Decimal(tx.get("value", "0")) / Decimal("1000000000")),
"fee": tx.get("fee", "0"),
"source": tx.get("in_msg", {}).get("source"),
"destination": tx.get("out_msgs", [{}])[0].get("destination"),
"message": tx.get("in_msg", {}).get("message", ""),
"type": "incoming" if tx.get("in_msg") else "outgoing",
"status": "success" if tx.get("success") else "failed"
}
formatted_transactions.append(formatted_tx)
# Cache for 2 minutes
await cache_manager.set(cache_key, formatted_transactions, ttl=120)
return response.json({
"transactions": formatted_transactions,
"total": len(formatted_transactions),
"limit": limit,
"offset": offset,
"cached": False
})
except Exception as e:
await logger.aerror(
"Failed to get wallet transactions",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Failed to get transactions", "code": "TRANSACTIONS_FAILED"},
status=500
)
@blockchain_bp.route("/transaction/send", methods=["POST"])
@rate_limit(limit=10, window=3600) # 10 transactions per hour
@require_auth(permissions=["blockchain.write"])
@validate_request(BlockchainTransactionSchema)
async def send_transaction(request: Request) -> JSONResponse:
"""
Send TON transaction with comprehensive validation and monitoring.
Args:
request: Sanic request with transaction data
Returns:
JSONResponse: Transaction submission result
"""
try:
user_id = request.ctx.user.id
data = request.json
async with get_async_session() as session:
# Get user with wallet
user_stmt = select(User).where(User.id == user_id)
user_result = await session.execute(user_stmt)
user = user_result.scalar_one_or_none()
if not user or not user.wallet_address or not user.wallet_private_key:
return response.json(
{"error": "Wallet not properly configured", "code": "WALLET_INCOMPLETE"},
status=400
)
# Validate transaction limits
amount_nanotons = data.get("amount", 0)
max_transaction = settings.MAX_TRANSACTION_AMOUNT * 1000000000 # Convert to nanotons
if amount_nanotons > max_transaction:
return response.json(
{"error": f"Amount exceeds maximum allowed ({settings.MAX_TRANSACTION_AMOUNT} TON)",
"code": "AMOUNT_EXCEEDED"},
status=400
)
# Check daily transaction limit
cache_manager = get_cache_manager()
daily_limit_key = f"daily_transactions:{user_id}:{datetime.utcnow().date()}"
daily_amount = await cache_manager.get(daily_limit_key, default=0)
if daily_amount + amount_nanotons > settings.DAILY_TRANSACTION_LIMIT * 1000000000:
return response.json(
{"error": "Daily transaction limit exceeded", "code": "DAILY_LIMIT_EXCEEDED"},
status=429
)
# Prepare transaction
transaction_data = {
"transaction_type": data["transaction_type"],
"recipient_address": data.get("recipient_address"),
"amount": amount_nanotons,
"message": data.get("message", ""),
"sender_address": user.wallet_address
}
# Send transaction via TON service
ton_service = TONService()
tx_result = await ton_service.send_transaction(
private_key=user.wallet_private_key,
**transaction_data
)
if tx_result.get("error"):
await logger.awarning(
"Transaction failed",
user_id=str(user_id),
error=tx_result["error"],
**transaction_data
)
return response.json(
{"error": tx_result["error"], "code": "TRANSACTION_FAILED"},
status=400
)
# Update daily limit counter
await cache_manager.increment(daily_limit_key, amount_nanotons, ttl=86400)
# Store transaction record
from app.core.models.blockchain import BlockchainTransaction
async with get_async_session() as session:
tx_record = BlockchainTransaction(
id=uuid4(),
user_id=user_id,
transaction_hash=tx_result["hash"],
transaction_type=data["transaction_type"],
amount=amount_nanotons,
recipient_address=data.get("recipient_address"),
sender_address=user.wallet_address,
message=data.get("message", ""),
status="pending",
network_fee=tx_result.get("fee", 0),
block_hash=tx_result.get("block_hash"),
logical_time=tx_result.get("lt")
)
session.add(tx_record)
await session.commit()
# Clear balance cache
balance_key = f"wallet_balance:{user_id}"
await cache_manager.delete(balance_key)
await logger.ainfo(
"Transaction sent successfully",
user_id=str(user_id),
transaction_hash=tx_result["hash"],
amount=amount_nanotons,
recipient=data.get("recipient_address")
)
return response.json({
"message": "Transaction sent successfully",
"transaction": {
"hash": tx_result["hash"],
"amount": amount_nanotons,
"amount_tons": str(Decimal(amount_nanotons) / Decimal("1000000000")),
"recipient": data.get("recipient_address"),
"fee": tx_result.get("fee", 0),
"status": "pending",
"timestamp": datetime.utcnow().isoformat()
}
}, status=201)
except Exception as e:
await logger.aerror(
"Failed to send transaction",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Failed to send transaction", "code": "SEND_FAILED"},
status=500
)
@blockchain_bp.route("/transaction/<tx_hash>/status", methods=["GET"])
@rate_limit(limit=100, window=3600) # 100 status checks per hour
@require_auth(permissions=["blockchain.read"])
async def get_transaction_status(request: Request, tx_hash: str) -> JSONResponse:
"""
Get transaction status and confirmation details.
Args:
request: Sanic request object
tx_hash: Transaction hash to check
Returns:
JSONResponse: Transaction status information
"""
try:
user_id = request.ctx.user.id
# Check cache first
cache_manager = get_cache_manager()
status_key = f"tx_status:{tx_hash}"
cached_status = await cache_manager.get(status_key)
if cached_status and cached_status.get("status") in ["confirmed", "failed"]:
# Cache confirmed/failed transactions longer
return response.json(cached_status)
# Get transaction from database
async with get_async_session() as session:
from app.core.models.blockchain import BlockchainTransaction
tx_stmt = select(BlockchainTransaction).where(
and_(
BlockchainTransaction.transaction_hash == tx_hash,
BlockchainTransaction.user_id == user_id
)
)
tx_result = await session.execute(tx_stmt)
tx_record = tx_result.scalar_one_or_none()
if not tx_record:
return response.json(
{"error": "Transaction not found", "code": "TRANSACTION_NOT_FOUND"},
status=404
)
# Get current status from blockchain
ton_service = TONService()
status_data = await ton_service.get_transaction_status(tx_hash)
if status_data.get("error"):
# Return database status if blockchain query fails
tx_status = {
"hash": tx_record.transaction_hash,
"status": tx_record.status,
"confirmations": 0,
"amount": tx_record.amount,
"created_at": tx_record.created_at.isoformat(),
"blockchain_error": status_data["error"]
}
else:
# Update status based on blockchain data
new_status = "confirmed" if status_data.get("confirmed") else "pending"
if status_data.get("failed"):
new_status = "failed"
tx_status = {
"hash": tx_record.transaction_hash,
"status": new_status,
"confirmations": status_data.get("confirmations", 0),
"block_hash": status_data.get("block_hash"),
"block_time": status_data.get("block_time"),
"amount": tx_record.amount,
"fee": status_data.get("fee", tx_record.network_fee),
"created_at": tx_record.created_at.isoformat(),
"confirmed_at": status_data.get("confirmed_at")
}
# Update database record if status changed
if tx_record.status != new_status:
async with get_async_session() as session:
update_stmt = (
update(BlockchainTransaction)
.where(BlockchainTransaction.id == tx_record.id)
.values(
status=new_status,
confirmations=status_data.get("confirmations", 0),
confirmed_at=datetime.fromisoformat(status_data["confirmed_at"])
if status_data.get("confirmed_at") else None
)
)
await session.execute(update_stmt)
await session.commit()
# Cache status (longer for final states)
cache_ttl = 300 if tx_status["status"] == "pending" else 3600 # 5 min vs 1 hour
await cache_manager.set(status_key, tx_status, ttl=cache_ttl)
return response.json(tx_status)
except Exception as e:
await logger.aerror(
"Failed to get transaction status",
user_id=str(request.ctx.user.id),
tx_hash=tx_hash,
error=str(e)
)
return response.json(
{"error": "Failed to get transaction status", "code": "STATUS_FAILED"},
status=500
)
@blockchain_bp.route("/wallet/create", methods=["POST"])
@rate_limit(limit=1, window=86400) # 1 wallet creation per day
@require_auth(permissions=["blockchain.wallet.create"])
async def create_wallet(request: Request) -> JSONResponse:
"""
Create new TON wallet for user (one per user).
Args:
request: Sanic request object
Returns:
JSONResponse: Wallet creation result
"""
try:
user_id = request.ctx.user.id
async with get_async_session() as session:
# Check if user already has a wallet
user_stmt = select(User).where(User.id == user_id)
user_result = await session.execute(user_stmt)
user = user_result.scalar_one_or_none()
if not user:
return response.json(
{"error": "User not found", "code": "USER_NOT_FOUND"},
status=404
)
if user.wallet_address:
return response.json(
{"error": "Wallet already exists", "code": "WALLET_EXISTS"},
status=400
)
# Create wallet via TON service
ton_service = TONService()
wallet_data = await ton_service.create_wallet()
if wallet_data.get("error"):
return response.json(
{"error": wallet_data["error"], "code": "WALLET_CREATION_FAILED"},
status=500
)
# Store wallet information (encrypt private key)
from app.core.security import encrypt_data
encrypted_private_key = encrypt_data(
wallet_data["private_key"],
context=f"wallet:{user_id}"
)
user.wallet_address = wallet_data["address"]
user.wallet_private_key = encrypted_private_key
user.wallet_created_at = datetime.utcnow()
await session.commit()
await logger.ainfo(
"Wallet created successfully",
user_id=str(user_id),
wallet_address=wallet_data["address"]
)
return response.json({
"message": "Wallet created successfully",
"wallet": {
"address": wallet_data["address"],
"created_at": datetime.utcnow().isoformat(),
"balance": "0",
"network": "TON"
},
"security_note": "Private key is encrypted and stored securely. Keep your account secure."
}, status=201)
except Exception as e:
await logger.aerror(
"Failed to create wallet",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Failed to create wallet", "code": "WALLET_FAILED"},
status=500
)
@blockchain_bp.route("/stats", methods=["GET"])
@rate_limit(limit=50, window=3600) # 50 stats requests per hour
@require_auth(permissions=["blockchain.read"])
async def get_blockchain_stats(request: Request) -> JSONResponse:
"""
Get user blockchain activity statistics.
Args:
request: Sanic request object
Returns:
JSONResponse: Blockchain activity statistics
"""
try:
user_id = request.ctx.user.id
async with get_async_session() as session:
from sqlalchemy import func
from app.core.models.blockchain import BlockchainTransaction
# Get transaction statistics
stats_stmt = select(
func.count(BlockchainTransaction.id).label('total_transactions'),
func.sum(BlockchainTransaction.amount).label('total_amount'),
func.sum(BlockchainTransaction.network_fee).label('total_fees')
).where(BlockchainTransaction.user_id == user_id)
stats_result = await session.execute(stats_stmt)
stats = stats_result.first()
# Get transactions by type
type_stats_stmt = select(
BlockchainTransaction.transaction_type,
func.count(BlockchainTransaction.id).label('count'),
func.sum(BlockchainTransaction.amount).label('amount')
).where(
BlockchainTransaction.user_id == user_id
).group_by(BlockchainTransaction.transaction_type)
type_result = await session.execute(type_stats_stmt)
type_stats = {
row.transaction_type: {
'count': row.count,
'total_amount': row.amount or 0
}
for row in type_result
}
# Get recent activity (last 30 days)
recent_date = datetime.utcnow() - timedelta(days=30)
recent_stmt = select(
func.count(BlockchainTransaction.id).label('recent_count'),
func.sum(BlockchainTransaction.amount).label('recent_amount')
).where(
and_(
BlockchainTransaction.user_id == user_id,
BlockchainTransaction.created_at >= recent_date
)
)
recent_result = await session.execute(recent_stmt)
recent_stats = recent_result.first()
blockchain_stats = {
"total_transactions": stats.total_transactions or 0,
"total_amount_nanotons": stats.total_amount or 0,
"total_amount_tons": str(Decimal(stats.total_amount or 0) / Decimal("1000000000")),
"total_fees_nanotons": stats.total_fees or 0,
"total_fees_tons": str(Decimal(stats.total_fees or 0) / Decimal("1000000000")),
"by_type": type_stats,
"recent_activity": {
"transactions_30d": recent_stats.recent_count or 0,
"amount_30d_nanotons": recent_stats.recent_amount or 0,
"amount_30d_tons": str(Decimal(recent_stats.recent_amount or 0) / Decimal("1000000000"))
},
"generated_at": datetime.utcnow().isoformat()
}
return response.json(blockchain_stats)
except Exception as e:
await logger.aerror(
"Failed to get blockchain stats",
user_id=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Failed to get blockchain statistics", "code": "STATS_FAILED"},
status=500
)
+591
View File
@@ -0,0 +1,591 @@
"""
Enhanced content management routes with async operations and comprehensive validation.
Provides secure upload, download, metadata management with Redis caching.
"""
import asyncio
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any
from uuid import UUID, uuid4
from sanic import Blueprint, Request, response
from sanic.response import JSONResponse, ResponseStream
from sqlalchemy import select, update, delete, and_, or_
from sqlalchemy.orm import selectinload
from app.core.config import get_settings
from app.core.database import get_async_session, get_cache_manager
from app.core.logging import get_logger
from app.core.models.content import Content, ContentMetadata, ContentAccess, License
from app.core.models.user import User
from app.api.middleware import require_auth, validate_request, rate_limit
from app.core.validation import ContentSchema, ContentUpdateSchema, ContentSearchSchema
from app.core.storage import StorageManager
from app.core.security import encrypt_data, decrypt_data, generate_access_token
# Initialize blueprint
content_bp = Blueprint("content", url_prefix="/api/v1/content")
logger = get_logger(__name__)
settings = get_settings()
@content_bp.route("/", methods=["POST"])
@rate_limit(limit=50, window=3600) # 50 uploads per hour
@require_auth(permissions=["content.create"])
@validate_request(ContentSchema)
async def create_content(request: Request) -> JSONResponse:
"""
Create new content with metadata and security validation.
Args:
request: Sanic request with validated content data
Returns:
JSONResponse: Created content information with upload URLs
"""
try:
data = request.json
user_id = request.ctx.user.id
async with get_async_session() as session:
# Check user upload quota
quota_key = f"user:{user_id}:upload_quota"
cache_manager = get_cache_manager()
current_quota = await cache_manager.get(quota_key, default=0)
if current_quota >= settings.MAX_UPLOADS_PER_DAY:
return response.json(
{"error": "Upload quota exceeded", "code": "QUOTA_EXCEEDED"},
status=429
)
# Create content record
content = Content(
id=uuid4(),
user_id=user_id,
title=data["title"],
description=data.get("description"),
content_type=data["content_type"],
file_size=data.get("file_size", 0),
status="pending",
visibility=data.get("visibility", "private"),
tags=data.get("tags", []),
license_id=data.get("license_id")
)
session.add(content)
# Create metadata if provided
if data.get("metadata"):
metadata = ContentMetadata(
content_id=content.id,
metadata_type="custom",
data=data["metadata"]
)
session.add(metadata)
await session.commit()
await session.refresh(content)
# Update quota counter
await cache_manager.increment(quota_key, ttl=86400) # 24 hours
# Generate upload URLs for chunked upload
storage_manager = StorageManager()
upload_info = await storage_manager.create_upload_session(
content.id, data.get("file_size", 0)
)
# Cache content for quick access
content_cache_key = f"content:{content.id}"
await cache_manager.set(
content_cache_key,
{
"id": str(content.id),
"title": content.title,
"status": content.status,
"user_id": str(content.user_id)
},
ttl=3600
)
await logger.ainfo(
"Content created successfully",
content_id=str(content.id),
user_id=str(user_id),
title=content.title
)
return response.json({
"content_id": str(content.id),
"upload_session": upload_info,
"status": content.status,
"created_at": content.created_at.isoformat()
}, status=201)
except Exception as e:
await logger.aerror(
"Failed to create content",
error=str(e),
user_id=str(user_id)
)
return response.json(
{"error": "Failed to create content", "code": "CREATION_FAILED"},
status=500
)
@content_bp.route("/<content_id:uuid>", methods=["GET"])
@rate_limit(limit=200, window=3600) # 200 requests per hour
@require_auth(permissions=["content.read"])
async def get_content(request: Request, content_id: UUID) -> JSONResponse:
"""
Retrieve content information with access control and caching.
Args:
request: Sanic request object
content_id: UUID of the content to retrieve
Returns:
JSONResponse: Content information or error
"""
try:
user_id = request.ctx.user.id
cache_manager = get_cache_manager()
# Try cache first
cache_key = f"content:{content_id}:full"
cached_content = await cache_manager.get(cache_key)
if cached_content:
# Check access permissions from cache
if await _check_content_access(content_id, user_id, "read"):
return response.json(cached_content)
else:
return response.json(
{"error": "Access denied", "code": "ACCESS_DENIED"},
status=403
)
async with get_async_session() as session:
# Load content with relationships
stmt = (
select(Content)
.options(
selectinload(Content.metadata),
selectinload(Content.access_controls),
selectinload(Content.license)
)
.where(Content.id == content_id)
)
result = await session.execute(stmt)
content = result.scalar_one_or_none()
if not content:
return response.json(
{"error": "Content not found", "code": "NOT_FOUND"},
status=404
)
# Check access permissions
if not await _check_content_access_db(session, content, user_id, "read"):
return response.json(
{"error": "Access denied", "code": "ACCESS_DENIED"},
status=403
)
# Prepare response data
content_data = {
"id": str(content.id),
"title": content.title,
"description": content.description,
"content_type": content.content_type,
"file_size": content.file_size,
"status": content.status,
"visibility": content.visibility,
"tags": content.tags,
"created_at": content.created_at.isoformat(),
"updated_at": content.updated_at.isoformat(),
"metadata": [
{
"type": m.metadata_type,
"data": m.data
} for m in content.metadata
],
"license": {
"name": content.license.name,
"description": content.license.description
} if content.license else None
}
# Cache the result
await cache_manager.set(cache_key, content_data, ttl=1800) # 30 minutes
# Update access statistics
await _update_access_stats(content_id, user_id, "view")
return response.json(content_data)
except Exception as e:
await logger.aerror(
"Failed to retrieve content",
content_id=str(content_id),
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Failed to retrieve content", "code": "RETRIEVAL_FAILED"},
status=500
)
@content_bp.route("/<content_id:uuid>", methods=["PUT"])
@rate_limit(limit=100, window=3600) # 100 updates per hour
@require_auth(permissions=["content.update"])
@validate_request(ContentUpdateSchema)
async def update_content(request: Request, content_id: UUID) -> JSONResponse:
"""
Update content metadata and settings with validation.
Args:
request: Sanic request with update data
content_id: UUID of content to update
Returns:
JSONResponse: Updated content information
"""
try:
data = request.json
user_id = request.ctx.user.id
async with get_async_session() as session:
# Load existing content
stmt = select(Content).where(Content.id == content_id)
result = await session.execute(stmt)
content = result.scalar_one_or_none()
if not content:
return response.json(
{"error": "Content not found", "code": "NOT_FOUND"},
status=404
)
# Check update permissions
if not await _check_content_access_db(session, content, user_id, "update"):
return response.json(
{"error": "Access denied", "code": "ACCESS_DENIED"},
status=403
)
# Update fields
for field, value in data.items():
if hasattr(content, field) and field not in ["id", "user_id", "created_at"]:
setattr(content, field, value)
content.updated_at = datetime.utcnow()
await session.commit()
# Invalidate caches
cache_manager = get_cache_manager()
await cache_manager.delete(f"content:{content_id}")
await cache_manager.delete(f"content:{content_id}:full")
await logger.ainfo(
"Content updated successfully",
content_id=str(content_id),
user_id=str(user_id),
updated_fields=list(data.keys())
)
return response.json({
"content_id": str(content_id),
"status": "updated",
"updated_at": content.updated_at.isoformat()
})
except Exception as e:
await logger.aerror(
"Failed to update content",
content_id=str(content_id),
error=str(e)
)
return response.json(
{"error": "Failed to update content", "code": "UPDATE_FAILED"},
status=500
)
@content_bp.route("/search", methods=["POST"])
@rate_limit(limit=100, window=3600) # 100 searches per hour
@require_auth(permissions=["content.read"])
@validate_request(ContentSearchSchema)
async def search_content(request: Request) -> JSONResponse:
"""
Search content with filters, pagination and caching.
Args:
request: Sanic request with search parameters
Returns:
JSONResponse: Search results with pagination
"""
try:
data = request.json
user_id = request.ctx.user.id
# Build cache key from search parameters
search_key = f"search:{hash(str(sorted(data.items())))}:{user_id}"
cache_manager = get_cache_manager()
# Try cache first
cached_results = await cache_manager.get(search_key)
if cached_results:
return response.json(cached_results)
async with get_async_session() as session:
# Build base query
stmt = select(Content).where(
or_(
Content.visibility == "public",
Content.user_id == user_id
)
)
# Apply filters
if data.get("query"):
query = f"%{data['query']}%"
stmt = stmt.where(
or_(
Content.title.ilike(query),
Content.description.ilike(query)
)
)
if data.get("content_type"):
stmt = stmt.where(Content.content_type == data["content_type"])
if data.get("tags"):
for tag in data["tags"]:
stmt = stmt.where(Content.tags.contains([tag]))
if data.get("status"):
stmt = stmt.where(Content.status == data["status"])
# Apply date filters
if data.get("date_from"):
stmt = stmt.where(Content.created_at >= datetime.fromisoformat(data["date_from"]))
if data.get("date_to"):
stmt = stmt.where(Content.created_at <= datetime.fromisoformat(data["date_to"]))
# Apply pagination
page = data.get("page", 1)
per_page = min(data.get("per_page", 20), 100) # Max 100 items per page
offset = (page - 1) * per_page
# Get total count
from sqlalchemy import func
count_stmt = select(func.count(Content.id)).select_from(stmt.subquery())
total_result = await session.execute(count_stmt)
total = total_result.scalar()
# Apply ordering and pagination
if data.get("sort_by") == "created_at":
stmt = stmt.order_by(Content.created_at.desc())
elif data.get("sort_by") == "title":
stmt = stmt.order_by(Content.title.asc())
else:
stmt = stmt.order_by(Content.updated_at.desc())
stmt = stmt.offset(offset).limit(per_page)
# Execute query
result = await session.execute(stmt)
content_list = result.scalars().all()
# Prepare response
search_results = {
"results": [
{
"id": str(content.id),
"title": content.title,
"description": content.description,
"content_type": content.content_type,
"file_size": content.file_size,
"status": content.status,
"visibility": content.visibility,
"tags": content.tags,
"created_at": content.created_at.isoformat()
} for content in content_list
],
"pagination": {
"page": page,
"per_page": per_page,
"total": total,
"pages": (total + per_page - 1) // per_page
}
}
# Cache results for 5 minutes
await cache_manager.set(search_key, search_results, ttl=300)
return response.json(search_results)
except Exception as e:
await logger.aerror(
"Search failed",
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Search failed", "code": "SEARCH_FAILED"},
status=500
)
@content_bp.route("/<content_id:uuid>/download", methods=["GET"])
@rate_limit(limit=50, window=3600) # 50 downloads per hour
@require_auth(permissions=["content.download"])
async def download_content(request: Request, content_id: UUID) -> ResponseStream:
"""
Secure content download with access control and logging.
Args:
request: Sanic request object
content_id: UUID of content to download
Returns:
ResponseStream: File stream or error response
"""
try:
user_id = request.ctx.user.id
async with get_async_session() as session:
# Load content
stmt = select(Content).where(Content.id == content_id)
result = await session.execute(stmt)
content = result.scalar_one_or_none()
if not content:
return response.json(
{"error": "Content not found", "code": "NOT_FOUND"},
status=404
)
# Check download permissions
if not await _check_content_access_db(session, content, user_id, "download"):
return response.json(
{"error": "Access denied", "code": "ACCESS_DENIED"},
status=403
)
# Generate download token
download_token = generate_access_token(
{"content_id": str(content_id), "user_id": str(user_id)},
expires_in=3600 # 1 hour
)
# Log download activity
await _update_access_stats(content_id, user_id, "download")
# Get storage manager and create download stream
storage_manager = StorageManager()
file_stream = await storage_manager.get_file_stream(content.file_path)
await logger.ainfo(
"Content download initiated",
content_id=str(content_id),
user_id=str(user_id),
filename=content.title
)
return await response.stream(
file_stream,
headers={
"Content-Type": content.content_type or "application/octet-stream",
"Content-Disposition": f'attachment; filename="{content.title}"',
"Content-Length": str(content.file_size),
"X-Download-Token": download_token
}
)
except Exception as e:
await logger.aerror(
"Download failed",
content_id=str(content_id),
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Download failed", "code": "DOWNLOAD_FAILED"},
status=500
)
async def _check_content_access(content_id: UUID, user_id: UUID, action: str) -> bool:
"""Check user access to content from cache or database."""
cache_manager = get_cache_manager()
access_key = f"access:{content_id}:{user_id}:{action}"
cached_access = await cache_manager.get(access_key)
if cached_access is not None:
return cached_access
async with get_async_session() as session:
stmt = select(Content).where(Content.id == content_id)
result = await session.execute(stmt)
content = result.scalar_one_or_none()
if not content:
return False
has_access = await _check_content_access_db(session, content, user_id, action)
# Cache result for 5 minutes
await cache_manager.set(access_key, has_access, ttl=300)
return has_access
async def _check_content_access_db(session, content: Content, user_id: UUID, action: str) -> bool:
"""Check user access to content in database."""
# Content owner always has access
if content.user_id == user_id:
return True
# Public content allows read access
if content.visibility == "public" and action in ["read", "view"]:
return True
# Check explicit access controls
stmt = (
select(ContentAccess)
.where(
and_(
ContentAccess.content_id == content.id,
ContentAccess.user_id == user_id,
ContentAccess.permission == action,
ContentAccess.expires_at > datetime.utcnow()
)
)
)
result = await session.execute(stmt)
access_control = result.scalar_one_or_none()
return access_control is not None
async def _update_access_stats(content_id: UUID, user_id: UUID, action: str) -> None:
"""Update content access statistics."""
try:
cache_manager = get_cache_manager()
# Update daily stats
today = datetime.utcnow().date().isoformat()
stats_key = f"stats:{content_id}:{action}:{today}"
await cache_manager.increment(stats_key, ttl=86400)
# Update user activity
user_activity_key = f"activity:{user_id}:{action}:{today}"
await cache_manager.increment(user_activity_key, ttl=86400)
except Exception as e:
await logger.awarning(
"Failed to update access stats",
content_id=str(content_id),
user_id=str(user_id),
action=action,
error=str(e)
)
+226
View File
@@ -0,0 +1,226 @@
"""Health check and system status endpoints."""
import logging
import asyncio
from datetime import datetime
from typing import Dict, Any
from sanic import Blueprint, Request, response
from sanic.response import JSONResponse
from app.core.config import get_settings
from app.core.database import get_async_session
from app.core.metrics import get_metrics, get_metrics_content_type, metrics_collector
from app.core.background.indexer_service import indexer_service
from app.core.background.convert_service import convert_service
from app.core.background.ton_service import ton_service
logger = logging.getLogger(__name__)
health_bp = Blueprint("health", version=1)
@health_bp.route("/health", methods=["GET"])
async def health_check(request: Request) -> JSONResponse:
"""Basic health check endpoint."""
return response.json({
"status": "healthy",
"timestamp": datetime.utcnow().isoformat(),
"service": "my-uploader-bot",
"version": "2.0.0"
})
@health_bp.route("/health/detailed", methods=["GET"])
async def detailed_health_check(request: Request) -> JSONResponse:
"""Detailed health check with component status."""
health_status = {
"status": "healthy",
"timestamp": datetime.utcnow().isoformat(),
"service": "my-uploader-bot",
"version": "2.0.0",
"components": {}
}
overall_healthy = True
# Database health
try:
async with get_async_session() as session:
await session.execute("SELECT 1")
health_status["components"]["database"] = {
"status": "healthy",
"message": "Database connection successful"
}
except Exception as e:
health_status["components"]["database"] = {
"status": "unhealthy",
"message": f"Database error: {str(e)}"
}
overall_healthy = False
# Redis health
try:
import redis.asyncio as redis
settings = get_settings()
redis_client = redis.from_url(settings.redis_url)
await redis_client.ping()
await redis_client.close()
health_status["components"]["cache"] = {
"status": "healthy",
"message": "Redis connection successful"
}
except Exception as e:
health_status["components"]["cache"] = {
"status": "unhealthy",
"message": f"Redis error: {str(e)}"
}
overall_healthy = False
# TON service health
try:
# Check if TON service is responsive
test_result = await ton_service.ping()
health_status["components"]["blockchain"] = {
"status": "healthy" if test_result else "degraded",
"message": "TON service available" if test_result else "TON service degraded"
}
if not test_result:
overall_healthy = False
except Exception as e:
health_status["components"]["blockchain"] = {
"status": "unhealthy",
"message": f"TON service error: {str(e)}"
}
overall_healthy = False
# Background services health
health_status["components"]["background_services"] = {
"indexer": {
"status": "healthy" if indexer_service.is_running else "stopped",
"active_tasks": len([t for t in indexer_service.tasks if not t.done()])
},
"converter": {
"status": "healthy" if convert_service.is_running else "stopped",
"active_tasks": len([t for t in convert_service.tasks if not t.done()])
}
}
# Update overall status
if not overall_healthy:
health_status["status"] = "unhealthy"
status_code = 200 if overall_healthy else 503
return response.json(health_status, status=status_code)
@health_bp.route("/health/ready", methods=["GET"])
async def readiness_check(request: Request) -> JSONResponse:
"""Kubernetes readiness probe endpoint."""
try:
# Quick database check
async with get_async_session() as session:
await session.execute("SELECT 1")
return response.json({
"status": "ready",
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
return response.json({
"status": "not_ready",
"error": str(e),
"timestamp": datetime.utcnow().isoformat()
}, status=503)
@health_bp.route("/health/live", methods=["GET"])
async def liveness_check(request: Request) -> JSONResponse:
"""Kubernetes liveness probe endpoint."""
return response.json({
"status": "alive",
"timestamp": datetime.utcnow().isoformat()
})
@health_bp.route("/metrics", methods=["GET"])
async def prometheus_metrics(request: Request):
"""Prometheus metrics endpoint."""
try:
metrics_data = await get_metrics()
return response.raw(
metrics_data,
content_type=get_metrics_content_type()
)
except Exception as e:
logger.error(f"Error generating metrics: {e}")
return response.json({
"error": "Failed to generate metrics"
}, status=500)
@health_bp.route("/stats", methods=["GET"])
async def system_stats(request: Request) -> JSONResponse:
"""System statistics endpoint."""
try:
stats = {
"timestamp": datetime.utcnow().isoformat(),
"uptime": metrics_collector.start_time,
"services": {}
}
# Get indexer stats
try:
indexer_stats = await indexer_service.get_indexing_stats()
stats["services"]["indexer"] = indexer_stats
except Exception as e:
stats["services"]["indexer"] = {"error": str(e)}
# Get converter stats
try:
converter_stats = await convert_service.get_processing_stats()
stats["services"]["converter"] = converter_stats
except Exception as e:
stats["services"]["converter"] = {"error": str(e)}
return response.json(stats)
except Exception as e:
logger.error(f"Error getting system stats: {e}")
return response.json({
"error": "Failed to get system stats"
}, status=500)
@health_bp.route("/debug/info", methods=["GET"])
async def debug_info(request: Request) -> JSONResponse:
"""Debug information endpoint (development only)."""
settings = get_settings()
if settings.environment != "development":
return response.json({
"error": "Debug endpoint only available in development"
}, status=403)
debug_data = {
"timestamp": datetime.utcnow().isoformat(),
"environment": settings.environment,
"debug_mode": settings.debug,
"database_url": settings.database_url.replace(
settings.database_url.split('@')[0].split('//')[1],
"***:***"
) if '@' in settings.database_url else "***",
"redis_url": settings.redis_url.replace(
settings.redis_url.split('@')[0].split('//')[1],
"***:***"
) if '@' in settings.redis_url else "***",
"storage_backend": settings.storage_backend,
"ton_network": settings.ton_network,
"active_tasks": {
"indexer": len([t for t in indexer_service.tasks if not t.done()]),
"converter": len([t for t in convert_service.tasks if not t.done()])
}
}
return response.json(debug_data)
+379
View File
@@ -0,0 +1,379 @@
"""MY Network Monitoring Interface - веб-интерфейс мониторинга сети в хакерском стиле."""
import asyncio
import json
import logging
from datetime import datetime, timedelta
from typing import Dict, List, Any
from fastapi import APIRouter, Request, HTTPException
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
from pathlib import Path
logger = logging.getLogger(__name__)
# Создать router для мониторинга
router = APIRouter(prefix="/api/my/monitor", tags=["MY Network Monitoring"])
# Настроить шаблоны
templates_dir = Path(__file__).parent.parent.parent / "templates"
templates_dir.mkdir(exist_ok=True)
templates = Jinja2Templates(directory=str(templates_dir))
def get_node_service():
"""Получить сервис ноды."""
try:
from app.core.my_network.node_service import get_node_service
return get_node_service()
except Exception as e:
logger.error(f"Error getting node service: {e}")
return None
@router.get("/", response_class=HTMLResponse)
async def monitoring_dashboard(request: Request):
"""Главная страница мониторинга MY Network."""
try:
# Получить данные для дашборда
node_service = get_node_service()
if not node_service:
monitoring_data = {
"status": "offline",
"error": "MY Network service not available"
}
else:
# Собрать данные со всех компонентов
node_info = await node_service.get_node_info()
peers_info = await node_service.get_peers_info()
sync_status = await node_service.sync_manager.get_sync_status()
monitoring_data = {
"status": "online",
"node_info": node_info,
"peers_info": peers_info,
"sync_status": sync_status,
"timestamp": datetime.utcnow().isoformat()
}
return templates.TemplateResponse("my_network_monitor.html", {
"request": request,
"monitoring_data": monitoring_data
})
except Exception as e:
logger.error(f"Error rendering monitoring dashboard: {e}")
# Fallback HTML если шаблоны не работают
return HTMLResponse(content=generate_fallback_html(str(e)))
@router.get("/ascii")
async def get_ascii_status():
"""Получить ASCII статус сети."""
try:
node_service = get_node_service()
if not node_service:
return {"ascii": generate_offline_ascii(), "status": "offline"}
# Получить данные
node_info = await node_service.get_node_info()
peers_info = await node_service.get_peers_info()
sync_status = await node_service.sync_manager.get_sync_status()
# Генерировать ASCII
ascii_art = await generate_network_ascii(node_info, peers_info, sync_status)
return {
"ascii": ascii_art,
"status": "online",
"timestamp": datetime.utcnow().isoformat()
}
except Exception as e:
logger.error(f"Error generating ASCII status: {e}")
return {"ascii": generate_error_ascii(str(e)), "status": "error"}
@router.get("/live")
async def live_monitoring_data():
"""Получить живые данные для мониторинга."""
try:
node_service = get_node_service()
if not node_service:
raise HTTPException(status_code=503, detail="MY Network service unavailable")
# Получить свежие данные
node_info = await node_service.get_node_info()
peers_info = await node_service.get_peers_info()
sync_status = await node_service.sync_manager.get_sync_status()
# Статистика сети
network_stats = {
"connected_peers": peers_info["peer_count"],
"active_syncs": sync_status["active_syncs"],
"queue_size": sync_status["queue_size"],
"uptime": node_info["uptime"],
"status": node_info["status"]
}
return {
"success": True,
"data": {
"node_info": node_info,
"network_stats": network_stats,
"peers": peers_info["peers"][:10], # Показать только первые 10 пиров
"sync_status": sync_status
},
"timestamp": datetime.utcnow().isoformat()
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error getting live monitoring data: {e}")
raise HTTPException(status_code=500, detail=str(e))
async def generate_network_ascii(node_info: Dict[str, Any], peers_info: Dict[str, Any], sync_status: Dict[str, Any]) -> str:
"""Генерировать ASCII представление состояния сети."""
ascii_parts = []
# Заголовок
ascii_parts.append("""
╔══════════════════════════════════════════════════════════════════════════════╗
║ MY NETWORK v2.0 ║
║ Distributed Content Protocol ║
╚══════════════════════════════════════════════════════════════════════════════╝
""")
# Информация о ноде
status_indicator = "🟢" if node_info.get("status") == "running" else "🔴"
uptime_hours = int(node_info.get("uptime", 0) / 3600)
ascii_parts.append(f"""
┌─ NODE STATUS ────────────────────────────────────────────────────────────────┐
│ Node ID: {node_info.get('node_id', 'unknown')[:16]}... │
│ Status: {status_indicator} {node_info.get('status', 'unknown').upper()} │
│ Uptime: {uptime_hours}h {int((node_info.get('uptime', 0) % 3600) / 60)}m │
│ Version: MY Network {node_info.get('version', '2.0')} │
└──────────────────────────────────────────────────────────────────────────────┘
""")
# Информация о пирах
peer_count = peers_info.get("peer_count", 0)
peer_status = "🌐" if peer_count > 0 else "🏝️"
ascii_parts.append(f"""
┌─ NETWORK STATUS ─────────────────────────────────────────────────────────────┐
│ Connected Peers: {peer_status} {peer_count:>3} │
│ Known Nodes: {len(peers_info.get('peers', [])):>3} │
│ Network Health: {'CONNECTED' if peer_count > 0 else 'ISOLATED':>9} │
└──────────────────────────────────────────────────────────────────────────────┘
""")
# Статус синхронизации
sync_running = sync_status.get("is_running", False)
active_syncs = sync_status.get("active_syncs", 0)
queue_size = sync_status.get("queue_size", 0)
sync_indicator = "⚡" if sync_running else "⏸️"
ascii_parts.append(f"""
┌─ SYNC STATUS ────────────────────────────────────────────────────────────────┐
│ Sync Engine: {sync_indicator} {'RUNNING' if sync_running else 'STOPPED':>7} │
│ Active Syncs: {active_syncs:>3} │
│ Queue Size: {queue_size:>3} │
│ Workers: {sync_status.get('workers_count', 0):>3} │
└──────────────────────────────────────────────────────────────────────────────┘
""")
# Визуализация сети
if peer_count > 0:
ascii_parts.append(generate_network_topology(peers_info.get("peers", [])[:6]))
# Недавние события синхронизации
recent_syncs = sync_status.get("recent_syncs", [])
if recent_syncs:
ascii_parts.append(generate_sync_history(recent_syncs[-5:]))
# Подвал
current_time = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
ascii_parts.append(f"""
╔══════════════════════════════════════════════════════════════════════════════╗
║ Last Updated: {current_time} ║
║ MY Network Protocol - Decentralized Content Distribution System ║
╚══════════════════════════════════════════════════════════════════════════════╝
""")
return "".join(ascii_parts)
def generate_network_topology(peers: List[Dict[str, Any]]) -> str:
"""Генерировать ASCII топологию сети."""
topology = ["""
┌─ NETWORK TOPOLOGY ───────────────────────────────────────────────────────────┐
│ │
│ [THIS NODE] │
│ │ │"""]
if len(peers) == 1:
topology.append("│ │ │")
topology.append(f"│ [{peers[0].get('node_id', 'unknown')[:8]}...] │")
elif len(peers) <= 3:
topology.append("│ ┌───────┼───────┐ │")
for i, peer in enumerate(peers):
spaces = " " if i == 0 else (" " if i == 1 else " ")
topology.append(f"│{spaces}[{peer.get('node_id', 'unknown')[:8]}...] │")
else:
topology.append("│ ┌───────┬───────┼───────┬───────┐ │")
topology.append("│ │ │ │ │ │ │")
for i, peer in enumerate(peers[:5]):
if i < 5:
spaces = [" ", " ", " ", " ", " "][i]
topology.append(f"│{spaces}[{peer.get('node_id', 'unknown')[:6]}] │")
if len(peers) > 5:
topology.append("│ ... │")
topology.append("│ │")
topology.append("└──────────────────────────────────────────────────────────────────────────────┘")
return "\n".join(topology) + "\n"
def generate_sync_history(recent_syncs: List[Dict[str, Any]]) -> str:
"""Генерировать историю синхронизации."""
history = ["""
┌─ RECENT SYNC ACTIVITY ───────────────────────────────────────────────────────┐"""]
if not recent_syncs:
history.append("│ No recent sync activity │")
else:
for sync in recent_syncs:
content_hash = sync.get("content_hash", "unknown")[:12]
status = sync.get("status", "unknown")
status_icon = {"completed": "✅", "failed": "❌", "partial": "⚠️"}.get(status, "❓")
history.append(f"│ {status_icon} {content_hash}... - {status.upper():>9} │")
history.append("└──────────────────────────────────────────────────────────────────────────────┘")
return "\n".join(history) + "\n"
def generate_offline_ascii() -> str:
"""Генерировать ASCII для офлайн состояния."""
return """
╔══════════════════════════════════════════════════════════════════════════════╗
║ MY NETWORK v2.0 ║
║ Distributed Content Protocol ║
╚══════════════════════════════════════════════════════════════════════════════╝
┌─ SYSTEM STATUS ──────────────────────────────────────────────────────────────┐
│ │
│ 🔴 OFFLINE │
│ │
│ MY Network service is not available │
│ │
└──────────────────────────────────────────────────────────────────────────────┘
╔══════════════════════════════════════════════════════════════════════════════╗
║ Status: OFFLINE - Service not initialized ║
╚══════════════════════════════════════════════════════════════════════════════╝
"""
def generate_error_ascii(error_message: str) -> str:
"""Генерировать ASCII для ошибки."""
return f"""
╔══════════════════════════════════════════════════════════════════════════════╗
║ MY NETWORK v2.0 ║
║ Distributed Content Protocol ║
╚══════════════════════════════════════════════════════════════════════════════╝
┌─ ERROR STATE ────────────────────────────────────────────────────────────────┐
│ │
│ ❌ ERROR │
│ │
│ {error_message[:64]:^64} │
│ │
└──────────────────────────────────────────────────────────────────────────────┘
╔══════════════════════════════════════════════════════════════════════════════╗
║ Status: ERROR - Check system logs for details ║
╚══════════════════════════════════════════════════════════════════════════════╝
"""
def generate_fallback_html(error_message: str = "") -> str:
"""Генерировать fallback HTML если шаблоны не работают."""
return f'''
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>MY Network Monitor</title>
<style>
body {{
background: #000;
color: #0f0;
font-family: 'Courier New', monospace;
margin: 0;
padding: 20px;
overflow-x: auto;
}}
.container {{
max-width: 1200px;
margin: 0 auto;
}}
.ascii-art {{
white-space: pre;
font-size: 12px;
line-height: 1.2;
}}
.error {{
color: #f00;
text-align: center;
padding: 20px;
}}
.refresh-btn {{
background: #0f0;
color: #000;
border: none;
padding: 10px 20px;
font-family: inherit;
cursor: pointer;
margin: 20px 0;
}}
.refresh-btn:hover {{
background: #fff;
}}
</style>
</head>
<body>
<div class="container">
<div class="ascii-art">
{generate_error_ascii(error_message) if error_message else generate_offline_ascii()}
</div>
<button class="refresh-btn" onclick="location.reload()">REFRESH SYSTEM STATUS</button>
<div class="error">
{f"Error: {error_message}" if error_message else "MY Network service not available"}
</div>
</div>
<script>
// Автообновление каждые 30 секунд
setTimeout(() => location.reload(), 30000);
</script>
</body>
</html>
'''
+452
View File
@@ -0,0 +1,452 @@
"""MY Network Monitoring Sanic Blueprint - веб-интерфейс мониторинга сети."""
import asyncio
import json
import logging
from datetime import datetime, timedelta
from typing import Dict, List, Any
from pathlib import Path
from sanic import Blueprint, Request
from sanic.response import json as json_response, html as html_response
from sanic.exceptions import SanicException
from app.core.logging import get_logger
logger = get_logger(__name__)
# Создать blueprint для мониторинга
bp = Blueprint("my_monitoring", url_prefix="/api/my/monitor")
def get_node_service():
"""Получить сервис ноды."""
try:
from app.core.my_network.node_service import get_node_service
return get_node_service()
except Exception as e:
logger.error(f"Error getting node service: {e}")
return None
@bp.get("/")
async def monitoring_dashboard(request: Request):
"""Главная страница мониторинга MY Network."""
try:
# Получить данные для дашборда
node_service = get_node_service()
if not node_service:
monitoring_data = {
"status": "offline",
"error": "MY Network service not available"
}
else:
# Собрать данные со всех компонентов
node_info = await node_service.get_node_info()
peers_info = await node_service.get_peers_info()
sync_status = await node_service.sync_manager.get_sync_status()
monitoring_data = {
"status": "online",
"node_info": node_info,
"peers_info": peers_info,
"sync_status": sync_status,
"timestamp": datetime.utcnow().isoformat()
}
# Попробовать использовать шаблон
try:
from jinja2 import Environment, FileSystemLoader
# Настроить Jinja2
templates_dir = Path(__file__).parent.parent.parent / "templates"
if templates_dir.exists():
env = Environment(loader=FileSystemLoader(str(templates_dir)))
template = env.get_template("my_network_monitor.html")
html_content = template.render(monitoring_data=monitoring_data)
return html_response(html_content)
except Exception as e:
logger.warning(f"Template rendering failed: {e}")
# Fallback HTML если шаблоны не работают
return html_response(generate_fallback_html(monitoring_data))
except Exception as e:
logger.error(f"Error rendering monitoring dashboard: {e}")
return html_response(generate_fallback_html({"status": "error", "error": str(e)}))
@bp.get("/ascii")
async def get_ascii_status(request: Request):
"""Получить ASCII статус сети."""
try:
node_service = get_node_service()
if not node_service:
return json_response({"ascii": generate_offline_ascii(), "status": "offline"})
# Получить данные
node_info = await node_service.get_node_info()
peers_info = await node_service.get_peers_info()
sync_status = await node_service.sync_manager.get_sync_status()
# Генерировать ASCII
ascii_art = await generate_network_ascii(node_info, peers_info, sync_status)
return json_response({
"ascii": ascii_art,
"status": "online",
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.error(f"Error generating ASCII status: {e}")
return json_response({"ascii": generate_error_ascii(str(e)), "status": "error"})
@bp.get("/live")
async def live_monitoring_data(request: Request):
"""Получить живые данные для мониторинга."""
try:
node_service = get_node_service()
if not node_service:
return json_response(
{"error": "MY Network service unavailable"},
status=503
)
# Получить свежие данные
node_info = await node_service.get_node_info()
peers_info = await node_service.get_peers_info()
sync_status = await node_service.sync_manager.get_sync_status()
# Статистика сети
network_stats = {
"connected_peers": peers_info["peer_count"],
"active_syncs": sync_status["active_syncs"],
"queue_size": sync_status["queue_size"],
"uptime": node_info["uptime"],
"status": node_info["status"]
}
return json_response({
"success": True,
"data": {
"node_info": node_info,
"network_stats": network_stats,
"peers": peers_info["peers"][:10], # Показать только первые 10 пиров
"sync_status": sync_status
},
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.error(f"Error getting live monitoring data: {e}")
return json_response({"error": str(e)}, status=500)
async def generate_network_ascii(node_info: Dict[str, Any], peers_info: Dict[str, Any], sync_status: Dict[str, Any]) -> str:
"""Генерировать ASCII представление состояния сети."""
ascii_parts = []
# Заголовок
ascii_parts.append("""
╔══════════════════════════════════════════════════════════════════════════════╗
║ MY NETWORK v2.0 ║
║ Distributed Content Protocol ║
╚══════════════════════════════════════════════════════════════════════════════╝
""")
# Информация о ноде
status_indicator = "🟢" if node_info.get("status") == "running" else "🔴"
uptime_hours = int(node_info.get("uptime", 0) / 3600)
ascii_parts.append(f"""
┌─ NODE STATUS ────────────────────────────────────────────────────────────────┐
│ Node ID: {node_info.get('node_id', 'unknown')[:16]}... │
│ Status: {status_indicator} {node_info.get('status', 'unknown').upper()} │
│ Uptime: {uptime_hours}h {int((node_info.get('uptime', 0) % 3600) / 60)}m │
│ Version: MY Network {node_info.get('version', '2.0')} │
└──────────────────────────────────────────────────────────────────────────────┘
""")
# Информация о пирах
peer_count = peers_info.get("peer_count", 0)
peer_status = "🌐" if peer_count > 0 else "🏝️"
ascii_parts.append(f"""
┌─ NETWORK STATUS ─────────────────────────────────────────────────────────────┐
│ Connected Peers: {peer_status} {peer_count:>3} │
│ Known Nodes: {len(peers_info.get('peers', [])):>3} │
│ Network Health: {'CONNECTED' if peer_count > 0 else 'ISOLATED':>9} │
└──────────────────────────────────────────────────────────────────────────────┘
""")
# Статус синхронизации
sync_running = sync_status.get("is_running", False)
active_syncs = sync_status.get("active_syncs", 0)
queue_size = sync_status.get("queue_size", 0)
sync_indicator = "⚡" if sync_running else "⏸️"
ascii_parts.append(f"""
┌─ SYNC STATUS ────────────────────────────────────────────────────────────────┐
│ Sync Engine: {sync_indicator} {'RUNNING' if sync_running else 'STOPPED':>7} │
│ Active Syncs: {active_syncs:>3} │
│ Queue Size: {queue_size:>3} │
│ Workers: {sync_status.get('workers_count', 0):>3} │
└──────────────────────────────────────────────────────────────────────────────┘
""")
# Подвал
current_time = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
ascii_parts.append(f"""
╔══════════════════════════════════════════════════════════════════════════════╗
║ Last Updated: {current_time} ║
║ MY Network Protocol - Decentralized Content Distribution System ║
╚══════════════════════════════════════════════════════════════════════════════╝
""")
return "".join(ascii_parts)
def generate_offline_ascii() -> str:
"""Генерировать ASCII для офлайн состояния."""
return """
╔══════════════════════════════════════════════════════════════════════════════╗
║ MY NETWORK v2.0 ║
║ Distributed Content Protocol ║
╚══════════════════════════════════════════════════════════════════════════════╝
┌─ SYSTEM STATUS ──────────────────────────────────────────────────────────────┐
│ │
│ 🔴 OFFLINE │
│ │
│ MY Network service is not available │
│ │
└──────────────────────────────────────────────────────────────────────────────┘
╔══════════════════════════════════════════════════════════════════════════════╗
║ Status: OFFLINE - Service not initialized ║
╚══════════════════════════════════════════════════════════════════════════════╝
"""
def generate_error_ascii(error_message: str) -> str:
"""Генерировать ASCII для ошибки."""
return f"""
╔══════════════════════════════════════════════════════════════════════════════╗
║ MY NETWORK v2.0 ║
║ Distributed Content Protocol ║
╚══════════════════════════════════════════════════════════════════════════════╝
┌─ ERROR STATE ────────────────────────────────────────────────────────────────┐
│ │
│ ❌ ERROR │
│ │
│ {error_message[:64]:^64} │
│ │
└──────────────────────────────────────────────────────────────────────────────┘
╔══════════════════════════════════════════════════════════════════════════════╗
║ Status: ERROR - Check system logs for details ║
╚══════════════════════════════════════════════════════════════════════════════╝
"""
def generate_fallback_html(monitoring_data: Dict[str, Any]) -> str:
"""Генерировать fallback HTML если шаблоны не работают."""
status = monitoring_data.get("status", "unknown")
error_message = monitoring_data.get("error", "")
# Генерировать информацию о статусе
if status == "online":
node_info = monitoring_data.get("node_info", {})
peers_info = monitoring_data.get("peers_info", {})
sync_status = monitoring_data.get("sync_status", {})
status_info = f"""
<div class="status-section">
<h3>Node Status</h3>
<ul>
<li>Node ID: {node_info.get('node_id', 'unknown')[:16]}...</li>
<li>Status: {node_info.get('status', 'unknown').upper()}</li>
<li>Uptime: {int(node_info.get('uptime', 0) / 3600)}h {int((node_info.get('uptime', 0) % 3600) / 60)}m</li>
<li>Version: MY Network {node_info.get('version', '2.0')}</li>
</ul>
</div>
<div class="status-section">
<h3>Network Status</h3>
<ul>
<li>Connected Peers: {peers_info.get('peer_count', 0)}</li>
<li>Known Nodes: {len(peers_info.get('peers', []))}</li>
<li>Network Health: {'CONNECTED' if peers_info.get('peer_count', 0) > 0 else 'ISOLATED'}</li>
</ul>
</div>
<div class="status-section">
<h3>Sync Status</h3>
<ul>
<li>Sync Engine: {'RUNNING' if sync_status.get('is_running', False) else 'STOPPED'}</li>
<li>Active Syncs: {sync_status.get('active_syncs', 0)}</li>
<li>Queue Size: {sync_status.get('queue_size', 0)}</li>
<li>Workers: {sync_status.get('workers_count', 0)}</li>
</ul>
</div>
"""
else:
status_info = f"""
<div class="error-section">
<h3>Status: {status.upper()}</h3>
<p>{error_message if error_message else 'MY Network service not available'}</p>
</div>
"""
return f'''
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>MY Network Monitor</title>
<style>
body {{
background: linear-gradient(135deg, #0a0a0a 0%, #1a1a2e 50%, #16213e 100%);
color: #00ff41;
font-family: 'Courier New', monospace;
margin: 0;
padding: 20px;
min-height: 100vh;
}}
.container {{
max-width: 1200px;
margin: 0 auto;
}}
.header {{
text-align: center;
margin-bottom: 30px;
padding: 20px;
border: 2px solid #00ff41;
border-radius: 10px;
background: rgba(0, 255, 65, 0.05);
}}
.header h1 {{
font-size: 2.5em;
text-shadow: 0 0 10px #00ff41;
margin: 0;
}}
.status-section {{
background: rgba(0, 0, 0, 0.7);
border: 1px solid #00ff41;
border-radius: 8px;
padding: 20px;
margin: 20px 0;
}}
.status-section h3 {{
color: #00ff41;
margin-bottom: 15px;
text-transform: uppercase;
border-bottom: 1px solid #00ff41;
padding-bottom: 5px;
}}
.status-section ul {{
list-style: none;
padding: 0;
}}
.status-section li {{
margin: 10px 0;
padding: 5px 0;
border-bottom: 1px dotted #333;
}}
.error-section {{
background: rgba(255, 0, 0, 0.1);
border: 1px solid #ff0000;
border-radius: 8px;
padding: 20px;
margin: 20px 0;
text-align: center;
}}
.error-section h3 {{
color: #ff0000;
margin-bottom: 15px;
}}
.controls {{
text-align: center;
margin: 30px 0;
}}
.btn {{
background: linear-gradient(45deg, #00ff41, #00cc33);
color: #000;
border: none;
padding: 12px 24px;
font-family: inherit;
font-weight: bold;
cursor: pointer;
border-radius: 5px;
text-transform: uppercase;
margin: 0 10px;
text-decoration: none;
display: inline-block;
}}
.btn:hover {{
background: linear-gradient(45deg, #00cc33, #00ff41);
}}
.footer {{
text-align: center;
margin-top: 40px;
padding: 20px;
border-top: 1px solid #00ff41;
color: #888;
}}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>MY NETWORK MONITOR</h1>
<p>Distributed Content Protocol v2.0</p>
<p>Last Update: {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')}</p>
</div>
{status_info}
<div class="controls">
<a href="/api/my/monitor/" class="btn">🔄 REFRESH</a>
<a href="/api/my/monitor/ascii" class="btn">📊 ASCII VIEW</a>
<a href="/api/my/node/info" class="btn">ℹ️ NODE INFO</a>
<a href="/api/my/health" class="btn">❤️ HEALTH</a>
</div>
<div class="footer">
<p>MY Network Protocol - Decentralized Content Distribution System</p>
<p>Real-time monitoring dashboard</p>
</div>
</div>
<script>
// Автообновление каждые 30 секунд
setTimeout(() => location.reload(), 30000);
</script>
</body>
</html>
'''
+655
View File
@@ -0,0 +1,655 @@
"""MY Network API Routes - эндпоинты для работы с распределенной сетью."""
import asyncio
import logging
import json
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Any
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Query
from fastapi.responses import FileResponse, StreamingResponse
from sqlalchemy import select, and_, func
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.database_compatible import get_async_session
from app.core.models.content_compatible import Content, ContentMetadata
from app.core.security import get_current_user_optional
from app.core.cache import cache
logger = logging.getLogger(__name__)
# Создать router для MY Network API
router = APIRouter(prefix="/api/my", tags=["MY Network"])
def get_node_service():
"""Получить сервис ноды."""
try:
from app.core.my_network.node_service import get_node_service
return get_node_service()
except Exception as e:
logger.error(f"Error getting node service: {e}")
raise HTTPException(status_code=503, detail="MY Network service unavailable")
@router.get("/node/info")
async def get_node_info():
"""Получить информацию о текущей ноде."""
try:
node_service = get_node_service()
if not node_service:
raise HTTPException(status_code=503, detail="Node service not available")
node_info = await node_service.get_node_info()
return {
"success": True,
"data": node_info,
"timestamp": datetime.utcnow().isoformat()
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error getting node info: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/node/peers")
async def get_node_peers():
"""Получить список подключенных пиров."""
try:
node_service = get_node_service()
peers_info = await node_service.get_peers_info()
return {
"success": True,
"data": {
"connected_peers": peers_info["connected_peers"],
"peer_count": peers_info["peer_count"],
"peers": peers_info["peers"]
},
"timestamp": datetime.utcnow().isoformat()
}
except Exception as e:
logger.error(f"Error getting peers: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/node/peers/connect")
async def connect_to_peer(peer_data: Dict[str, Any]):
"""Подключиться к новому пиру."""
try:
peer_address = peer_data.get("address")
if not peer_address:
raise HTTPException(status_code=400, detail="Peer address is required")
node_service = get_node_service()
success = await node_service.peer_manager.connect_to_peer(peer_address)
if success:
return {
"success": True,
"message": f"Successfully connected to peer: {peer_address}",
"timestamp": datetime.utcnow().isoformat()
}
else:
raise HTTPException(status_code=400, detail="Failed to connect to peer")
except HTTPException:
raise
except Exception as e:
logger.error(f"Error connecting to peer: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.delete("/node/peers/{peer_id}")
async def disconnect_peer(peer_id: str):
"""Отключиться от пира."""
try:
node_service = get_node_service()
success = await node_service.peer_manager.disconnect_peer(peer_id)
if success:
return {
"success": True,
"message": f"Successfully disconnected from peer: {peer_id}",
"timestamp": datetime.utcnow().isoformat()
}
else:
raise HTTPException(status_code=404, detail="Peer not found or already disconnected")
except HTTPException:
raise
except Exception as e:
logger.error(f"Error disconnecting peer: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/content/list")
async def get_content_list(
limit: int = Query(100, ge=1, le=1000),
offset: int = Query(0, ge=0),
session: AsyncSession = Depends(get_async_session)
):
"""Получить список доступного контента."""
try:
# Кэшировать результат на 5 минут
cache_key = f"my_network:content_list:{limit}:{offset}"
cached_result = await cache.get(cache_key)
if cached_result:
return json.loads(cached_result)
# Получить контент из БД
stmt = (
select(Content, ContentMetadata)
.outerjoin(ContentMetadata, Content.id == ContentMetadata.content_id)
.where(Content.is_active == True)
.order_by(Content.created_at.desc())
.limit(limit)
.offset(offset)
)
result = await session.execute(stmt)
content_items = []
for content, metadata in result:
content_data = {
"hash": content.sha256_hash or content.md5_hash,
"filename": content.filename,
"original_filename": content.original_filename,
"file_size": content.file_size,
"file_type": content.file_type,
"mime_type": content.mime_type,
"created_at": content.created_at.isoformat(),
"encrypted": getattr(content, 'encrypted', False),
"metadata": metadata.to_dict() if metadata else {}
}
content_items.append(content_data)
# Получить общее количество
count_stmt = select(func.count(Content.id)).where(Content.is_active == True)
count_result = await session.execute(count_stmt)
total_count = count_result.scalar()
response_data = {
"success": True,
"data": {
"content": content_items,
"total": total_count,
"limit": limit,
"offset": offset
},
"timestamp": datetime.utcnow().isoformat()
}
# Кэшировать результат
await cache.set(cache_key, json.dumps(response_data), expire=300)
return response_data
except Exception as e:
logger.error(f"Error getting content list: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/content/{content_hash}/exists")
async def check_content_exists(
content_hash: str,
session: AsyncSession = Depends(get_async_session)
):
"""Проверить существование контента по хешу."""
try:
# Кэшировать результат на 30 минут
cache_key = f"my_network:content_exists:{content_hash}"
cached_result = await cache.get(cache_key)
if cached_result is not None:
return {"exists": cached_result == "true", "hash": content_hash}
# Проверить в БД
stmt = select(Content.id).where(
and_(
Content.is_active == True,
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
)
)
result = await session.execute(stmt)
exists = result.scalar_one_or_none() is not None
# Кэшировать результат
await cache.set(cache_key, "true" if exists else "false", expire=1800)
return {
"exists": exists,
"hash": content_hash,
"timestamp": datetime.utcnow().isoformat()
}
except Exception as e:
logger.error(f"Error checking content existence: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/content/{content_hash}/metadata")
async def get_content_metadata(
content_hash: str,
session: AsyncSession = Depends(get_async_session)
):
"""Получить метаданные контента."""
try:
# Кэшировать результат на 10 минут
cache_key = f"my_network:content_metadata:{content_hash}"
cached_result = await cache.get(cache_key)
if cached_result:
return json.loads(cached_result)
# Найти контент в БД
stmt = (
select(Content, ContentMetadata)
.outerjoin(ContentMetadata, Content.id == ContentMetadata.content_id)
.where(
and_(
Content.is_active == True,
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
)
)
)
result = await session.execute(stmt)
content_data = result.first()
if not content_data:
raise HTTPException(status_code=404, detail="Content not found")
content, metadata = content_data
response_data = {
"success": True,
"data": {
"hash": content_hash,
"filename": content.filename,
"original_filename": content.original_filename,
"file_size": content.file_size,
"file_type": content.file_type,
"mime_type": content.mime_type,
"created_at": content.created_at.isoformat(),
"updated_at": content.updated_at.isoformat() if content.updated_at else None,
"encrypted": getattr(content, 'encrypted', False),
"processing_status": getattr(content, 'processing_status', 'completed'),
"metadata": metadata.to_dict() if metadata else {}
},
"timestamp": datetime.utcnow().isoformat()
}
# Кэшировать результат
await cache.set(cache_key, json.dumps(response_data), expire=600)
return response_data
except HTTPException:
raise
except Exception as e:
logger.error(f"Error getting content metadata: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/content/{content_hash}/download")
async def download_content(
content_hash: str,
session: AsyncSession = Depends(get_async_session)
):
"""Скачать контент по хешу."""
try:
# Найти контент в БД
stmt = select(Content).where(
and_(
Content.is_active == True,
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
)
)
result = await session.execute(stmt)
content = result.scalar_one_or_none()
if not content:
raise HTTPException(status_code=404, detail="Content not found")
# Проверить существование файла
file_path = Path(content.file_path)
if not file_path.exists():
raise HTTPException(status_code=404, detail="File not found on disk")
# Вернуть файл
return FileResponse(
path=str(file_path),
filename=content.original_filename or content.filename,
media_type=content.mime_type or "application/octet-stream"
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Error downloading content: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/content/{content_hash}/upload")
async def upload_content(
content_hash: str,
file: UploadFile = File(...),
session: AsyncSession = Depends(get_async_session)
):
"""Загрузить контент в ноду."""
try:
# Проверить, не существует ли уже контент
exists_stmt = select(Content.id).where(
and_(
Content.is_active == True,
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
)
)
exists_result = await session.execute(exists_stmt)
if exists_result.scalar_one_or_none():
return {
"success": True,
"message": "Content already exists",
"hash": content_hash
}
# Создать директорию для хранения
storage_path = Path("./storage/my-network/received")
storage_path.mkdir(parents=True, exist_ok=True)
# Сохранить файл
file_path = storage_path / f"{content_hash}_{file.filename}"
with open(file_path, "wb") as buffer:
content_data = await file.read()
buffer.write(content_data)
# Вычислить хеши для проверки
import hashlib
md5_hash = hashlib.md5(content_data).hexdigest()
sha256_hash = hashlib.sha256(content_data).hexdigest()
# Проверить соответствие хеша
if content_hash not in [md5_hash, sha256_hash]:
file_path.unlink() # Удалить файл
raise HTTPException(status_code=400, detail="Content hash mismatch")
# Сохранить в БД
new_content = Content(
filename=file.filename,
original_filename=file.filename,
file_path=str(file_path),
file_size=len(content_data),
file_type=file.filename.split('.')[-1] if '.' in file.filename else 'unknown',
mime_type=file.content_type or "application/octet-stream",
md5_hash=md5_hash,
sha256_hash=sha256_hash,
is_active=True,
processing_status="completed"
)
session.add(new_content)
await session.commit()
logger.info(f"Successfully uploaded content {content_hash}")
return {
"success": True,
"message": "Content uploaded successfully",
"hash": content_hash,
"content_id": new_content.id,
"timestamp": datetime.utcnow().isoformat()
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error uploading content: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/content/replicate")
async def replicate_content(replication_request: Dict[str, Any]):
"""Принять запрос на репликацию контента."""
try:
content_hash = replication_request.get("content_hash")
metadata = replication_request.get("metadata", {})
source_node = replication_request.get("source_node")
if not content_hash:
raise HTTPException(status_code=400, detail="Content hash is required")
# Проверить, нужна ли репликация
async with get_async_session() as session:
exists_stmt = select(Content.id).where(
and_(
Content.is_active == True,
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
)
)
exists_result = await session.execute(exists_stmt)
if exists_result.scalar_one_or_none():
return {
"success": True,
"message": "Content already exists, replication not needed",
"hash": content_hash
}
# Подготовить для репликации
logger.info(f"Accepting replication request for {content_hash} from {source_node}")
return {
"success": True,
"message": "Replication request accepted",
"hash": content_hash,
"ready_for_upload": True,
"timestamp": datetime.utcnow().isoformat()
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error processing replication request: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/sync/status")
async def get_sync_status():
"""Получить статус синхронизации."""
try:
node_service = get_node_service()
sync_status = await node_service.sync_manager.get_sync_status()
return {
"success": True,
"data": sync_status,
"timestamp": datetime.utcnow().isoformat()
}
except Exception as e:
logger.error(f"Error getting sync status: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/sync/start")
async def start_network_sync():
"""Запустить синхронизацию с сетью."""
try:
node_service = get_node_service()
sync_result = await node_service.sync_manager.sync_with_network()
return {
"success": True,
"data": sync_result,
"timestamp": datetime.utcnow().isoformat()
}
except Exception as e:
logger.error(f"Error starting network sync: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/sync/content/{content_hash}")
async def get_content_sync_status(content_hash: str):
"""Получить статус синхронизации конкретного контента."""
try:
node_service = get_node_service()
sync_status = await node_service.sync_manager.get_content_sync_status(content_hash)
return {
"success": True,
"data": sync_status,
"timestamp": datetime.utcnow().isoformat()
}
except Exception as e:
logger.error(f"Error getting content sync status: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/content/{content_hash}/replicate")
async def replicate_content_to_nodes(
content_hash: str,
replication_config: Dict[str, Any]
):
"""Реплицировать контент на указанные ноды."""
try:
target_nodes = replication_config.get("target_nodes", [])
if not target_nodes:
raise HTTPException(status_code=400, detail="Target nodes are required")
node_service = get_node_service()
replication_result = await node_service.sync_manager.replicate_content_to_nodes(
content_hash,
target_nodes
)
return {
"success": True,
"data": replication_result,
"timestamp": datetime.utcnow().isoformat()
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error replicating content: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/network/stats")
async def get_network_stats():
"""Получить статистику сети."""
try:
node_service = get_node_service()
# Получить информацию о ноде и пирах
node_info = await node_service.get_node_info()
peers_info = await node_service.get_peers_info()
sync_status = await node_service.sync_manager.get_sync_status()
# Статистика контента
async with get_async_session() as session:
# Общее количество контента
content_count_stmt = select(func.count(Content.id)).where(Content.is_active == True)
content_count_result = await session.execute(content_count_stmt)
total_content = content_count_result.scalar()
# Размер контента
size_stmt = select(func.sum(Content.file_size)).where(Content.is_active == True)
size_result = await session.execute(size_stmt)
total_size = size_result.scalar() or 0
# Контент по типам
type_stmt = select(Content.file_type, func.count(Content.id)).where(Content.is_active == True).group_by(Content.file_type)
type_result = await session.execute(type_stmt)
content_by_type = {row[0]: row[1] for row in type_result}
network_stats = {
"node_info": {
"node_id": node_info["node_id"],
"uptime": node_info["uptime"],
"version": node_info["version"],
"status": node_info["status"]
},
"network": {
"connected_peers": peers_info["peer_count"],
"known_peers": len(peers_info["peers"]),
"network_health": "good" if peers_info["peer_count"] > 0 else "isolated"
},
"content": {
"total_items": total_content,
"total_size_bytes": total_size,
"total_size_mb": round(total_size / (1024 * 1024), 2),
"content_by_type": content_by_type
},
"sync": {
"active_syncs": sync_status["active_syncs"],
"queue_size": sync_status["queue_size"],
"is_running": sync_status["is_running"]
}
}
return {
"success": True,
"data": network_stats,
"timestamp": datetime.utcnow().isoformat()
}
except Exception as e:
logger.error(f"Error getting network stats: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/health")
async def health_check():
"""Проверка здоровья MY Network ноды."""
try:
node_service = get_node_service()
# Базовая проверка сервисов
health_status = {
"status": "healthy",
"timestamp": datetime.utcnow().isoformat(),
"services": {
"node_service": node_service is not None,
"peer_manager": hasattr(node_service, 'peer_manager') if node_service else False,
"sync_manager": hasattr(node_service, 'sync_manager') if node_service else False,
"database": True # Если дошли до этой точки, БД работает
}
}
# Проверить подключение к пирам
if node_service:
peers_info = await node_service.get_peers_info()
health_status["network"] = {
"connected_peers": peers_info["peer_count"],
"status": "connected" if peers_info["peer_count"] > 0 else "isolated"
}
# Определить общий статус
if not all(health_status["services"].values()):
health_status["status"] = "unhealthy"
elif node_service and peers_info["peer_count"] == 0:
health_status["status"] = "isolated"
return health_status
except Exception as e:
logger.error(f"Health check failed: {e}")
return {
"status": "unhealthy",
"error": str(e),
"timestamp": datetime.utcnow().isoformat()
}
+426
View File
@@ -0,0 +1,426 @@
"""MY Network Sanic Blueprint - маршруты для работы с распределенной сетью."""
import asyncio
import json
import logging
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Any
from sanic import Blueprint, Request
from sanic.response import json as json_response, file as file_response
from sanic.exceptions import SanicException
from app.core.logging import get_logger
logger = get_logger(__name__)
# Создать blueprint для MY Network API
bp = Blueprint("my_network", url_prefix="/api/my")
def get_node_service():
"""Получить сервис ноды."""
try:
from app.core.my_network.node_service import get_node_service
return get_node_service()
except Exception as e:
logger.error(f"Error getting node service: {e}")
return None
@bp.get("/node/info")
async def get_node_info(request: Request):
"""Получить информацию о текущей ноде."""
try:
node_service = get_node_service()
if not node_service:
return json_response(
{"error": "MY Network service not available"},
status=503
)
node_info = await node_service.get_node_info()
return json_response({
"success": True,
"data": node_info,
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.error(f"Error getting node info: {e}")
return json_response({"error": str(e)}, status=500)
@bp.get("/node/peers")
async def get_node_peers(request: Request):
"""Получить список подключенных пиров."""
try:
node_service = get_node_service()
if not node_service:
return json_response(
{"error": "MY Network service not available"},
status=503
)
peers_info = await node_service.get_peers_info()
return json_response({
"success": True,
"data": {
"connected_peers": peers_info["connected_peers"],
"peer_count": peers_info["peer_count"],
"peers": peers_info["peers"]
},
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.error(f"Error getting peers: {e}")
return json_response({"error": str(e)}, status=500)
@bp.post("/node/peers/connect")
async def connect_to_peer(request: Request):
"""Подключиться к новому пиру."""
try:
peer_data = request.json
peer_address = peer_data.get("address")
if not peer_address:
return json_response({"error": "Peer address is required"}, status=400)
node_service = get_node_service()
if not node_service:
return json_response(
{"error": "MY Network service not available"},
status=503
)
success = await node_service.peer_manager.connect_to_peer(peer_address)
if success:
return json_response({
"success": True,
"message": f"Successfully connected to peer: {peer_address}",
"timestamp": datetime.utcnow().isoformat()
})
else:
return json_response({"error": "Failed to connect to peer"}, status=400)
except Exception as e:
logger.error(f"Error connecting to peer: {e}")
return json_response({"error": str(e)}, status=500)
@bp.delete("/node/peers/<peer_id>")
async def disconnect_peer(request: Request, peer_id: str):
"""Отключиться от пира."""
try:
node_service = get_node_service()
if not node_service:
return json_response(
{"error": "MY Network service not available"},
status=503
)
success = await node_service.peer_manager.disconnect_peer(peer_id)
if success:
return json_response({
"success": True,
"message": f"Successfully disconnected from peer: {peer_id}",
"timestamp": datetime.utcnow().isoformat()
})
else:
return json_response(
{"error": "Peer not found or already disconnected"},
status=404
)
except Exception as e:
logger.error(f"Error disconnecting peer: {e}")
return json_response({"error": str(e)}, status=500)
@bp.get("/content/list")
async def get_content_list(request: Request):
"""Получить список доступного контента."""
try:
# Получить параметры запроса
limit = min(int(request.args.get("limit", 100)), 1000)
offset = max(int(request.args.get("offset", 0)), 0)
# Кэшировать результат на 5 минут
from app.core.cache import cache
cache_key = f"my_network:content_list:{limit}:{offset}"
cached_result = await cache.get(cache_key)
if cached_result:
return json_response(json.loads(cached_result))
# Получить контент из БД
from app.core.database_compatible import get_async_session
from app.core.models.content_compatible import Content, ContentMetadata
from sqlalchemy import select, func
async with get_async_session() as session:
stmt = (
select(Content, ContentMetadata)
.outerjoin(ContentMetadata, Content.id == ContentMetadata.content_id)
.where(Content.is_active == True)
.order_by(Content.created_at.desc())
.limit(limit)
.offset(offset)
)
result = await session.execute(stmt)
content_items = []
for content, metadata in result:
content_data = {
"hash": content.sha256_hash or content.md5_hash,
"filename": content.filename,
"original_filename": content.original_filename,
"file_size": content.file_size,
"file_type": content.file_type,
"mime_type": content.mime_type,
"created_at": content.created_at.isoformat(),
"encrypted": getattr(content, 'encrypted', False),
"metadata": metadata.to_dict() if metadata else {}
}
content_items.append(content_data)
# Получить общее количество
count_stmt = select(func.count(Content.id)).where(Content.is_active == True)
count_result = await session.execute(count_stmt)
total_count = count_result.scalar()
response_data = {
"success": True,
"data": {
"content": content_items,
"total": total_count,
"limit": limit,
"offset": offset
},
"timestamp": datetime.utcnow().isoformat()
}
# Кэшировать результат
await cache.set(cache_key, json.dumps(response_data), expire=300)
return json_response(response_data)
except Exception as e:
logger.error(f"Error getting content list: {e}")
return json_response({"error": str(e)}, status=500)
@bp.get("/content/<content_hash>/exists")
async def check_content_exists(request: Request, content_hash: str):
"""Проверить существование контента по хешу."""
try:
# Кэшировать результат на 30 минут
from app.core.cache import cache
cache_key = f"my_network:content_exists:{content_hash}"
cached_result = await cache.get(cache_key)
if cached_result is not None:
return json_response({"exists": cached_result == "true", "hash": content_hash})
# Проверить в БД
from app.core.database_compatible import get_async_session
from app.core.models.content_compatible import Content
from sqlalchemy import select, and_
async with get_async_session() as session:
stmt = select(Content.id).where(
and_(
Content.is_active == True,
(Content.md5_hash == content_hash) | (Content.sha256_hash == content_hash)
)
)
result = await session.execute(stmt)
exists = result.scalar_one_or_none() is not None
# Кэшировать результат
await cache.set(cache_key, "true" if exists else "false", expire=1800)
return json_response({
"exists": exists,
"hash": content_hash,
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.error(f"Error checking content existence: {e}")
return json_response({"error": str(e)}, status=500)
@bp.get("/sync/status")
async def get_sync_status(request: Request):
"""Получить статус синхронизации."""
try:
node_service = get_node_service()
if not node_service:
return json_response(
{"error": "MY Network service not available"},
status=503
)
sync_status = await node_service.sync_manager.get_sync_status()
return json_response({
"success": True,
"data": sync_status,
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.error(f"Error getting sync status: {e}")
return json_response({"error": str(e)}, status=500)
@bp.post("/sync/start")
async def start_network_sync(request: Request):
"""Запустить синхронизацию с сетью."""
try:
node_service = get_node_service()
if not node_service:
return json_response(
{"error": "MY Network service not available"},
status=503
)
sync_result = await node_service.sync_manager.sync_with_network()
return json_response({
"success": True,
"data": sync_result,
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.error(f"Error starting network sync: {e}")
return json_response({"error": str(e)}, status=500)
@bp.get("/network/stats")
async def get_network_stats(request: Request):
"""Получить статистику сети."""
try:
node_service = get_node_service()
if not node_service:
return json_response(
{"error": "MY Network service not available"},
status=503
)
# Получить информацию о ноде и пирах
node_info = await node_service.get_node_info()
peers_info = await node_service.get_peers_info()
sync_status = await node_service.sync_manager.get_sync_status()
# Статистика контента
from app.core.database_compatible import get_async_session
from app.core.models.content_compatible import Content
from sqlalchemy import select, func
async with get_async_session() as session:
# Общее количество контента
content_count_stmt = select(func.count(Content.id)).where(Content.is_active == True)
content_count_result = await session.execute(content_count_stmt)
total_content = content_count_result.scalar()
# Размер контента
size_stmt = select(func.sum(Content.file_size)).where(Content.is_active == True)
size_result = await session.execute(size_stmt)
total_size = size_result.scalar() or 0
# Контент по типам
type_stmt = select(Content.file_type, func.count(Content.id)).where(Content.is_active == True).group_by(Content.file_type)
type_result = await session.execute(type_stmt)
content_by_type = {row[0]: row[1] for row in type_result}
network_stats = {
"node_info": {
"node_id": node_info["node_id"],
"uptime": node_info["uptime"],
"version": node_info["version"],
"status": node_info["status"]
},
"network": {
"connected_peers": peers_info["peer_count"],
"known_peers": len(peers_info["peers"]),
"network_health": "good" if peers_info["peer_count"] > 0 else "isolated"
},
"content": {
"total_items": total_content,
"total_size_bytes": total_size,
"total_size_mb": round(total_size / (1024 * 1024), 2),
"content_by_type": content_by_type
},
"sync": {
"active_syncs": sync_status["active_syncs"],
"queue_size": sync_status["queue_size"],
"is_running": sync_status["is_running"]
}
}
return json_response({
"success": True,
"data": network_stats,
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
logger.error(f"Error getting network stats: {e}")
return json_response({"error": str(e)}, status=500)
@bp.get("/health")
async def health_check(request: Request):
"""Проверка здоровья MY Network ноды."""
try:
node_service = get_node_service()
# Базовая проверка сервисов
health_status = {
"status": "healthy",
"timestamp": datetime.utcnow().isoformat(),
"services": {
"node_service": node_service is not None,
"peer_manager": hasattr(node_service, 'peer_manager') if node_service else False,
"sync_manager": hasattr(node_service, 'sync_manager') if node_service else False,
"database": True # Если дошли до этой точки, БД работает
}
}
# Проверить подключение к пирам
if node_service:
peers_info = await node_service.get_peers_info()
health_status["network"] = {
"connected_peers": peers_info["peer_count"],
"status": "connected" if peers_info["peer_count"] > 0 else "isolated"
}
# Определить общий статус
if not all(health_status["services"].values()):
health_status["status"] = "unhealthy"
elif node_service and peers_info["peer_count"] == 0:
health_status["status"] = "isolated"
return json_response(health_status)
except Exception as e:
logger.error(f"Health check failed: {e}")
return json_response({
"status": "unhealthy",
"error": str(e),
"timestamp": datetime.utcnow().isoformat()
}, status=500)
+708
View File
@@ -0,0 +1,708 @@
"""
Storage management routes with chunked uploads, download handling, and file operations.
Provides secure file operations with progress tracking and comprehensive validation.
"""
import asyncio
from datetime import datetime
from typing import Dict, List, Optional, Any
from uuid import UUID
from sanic import Blueprint, Request, response
from sanic.response import JSONResponse, ResponseStream
from sqlalchemy import select, update
from app.core.config import get_settings
from app.core.database import get_async_session, get_cache_manager
from app.core.logging import get_logger
from app.core.storage import StorageManager
from app.core.security import validate_file_signature, generate_secure_filename
from app.api.middleware import require_auth, validate_request, rate_limit
from app.core.validation import StorageUploadSchema, ChunkUploadSchema
# Initialize blueprint
storage_bp = Blueprint("storage", url_prefix="/api/v1/storage")
logger = get_logger(__name__)
settings = get_settings()
@storage_bp.route("/upload", methods=["POST"])
@rate_limit(limit=10, window=3600) # 10 upload sessions per hour
@require_auth(permissions=["storage.upload"])
@validate_request(StorageUploadSchema)
async def initiate_upload(request: Request) -> JSONResponse:
"""
Initiate chunked file upload session with security validation.
Args:
request: Sanic request with upload parameters
Returns:
JSONResponse: Upload session information
"""
try:
data = request.json
user_id = request.ctx.user.id
# Validate file size against user quota
cache_manager = get_cache_manager()
quota_key = f"user:{user_id}:storage_quota"
current_usage = await cache_manager.get(quota_key, default=0)
if current_usage + data["file_size"] > settings.MAX_STORAGE_PER_USER:
return response.json(
{"error": "Storage quota exceeded", "code": "QUOTA_EXCEEDED"},
status=429
)
# Generate secure filename
secure_filename = generate_secure_filename(data["filename"], user_id)
# Validate content type
allowed_types = {
'image/jpeg', 'image/png', 'image/gif', 'image/webp',
'video/mp4', 'video/webm', 'video/avi',
'audio/mpeg', 'audio/wav', 'audio/flac', 'audio/ogg',
'application/pdf', 'text/plain', 'application/json',
'application/zip', 'application/x-rar'
}
if data["content_type"] not in allowed_types:
return response.json(
{"error": "File type not allowed", "code": "TYPE_NOT_ALLOWED"},
status=400
)
# Create content record first
async with get_async_session() as session:
from app.core.models.content import Content
content = Content(
user_id=user_id,
title=secure_filename,
content_type=data["content_type"],
file_size=data["file_size"],
status="uploading",
visibility="private"
)
session.add(content)
await session.commit()
await session.refresh(content)
# Create upload session
storage_manager = StorageManager()
upload_session = await storage_manager.create_upload_session(
content.id,
data["file_size"]
)
# Update user quota
await cache_manager.increment(quota_key, data["file_size"], ttl=86400)
await logger.ainfo(
"Upload session initiated",
user_id=str(user_id),
content_id=str(content.id),
filename=secure_filename,
file_size=data["file_size"]
)
return response.json({
"upload_session": upload_session,
"content_id": str(content.id),
"secure_filename": secure_filename,
"status": "ready_for_upload"
}, status=201)
except Exception as e:
await logger.aerror(
"Failed to initiate upload",
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Failed to initiate upload", "code": "UPLOAD_INIT_FAILED"},
status=500
)
@storage_bp.route("/upload/<upload_id:uuid>/chunk", methods=["POST"])
@rate_limit(limit=1000, window=3600) # 1000 chunks per hour
@require_auth(permissions=["storage.upload"])
async def upload_chunk(request: Request, upload_id: UUID) -> JSONResponse:
"""
Upload individual file chunk with validation and progress tracking.
Args:
request: Sanic request with chunk data
upload_id: Upload session UUID
Returns:
JSONResponse: Chunk upload status
"""
try:
user_id = request.ctx.user.id
# Get chunk data from form
if 'chunk' not in request.files:
return response.json(
{"error": "No chunk data provided", "code": "NO_CHUNK_DATA"},
status=400
)
chunk_file = request.files['chunk'][0]
chunk_data = chunk_file.body
# Get chunk metadata
chunk_index = int(request.form.get('chunk_index', 0))
chunk_hash = request.form.get('chunk_hash', '')
is_final = request.form.get('is_final', 'false').lower() == 'true'
if not chunk_hash:
return response.json(
{"error": "Chunk hash required", "code": "HASH_REQUIRED"},
status=400
)
# Validate chunk size
if len(chunk_data) > settings.MAX_CHUNK_SIZE:
return response.json(
{"error": "Chunk too large", "code": "CHUNK_TOO_LARGE"},
status=400
)
# Upload chunk
storage_manager = StorageManager()
result = await storage_manager.upload_chunk(
upload_id,
chunk_index,
chunk_data,
chunk_hash
)
# Check if upload is complete
if is_final or result["uploaded_chunks"] == result["total_chunks"]:
# Finalize upload
finalize_result = await storage_manager.finalize_upload(upload_id)
result.update(finalize_result)
await logger.ainfo(
"Upload completed",
upload_id=str(upload_id),
user_id=str(user_id),
content_id=finalize_result.get("content_id")
)
return response.json(result)
except ValueError as e:
await logger.awarning(
"Chunk upload validation failed",
upload_id=str(upload_id),
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": str(e), "code": "VALIDATION_FAILED"},
status=400
)
except Exception as e:
await logger.aerror(
"Chunk upload failed",
upload_id=str(upload_id),
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Chunk upload failed", "code": "CHUNK_UPLOAD_FAILED"},
status=500
)
@storage_bp.route("/upload/<upload_id:uuid>/status", methods=["GET"])
@rate_limit(limit=100, window=3600) # 100 status checks per hour
@require_auth(permissions=["storage.upload"])
async def get_upload_status(request: Request, upload_id: UUID) -> JSONResponse:
"""
Get upload session status and progress.
Args:
request: Sanic request object
upload_id: Upload session UUID
Returns:
JSONResponse: Upload progress information
"""
try:
user_id = request.ctx.user.id
storage_manager = StorageManager()
# Get session data
session_data = await storage_manager._get_upload_session(upload_id)
if not session_data:
return response.json(
{"error": "Upload session not found", "code": "SESSION_NOT_FOUND"},
status=404
)
# Verify user ownership
async with get_async_session() as session:
from app.core.models.content import Content
stmt = select(Content).where(
Content.id == UUID(session_data["content_id"])
)
result = await session.execute(stmt)
content = result.scalar_one_or_none()
if not content or content.user_id != user_id:
return response.json(
{"error": "Access denied", "code": "ACCESS_DENIED"},
status=403
)
# Calculate progress
uploaded_chunks = len(session_data.get("uploaded_chunks", []))
total_chunks = session_data["total_chunks"]
progress_percent = (uploaded_chunks / total_chunks * 100) if total_chunks > 0 else 0
return response.json({
"upload_id": str(upload_id),
"status": session_data["status"],
"progress": {
"uploaded_chunks": uploaded_chunks,
"total_chunks": total_chunks,
"percent": round(progress_percent, 2)
},
"created_at": session_data["created_at"],
"expires_at": session_data["expires_at"]
})
except Exception as e:
await logger.aerror(
"Failed to get upload status",
upload_id=str(upload_id),
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Failed to get upload status", "code": "STATUS_FAILED"},
status=500
)
@storage_bp.route("/upload/<upload_id:uuid>", methods=["DELETE"])
@rate_limit(limit=50, window=3600) # 50 cancellations per hour
@require_auth(permissions=["storage.upload"])
async def cancel_upload(request: Request, upload_id: UUID) -> JSONResponse:
"""
Cancel upload session and clean up temporary files.
Args:
request: Sanic request object
upload_id: Upload session UUID
Returns:
JSONResponse: Cancellation status
"""
try:
user_id = request.ctx.user.id
storage_manager = StorageManager()
# Get session data
session_data = await storage_manager._get_upload_session(upload_id)
if not session_data:
return response.json(
{"error": "Upload session not found", "code": "SESSION_NOT_FOUND"},
status=404
)
# Verify user ownership
content_id = UUID(session_data["content_id"])
async with get_async_session() as session:
from app.core.models.content import Content
stmt = select(Content).where(Content.id == content_id)
result = await session.execute(stmt)
content = result.scalar_one_or_none()
if not content or content.user_id != user_id:
return response.json(
{"error": "Access denied", "code": "ACCESS_DENIED"},
status=403
)
# Delete content record
await session.delete(content)
await session.commit()
# Clean up chunks and session
cache_manager = get_cache_manager()
session_key = f"upload_session:{upload_id}"
await cache_manager.delete(session_key)
# Clean up chunks from storage
for chunk_index in session_data.get("uploaded_chunks", []):
chunk_id = f"{upload_id}_{chunk_index:06d}"
await storage_manager.backend.delete_chunk(chunk_id)
# Update user quota
quota_key = f"user:{user_id}:storage_quota"
await cache_manager.decrement(quota_key, session_data.get("total_size", 0))
await logger.ainfo(
"Upload cancelled",
upload_id=str(upload_id),
user_id=str(user_id),
content_id=str(content_id)
)
return response.json({
"status": "cancelled",
"upload_id": str(upload_id)
})
except Exception as e:
await logger.aerror(
"Failed to cancel upload",
upload_id=str(upload_id),
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Failed to cancel upload", "code": "CANCEL_FAILED"},
status=500
)
@storage_bp.route("/files/<content_id:uuid>", methods=["DELETE"])
@rate_limit(limit=50, window=3600) # 50 deletions per hour
@require_auth(permissions=["storage.delete"])
async def delete_file(request: Request, content_id: UUID) -> JSONResponse:
"""
Delete content file and cleanup storage.
Args:
request: Sanic request object
content_id: Content UUID to delete
Returns:
JSONResponse: Deletion status
"""
try:
user_id = request.ctx.user.id
async with get_async_session() as session:
from app.core.models.content import Content
# Get content
stmt = select(Content).where(Content.id == content_id)
result = await session.execute(stmt)
content = result.scalar_one_or_none()
if not content:
return response.json(
{"error": "Content not found", "code": "NOT_FOUND"},
status=404
)
# Check permissions
if content.user_id != user_id and not request.ctx.user.is_admin:
return response.json(
{"error": "Access denied", "code": "ACCESS_DENIED"},
status=403
)
# Delete files
storage_manager = StorageManager()
deletion_success = await storage_manager.delete_content_files(content_id)
if not deletion_success:
await logger.awarning(
"File deletion partially failed",
content_id=str(content_id),
user_id=str(user_id)
)
# Update user quota
cache_manager = get_cache_manager()
quota_key = f"user:{user_id}:storage_quota"
await cache_manager.decrement(quota_key, content.file_size or 0)
# Clear caches
await cache_manager.delete(f"content:{content_id}")
await cache_manager.delete(f"content:{content_id}:full")
await logger.ainfo(
"Content deleted",
content_id=str(content_id),
user_id=str(user_id),
file_size=content.file_size
)
return response.json({
"status": "deleted",
"content_id": str(content_id)
})
except Exception as e:
await logger.aerror(
"Failed to delete content",
content_id=str(content_id),
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Failed to delete content", "code": "DELETE_FAILED"},
status=500
)
@storage_bp.route("/quota", methods=["GET"])
@rate_limit(limit=100, window=3600) # 100 quota checks per hour
@require_auth(permissions=["storage.read"])
async def get_storage_quota(request: Request) -> JSONResponse:
"""
Get user storage quota and usage information.
Args:
request: Sanic request object
Returns:
JSONResponse: Quota information
"""
try:
user_id = request.ctx.user.id
# Get current usage from cache
cache_manager = get_cache_manager()
quota_key = f"user:{user_id}:storage_quota"
current_usage = await cache_manager.get(quota_key, default=0)
# Calculate accurate usage from database
async with get_async_session() as session:
from sqlalchemy import func
from app.core.models.content import Content
stmt = select(
func.count(Content.id).label('file_count'),
func.sum(Content.file_size).label('total_size')
).where(
Content.user_id == user_id,
Content.status == 'completed'
)
result = await session.execute(stmt)
stats = result.first()
accurate_usage = stats.total_size or 0
file_count = stats.file_count or 0
# Update cache with accurate value
if abs(current_usage - accurate_usage) > 1024: # Update if difference > 1KB
await cache_manager.set(quota_key, accurate_usage, ttl=86400)
current_usage = accurate_usage
# Calculate quota information
max_quota = settings.MAX_STORAGE_PER_USER
usage_percent = (current_usage / max_quota * 100) if max_quota > 0 else 0
return response.json({
"quota": {
"used_bytes": current_usage,
"max_bytes": max_quota,
"available_bytes": max(0, max_quota - current_usage),
"usage_percent": round(usage_percent, 2)
},
"files": {
"count": file_count,
"max_files": settings.MAX_FILES_PER_USER
},
"updated_at": datetime.utcnow().isoformat()
})
except Exception as e:
await logger.aerror(
"Failed to get storage quota",
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Failed to get quota information", "code": "QUOTA_FAILED"},
status=500
)
@storage_bp.route("/stats", methods=["GET"])
@rate_limit(limit=50, window=3600) # 50 stats requests per hour
@require_auth(permissions=["storage.read"])
async def get_storage_stats(request: Request) -> JSONResponse:
"""
Get detailed storage statistics for user.
Args:
request: Sanic request object
Returns:
JSONResponse: Detailed storage statistics
"""
try:
user_id = request.ctx.user.id
async with get_async_session() as session:
from sqlalchemy import func
from app.core.models.content import Content
# Get statistics by content type
type_stmt = select(
Content.content_type,
func.count(Content.id).label('count'),
func.sum(Content.file_size).label('size'),
func.avg(Content.file_size).label('avg_size')
).where(
Content.user_id == user_id,
Content.status == 'completed'
).group_by(Content.content_type)
type_result = await session.execute(type_stmt)
type_stats = {
row.content_type: {
'count': row.count,
'total_size': row.size or 0,
'average_size': row.avg_size or 0
}
for row in type_result
}
# Get upload statistics by month
monthly_stmt = select(
func.date_trunc('month', Content.created_at).label('month'),
func.count(Content.id).label('uploads'),
func.sum(Content.file_size).label('size')
).where(
Content.user_id == user_id,
Content.status == 'completed',
Content.created_at >= datetime.utcnow().replace(day=1) - timedelta(days=365)
).group_by(func.date_trunc('month', Content.created_at))
monthly_result = await session.execute(monthly_stmt)
monthly_stats = [
{
'month': row.month.isoformat(),
'uploads': row.uploads,
'size': row.size or 0
}
for row in monthly_result
]
return response.json({
"by_type": type_stats,
"monthly": monthly_stats,
"generated_at": datetime.utcnow().isoformat()
})
except Exception as e:
await logger.aerror(
"Failed to get storage stats",
user_id=str(user_id),
error=str(e)
)
return response.json(
{"error": "Failed to get storage statistics", "code": "STATS_FAILED"},
status=500
)
@storage_bp.route("/cleanup", methods=["POST"])
@rate_limit(limit=5, window=3600) # 5 cleanup operations per hour
@require_auth(permissions=["storage.admin"])
async def cleanup_orphaned_files(request: Request) -> JSONResponse:
"""
Clean up orphaned files and incomplete uploads (admin only).
Args:
request: Sanic request object
Returns:
JSONResponse: Cleanup results
"""
try:
if not request.ctx.user.is_admin:
return response.json(
{"error": "Admin access required", "code": "ADMIN_REQUIRED"},
status=403
)
storage_manager = StorageManager()
cache_manager = get_cache_manager()
cleanup_stats = {
"orphaned_chunks": 0,
"expired_sessions": 0,
"failed_uploads": 0,
"freed_space": 0
}
# Clean up expired upload sessions
async with get_async_session() as session:
from app.core.models.storage import ContentUploadSession
from app.core.models.content import Content
# Get expired sessions
expired_sessions_stmt = select(ContentUploadSession).where(
ContentUploadSession.expires_at < datetime.utcnow()
)
expired_result = await session.execute(expired_sessions_stmt)
expired_sessions = expired_result.scalars().all()
for upload_session in expired_sessions:
# Clean up chunks
session_key = f"upload_session:{upload_session.id}"
session_data = await cache_manager.get(session_key)
if session_data:
for chunk_index in session_data.get("uploaded_chunks", []):
chunk_id = f"{upload_session.id}_{chunk_index:06d}"
if await storage_manager.backend.delete_chunk(chunk_id):
cleanup_stats["orphaned_chunks"] += 1
# Delete session
await session.delete(upload_session)
await cache_manager.delete(session_key)
cleanup_stats["expired_sessions"] += 1
# Clean up failed uploads (older than 24 hours)
failed_uploads_stmt = select(Content).where(
Content.status.in_(['uploading', 'processing', 'failed']),
Content.created_at < datetime.utcnow() - timedelta(hours=24)
)
failed_result = await session.execute(failed_uploads_stmt)
failed_uploads = failed_result.scalars().all()
for content in failed_uploads:
if content.file_path:
if await storage_manager.backend.delete_file(content.file_path):
cleanup_stats["freed_space"] += content.file_size or 0
await session.delete(content)
cleanup_stats["failed_uploads"] += 1
await session.commit()
await logger.ainfo(
"Storage cleanup completed",
**cleanup_stats,
admin_user=str(request.ctx.user.id)
)
return response.json({
"status": "cleanup_completed",
"results": cleanup_stats,
"timestamp": datetime.utcnow().isoformat()
})
except Exception as e:
await logger.aerror(
"Storage cleanup failed",
admin_user=str(request.ctx.user.id),
error=str(e)
)
return response.json(
{"error": "Cleanup operation failed", "code": "CLEANUP_FAILED"},
status=500
)