REFACTORING DIRECTORIES

This commit is contained in:
user committed 2025-07-05 20:02:00 +03:00
1 parent c645019380
commit c8e1d5046c
54 files changed
+6193 -747

No files matched your search

+130
View File
@@ -0,0 +1,130 @@
# Multi-stage Dockerfile for optimized production builds
FROM python:3.11-slim as base
# Set environment variables
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
POETRY_VERSION=1.6.1
# Install system dependencies
RUN apt-get update && apt-get install -y \
build-essential \
curl \
ffmpeg \
libmagic1 \
libpq-dev \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
# Install Poetry
RUN pip install poetry==$POETRY_VERSION
# Configure Poetry
ENV POETRY_NO_INTERACTION=1 \
POETRY_VENV_IN_PROJECT=1 \
POETRY_CACHE_DIR=/tmp/poetry_cache
WORKDIR /app
# Copy dependency files
COPY pyproject.toml poetry.lock ./
# Development stage
FROM base as development
# Install dependencies including dev dependencies
RUN poetry install --with dev && rm -rf $POETRY_CACHE_DIR
# Copy source code
COPY . .
# Set development environment
ENV PYTHONPATH=/app
ENV DEBUG=true
# Expose ports
EXPOSE 15100 9090
# Default command for development
CMD ["poetry", "run", "python", "-m", "app"]
# Production dependencies stage
FROM base as deps
# Install only production dependencies
RUN poetry install --only=main && rm -rf $POETRY_CACHE_DIR
# Production stage
FROM python:3.11-slim as production
# Install runtime dependencies only
RUN apt-get update && apt-get install -y \
curl \
ffmpeg \
libmagic1 \
libpq5 \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean
# Create non-root user
RUN groupadd -r appuser && useradd -r -g appuser appuser
# Set working directory
WORKDIR /app
# Copy virtual environment from deps stage
COPY --from=deps /app/.venv /app/.venv
# Add virtual environment to PATH
ENV PATH="/app/.venv/bin:$PATH"
# Copy application code
COPY --chown=appuser:appuser . .
# Create necessary directories
RUN mkdir -p /app/data /app/logs && \
chown -R appuser:appuser /app/data /app/logs
# Set production environment
ENV PYTHONPATH=/app
ENV DEBUG=false
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
CMD curl -f http://localhost:15100/health || exit 1
# Switch to non-root user
USER appuser
# Expose ports
EXPOSE 15100 9090
# Default command
CMD ["python", "-m", "app"]
# Testing stage
FROM development as testing
# Install test dependencies
RUN poetry install --with dev,test
# Run tests
RUN poetry run pytest tests/ --cov=app --cov-report=term-missing
# Security scanning stage
FROM production as security
# Switch back to root for security scanning
USER root
# Install security tools
RUN pip install safety bandit
# Run security checks
RUN safety check
RUN bandit -r app/ -f json -o security-report.json || true
# Switch back to app user
USER appuser
+76
View File
@@ -0,0 +1,76 @@
# Simple Dockerfile using requirements.txt instead of Poetry
FROM python:3.11-slim as base
# Set environment variables
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
# Install system dependencies
RUN apt-get update && apt-get install -y \
build-essential \
curl \
git \
ffmpeg \
libmagic1 \
libpq-dev \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Copy requirements first for better caching
COPY requirements.txt ./requirements.txt
# Development stage
FROM base as development
# Install dependencies
RUN pip install -r requirements.txt
# Copy source code
COPY . .
# Set development environment
ENV PYTHONPATH=/app
ENV DEBUG=true
# Expose ports
EXPOSE 15100 9090
# Default command for development
CMD ["python", "-m", "app"]
# Production stage
FROM base as production
# Install dependencies
RUN pip install -r requirements.txt
# Create non-root user
RUN groupadd -r appuser && useradd -r -g appuser appuser
# Copy application code
COPY --chown=appuser:appuser . .
# Create necessary directories
RUN mkdir -p /app/data /app/logs && \
chown -R appuser:appuser /app/data /app/logs
# Set production environment
ENV PYTHONPATH=/app
ENV DEBUG=false
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
CMD curl -f http://localhost:15100/health || exit 1
# Switch to non-root user
USER appuser
# Expose ports
EXPOSE 15100 9090
# Default command
CMD ["python", "-m", "app"]
+244
View File
@@ -0,0 +1,244 @@
{
"version": "1.0.0",
"network_id": "my-network-mainnet",
"network_name": "MY Distributed Content Network",
"protocol_version": "1.0.0",
"created_at": "2025-01-02T15:00:00Z",
"description": "Bootstrap configuration for MY Network - Distributed content replication system",
"bootstrap_nodes": [
{
"id": "bootstrap-eu-001",
"address": "my://bootstrap-eu-1.mynetwork.io:8080",
"public_key": "ed25519:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
"region": "eu-west-1",
"country": "DE",
"weight": 100,
"features": [
"content_sync",
"consensus",
"dht",
"bootstrap",
"monitoring"
],
"metadata": {
"provider": "hetzner",
"datacenter": "fsn1",
"bandwidth_gbps": 10,
"storage_tb": 100
}
},
{
"id": "bootstrap-us-001",
"address": "my://bootstrap-us-1.mynetwork.io:8080",
"public_key": "ed25519:BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB",
"region": "us-east-1",
"country": "US",
"weight": 100,
"features": [
"content_sync",
"consensus",
"dht",
"bootstrap",
"monitoring"
],
"metadata": {
"provider": "aws",
"datacenter": "us-east-1a",
"bandwidth_gbps": 25,
"storage_tb": 200
}
},
{
"id": "bootstrap-asia-001",
"address": "my://bootstrap-asia-1.mynetwork.io:8080",
"public_key": "ed25519:CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC",
"region": "ap-southeast-1",
"country": "SG",
"weight": 90,
"features": [
"content_sync",
"consensus",
"dht",
"bootstrap"
],
"metadata": {
"provider": "digitalocean",
"datacenter": "sgp1",
"bandwidth_gbps": 5,
"storage_tb": 50
}
}
],
"consensus": {
"algorithm": "pbft",
"min_quorum": 3,
"consensus_threshold": 0.67,
"timeout_seconds": 30,
"retry_attempts": 3,
"max_byzantine_nodes": 1,
"view_change_timeout": 60,
"checkpoint_interval": 100
},
"sync_settings": {
"sync_interval_seconds": 60,
"initial_sync_timeout": 300,
"max_batch_size": 100,
"max_concurrent_syncs": 5,
"compression_enabled": true,
"compression_algorithm": "gzip",
"compression_level": 6,
"encryption_enabled": true,
"encryption_algorithm": "aes-256-gcm",
"delta_sync_enabled": true,
"checksum_verification": true
},
"network_settings": {
"max_peers": 50,
"max_inbound_connections": 100,
"max_outbound_connections": 25,
"connection_timeout_seconds": 30,
"keepalive_interval_seconds": 60,
"max_message_size_bytes": 16777216,
"rate_limit_per_peer": 1000,
"blacklist_duration_hours": 24
},
"content_settings": {
"max_content_size_bytes": 1073741824,
"supported_content_types": [
"audio/*",
"video/*",
"image/*",
"text/*",
"application/pdf",
"application/json"
],
"replication_factor": 3,
"min_replicas": 2,
"max_replicas": 10,
"gc_interval_hours": 24,
"cache_size_mb": 1024,
"preview_generation": true,
"thumbnail_sizes": [64, 128, 256, 512]
},
"security_settings": {
"require_tls": true,
"min_tls_version": "1.3",
"certificate_verification": true,
"peer_authentication": true,
"message_signing": true,
"signature_algorithm": "ed25519",
"key_rotation_days": 90,
"audit_logging": true,
"rate_limiting": true,
"ddos_protection": true
},
"storage_settings": {
"storage_path": "./storage/my-network",
"max_storage_gb": 1000,
"storage_cleanup_threshold": 0.9,
"backup_enabled": true,
"backup_interval_hours": 6,
"backup_retention_days": 30,
"indexing_enabled": true,
"deduplication_enabled": true
},
"monitoring_settings": {
"metrics_enabled": true,
"metrics_interval_seconds": 30,
"health_check_interval_seconds": 60,
"log_level": "info",
"log_rotation_mb": 100,
"log_retention_days": 7,
"alerting_enabled": true,
"webhook_alerts": [],
"telegram_alerts": {
"enabled": false,
"bot_token": "",
"chat_ids": []
}
},
"api_settings": {
"listen_address": "0.0.0.0",
"listen_port": 8080,
"api_prefix": "/api/my",
"cors_enabled": true,
"cors_origins": ["*"],
"auth_required": false,
"rate_limit_rpm": 1000,
"timeout_seconds": 30
},
"development_settings": {
"debug_mode": false,
"test_network": false,
"mock_bootstrap": false,
"simulation_mode": false,
"verbose_logging": false,
"profiling_enabled": false
},
"feature_flags": {
"experimental_dht": false,
"advanced_routing": true,
"content_caching": true,
"peer_scoring": true,
"adaptive_replication": false,
"quantum_encryption": false,
"ml_optimization": false
},
"regional_settings": {
"eu-west-1": {
"preferred_nodes": ["bootstrap-eu-001"],
"max_latency_ms": 100,
"compliance": ["gdpr"],
"data_residency": true
},
"us-east-1": {
"preferred_nodes": ["bootstrap-us-001"],
"max_latency_ms": 150,
"compliance": ["ccpa"],
"data_residency": false
},
"ap-southeast-1": {
"preferred_nodes": ["bootstrap-asia-001"],
"max_latency_ms": 200,
"compliance": [],
"data_residency": false
}
},
"emergency_settings": {
"emergency_mode": false,
"emergency_contacts": [
"admin@mynetwork.io",
"security@mynetwork.io"
],
"auto_isolation": true,
"failsafe_mode": {
"enabled": true,
"max_cpu_percent": 95,
"max_memory_percent": 95,
"max_disk_percent": 98
}
},
"version_compatibility": {
"min_supported_version": "1.0.0",
"max_supported_version": "1.9.9",
"deprecated_versions": [],
"upgrade_required_versions": []
},
"checksum": "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef12",
"signature": "ed25519:DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"
}
+143
View File
@@ -0,0 +1,143 @@
version: '3'
services:
postgres:
image: postgres:15-alpine
ports:
- "5432:5432"
env_file:
- .env
environment:
- POSTGRES_DB=${POSTGRES_DB:-my_uploader_db}
- POSTGRES_USER=${POSTGRES_USER:-my_user}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-secure_password}
- POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
volumes:
- postgres_data:/var/lib/postgresql/data
restart: always
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-my_user} -d ${POSTGRES_DB:-my_uploader_db}"]
interval: 10s
timeout: 5s
retries: 3
redis:
image: redis:7-alpine
ports:
- "6379:6379"
volumes:
- redis_data:/data
restart: always
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 3
app:
build:
context: .
dockerfile: Dockerfile
command: python -m app
env_file:
- .env
restart: always
links:
- postgres
- redis
ports:
- "15100:15100"
volumes:
- /Storage/logs:/app/logs
- /Storage/storedContent:/app/data
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
indexer:
build:
context: .
dockerfile: Dockerfile
restart: always
command: python -m app indexer
env_file:
- .env
links:
- postgres
- redis
volumes:
- /Storage/logs:/app/logs
- /Storage/storedContent:/app/data
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
ton_daemon:
build:
context: .
dockerfile: Dockerfile
command: python -m app ton_daemon
restart: always
env_file:
- .env
links:
- postgres
- redis
volumes:
- /Storage/logs:/app/logs
- /Storage/storedContent:/app/data
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
license_index:
build:
context: .
dockerfile: Dockerfile
command: python -m app license_index
restart: always
env_file:
- .env
links:
- postgres
- redis
volumes:
- /Storage/logs:/app/logs
- /Storage/storedContent:/app/data
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
convert_process:
build:
context: .
dockerfile: Dockerfile
command: python -m app convert_process
restart: always
env_file:
- .env
links:
- postgres
- redis
volumes:
- /Storage/logs:/app/logs
- /Storage/storedContent:/app/data
- /var/run/docker.sock:/var/run/docker.sock
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
volumes:
postgres_data:
driver: local
redis_data:
driver: local
+311
View File
@@ -0,0 +1,311 @@
version: '3.8'
services:
# PostgreSQL Database
postgres:
image: postgres:15-alpine
container_name: uploader_postgres
restart: unless-stopped
environment:
POSTGRES_DB: ${POSTGRES_DB:-uploader_bot}
POSTGRES_USER: ${POSTGRES_USER:-uploader}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-secure_password}
POSTGRES_INITDB_ARGS: "--encoding=UTF-8 --lc-collate=C --lc-ctype=C"
volumes:
- postgres_data:/var/lib/postgresql/data
- ./scripts/init-db-local.sql:/docker-entrypoint-initdb.d/init-db-local.sql:ro
ports:
- "${POSTGRES_PORT:-5432}:5432"
networks:
- uploader_network
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-uploader} -d ${POSTGRES_DB:-uploader_bot}"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
# Redis Cache
redis:
image: redis:7-alpine
container_name: uploader_redis
restart: unless-stopped
command: >
redis-server
--appendonly yes
--maxmemory 512mb
--maxmemory-policy allkeys-lru
--save 900 1
--save 300 10
--save 60 10000
volumes:
- redis_data:/data
- ./config/redis.conf:/usr/local/etc/redis/redis.conf:ro
ports:
- "${REDIS_PORT:-6379}:6379"
networks:
- uploader_network
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 3s
retries: 5
start_period: 10s
# Main Application
app:
build:
context: .
dockerfile: Dockerfile.simple
target: production
container_name: uploader_app
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
# Database
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-uploader}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-uploader_bot}
# Redis
REDIS_URL: redis://redis:6379/0
# Application
PROJECT_HOST: ${PROJECT_HOST:-http://localhost:15100}
SANIC_PORT: 15100
DEBUG: ${DEBUG:-false}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
# Telegram
TELEGRAM_API_KEY: ${TELEGRAM_API_KEY}
CLIENT_TELEGRAM_API_KEY: ${CLIENT_TELEGRAM_API_KEY}
# TON Blockchain
TESTNET: ${TESTNET:-false}
TONCENTER_HOST: ${TONCENTER_HOST:-https://toncenter.com/api/v2/}
TONCENTER_API_KEY: ${TONCENTER_API_KEY}
# Security
SECRET_KEY: ${SECRET_KEY}
JWT_SECRET_KEY: ${JWT_SECRET_KEY}
# File Storage
UPLOADS_DIR: /app/data
# Services
INDEXER_ENABLED: ${INDEXER_ENABLED:-true}
TON_DAEMON_ENABLED: ${TON_DAEMON_ENABLED:-true}
LICENSE_SERVICE_ENABLED: ${LICENSE_SERVICE_ENABLED:-true}
CONVERT_SERVICE_ENABLED: ${CONVERT_SERVICE_ENABLED:-true}
# Monitoring
METRICS_ENABLED: ${METRICS_ENABLED:-true}
HEALTH_CHECK_ENABLED: ${HEALTH_CHECK_ENABLED:-true}
# Rate Limiting
RATE_LIMIT_ENABLED: ${RATE_LIMIT_ENABLED:-true}
RATE_LIMIT_REQUESTS: ${RATE_LIMIT_REQUESTS:-100}
RATE_LIMIT_WINDOW: ${RATE_LIMIT_WINDOW:-60}
volumes:
- app_data:/app/data
- app_logs:/app/logs
- ./config:/app/config:ro
ports:
- "${SANIC_PORT:-15100}:15100"
- "${METRICS_PORT:-9090}:9090"
networks:
- uploader_network
labels:
- "traefik.enable=true"
- "traefik.http.routers.uploader.rule=Host(`${DOMAIN:-localhost}`)"
- "traefik.http.routers.uploader.entrypoints=web,websecure"
- "traefik.http.routers.uploader.tls.certresolver=letsencrypt"
- "traefik.http.services.uploader.loadbalancer.server.port=15100"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:15100/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 60s
# Background Services (Alternative architecture - separate containers)
indexer:
build:
context: .
dockerfile: Dockerfile.simple
target: production
container_name: uploader_indexer
restart: unless-stopped
command: python -m app.services.indexer
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-uploader}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-uploader_bot}
REDIS_URL: redis://redis:6379/0
TELEGRAM_API_KEY: ${TELEGRAM_API_KEY}
CLIENT_TELEGRAM_API_KEY: ${CLIENT_TELEGRAM_API_KEY}
TESTNET: ${TESTNET:-false}
TONCENTER_HOST: ${TONCENTER_HOST:-https://toncenter.com/api/v2/}
TONCENTER_API_KEY: ${TONCENTER_API_KEY}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
SERVICE_NAME: indexer
volumes:
- app_logs:/app/logs
networks:
- uploader_network
profiles:
- separate-services
ton_daemon:
build:
context: .
dockerfile: Dockerfile.simple
target: production
container_name: uploader_ton_daemon
restart: unless-stopped
command: python -m app.services.ton_daemon
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-uploader}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-uploader_bot}
REDIS_URL: redis://redis:6379/0
TESTNET: ${TESTNET:-false}
TONCENTER_HOST: ${TONCENTER_HOST:-https://toncenter.com/api/v2/}
TONCENTER_API_KEY: ${TONCENTER_API_KEY}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
SERVICE_NAME: ton_daemon
volumes:
- app_logs:/app/logs
networks:
- uploader_network
profiles:
- separate-services
# Monitoring and Observability
prometheus:
image: prom/prometheus:latest
container_name: uploader_prometheus
restart: unless-stopped
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--storage.tsdb.retention.time=200h'
- '--web.enable-lifecycle'
volumes:
- ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
ports:
- "9091:9090"
networks:
- uploader_network
profiles:
- monitoring
grafana:
image: grafana/grafana:latest
container_name: uploader_grafana
restart: unless-stopped
environment:
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin}
GF_USERS_ALLOW_SIGN_UP: false
volumes:
- grafana_data:/var/lib/grafana
- ./monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards:ro
- ./monitoring/grafana/datasources:/etc/grafana/provisioning/datasources:ro
ports:
- "3001:3000"
networks:
- uploader_network
profiles:
- monitoring
# Reverse Proxy (optional)
traefik:
image: traefik:v3.0
container_name: uploader_traefik
restart: unless-stopped
command:
- '--api.dashboard=true'
- '--api.insecure=true'
- '--providers.docker=true'
- '--providers.docker.exposedbydefault=false'
- '--entrypoints.web.address=:80'
- '--entrypoints.websecure.address=:443'
- '--certificatesresolvers.letsencrypt.acme.httpchallenge=true'
- '--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web'
- '--certificatesresolvers.letsencrypt.acme.email=${ACME_EMAIL:-admin@example.com}'
- '--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json'
ports:
- "80:80"
- "443:443"
- "8080:8080" # Traefik dashboard
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- traefik_data:/letsencrypt
networks:
- uploader_network
profiles:
- proxy
# Database backup service
postgres_backup:
image: postgres:15-alpine
container_name: uploader_backup
restart: "no"
depends_on:
- postgres
environment:
POSTGRES_DB: ${POSTGRES_DB:-uploader_bot}
POSTGRES_USER: ${POSTGRES_USER:-uploader}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-secure_password}
BACKUP_SCHEDULE: ${BACKUP_SCHEDULE:-0 2 * * *} # Daily at 2 AM
volumes:
- backup_data:/backups
- ./scripts/backup.sh:/backup.sh:ro
command: /backup.sh
networks:
- uploader_network
profiles:
- backup
# Named volumes for data persistence
volumes:
postgres_data:
driver: local
redis_data:
driver: local
app_data:
driver: local
app_logs:
driver: local
prometheus_data:
driver: local
grafana_data:
driver: local
traefik_data:
driver: local
backup_data:
driver: local
# Custom network
networks:
uploader_network:
driver: bridge
ipam:
config:
- subnet: 172.20.0.0/16
+284
View File
@@ -0,0 +1,284 @@
services:
# Основное приложение MY UPLOADER BOT
app:
build:
context: .
dockerfile: Dockerfile.simple
container_name: my-uploader-app
restart: unless-stopped
depends_on:
- postgres
- redis
ports:
- "127.0.0.1:15100:15100"
volumes:
- app_data:/app/data
- app_logs:/app/logs
- ./app/my_network/bootstrap.json:/app/app/my_network/bootstrap.json:ro
- /var/run/docker.sock:/var/run/docker.sock # Для управления converter контейнерами
- converter_shared:/shared/converter # Общая папка для converter заданий
environment:
# Основные настройки
- NODE_ENV=production
- DEBUG=false
# Database
- DATABASE_URL=postgresql://my_user:${POSTGRES_PASSWORD}@postgres:5432/my_uploader_db
- POSTGRES_HOST=postgres
- POSTGRES_PORT=5432
- POSTGRES_DB=my_uploader_db
- POSTGRES_USER=my_user
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
# Redis
- REDIS_URL=redis://redis:6379/0
- REDIS_HOST=redis
- REDIS_PORT=6379
# Security
- SECRET_KEY=${SECRET_KEY}
- JWT_SECRET=${JWT_SECRET}
- ENCRYPTION_KEY=${ENCRYPTION_KEY}
# MY Network
- MY_NETWORK_NODE_ID=${MY_NETWORK_NODE_ID}
- MY_NETWORK_PORT=15100
- MY_NETWORK_HOST=0.0.0.0
- MY_NETWORK_DOMAIN=${MY_NETWORK_DOMAIN}
- MY_NETWORK_SSL_ENABLED=true
# API Settings
- API_HOST=0.0.0.0
- API_PORT=15100
- API_WORKERS=4
- MAX_UPLOAD_SIZE=100MB
# Converter Settings (on-demand)
- CONVERTER_DOCKER_IMAGE=my-converter:latest
- CONVERTER_SHARED_PATH=/shared/converter
- CONVERTER_MAX_PARALLEL=3
- CONVERTER_TIMEOUT=300
# Logging
- LOG_LEVEL=INFO
- LOG_FORMAT=json
networks:
- uploader_network
healthcheck:
test: ["CMD", "python", "-c", "import requests; requests.get('http://localhost:15100/health')"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
# PostgreSQL Database
postgres:
image: postgres:15-alpine
container_name: my-postgres
restart: unless-stopped
volumes:
- postgres_data:/var/lib/postgresql/data
- ./scripts/init-db-production.sql:/docker-entrypoint-initdb.d/01-init.sql:ro
environment:
- POSTGRES_DB=my_uploader_db
- POSTGRES_USER=my_user
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_INITDB_ARGS=--auth-host=md5
ports:
- "127.0.0.1:5432:5432"
networks:
- uploader_network
healthcheck:
test: ["CMD-SHELL", "pg_isready -U my_user -d my_uploader_db"]
interval: 10s
timeout: 5s
retries: 5
# Redis Cache
redis:
image: redis:7-alpine
container_name: my-redis
restart: unless-stopped
volumes:
- redis_data:/data
- ./redis.conf:/usr/local/etc/redis/redis.conf:ro
command: redis-server /usr/local/etc/redis/redis.conf
ports:
- "127.0.0.1:6379:6379"
networks:
- uploader_network
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 3
# Web2 Client
web2-client:
build:
context: ./modules/web2-client
dockerfile: Dockerfile
profiles: ["main-node"] # Only build for main nodes
container_name: my-web2-client
restart: unless-stopped
depends_on:
- app
ports:
- "127.0.0.1:3000:3000"
volumes:
- web2_uploads:/app/uploads
environment:
- NODE_ENV=production
- API_URL=http://app:15100
- UPLOAD_PATH=/app/uploads
networks:
- uploader_network
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:3000/health"]
interval: 30s
timeout: 10s
retries: 3
# Nginx Reverse Proxy
nginx:
image: nginx:alpine
container_name: my-nginx
restart: unless-stopped
depends_on:
- app
ports:
- "80:80"
- "443:443"
volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./nginx/sites-enabled:/etc/nginx/sites-enabled:ro
- ./ssl:/etc/nginx/ssl:ro
- nginx_logs:/var/log/nginx
- ./nginx/html:/usr/share/nginx/html:ro
networks:
- uploader_network
healthcheck:
test: ["CMD", "nginx", "-t"]
interval: 30s
timeout: 10s
retries: 3
# Monitoring Stack (Prometheus + Grafana)
prometheus:
image: prom/prometheus:latest
container_name: my-prometheus
restart: unless-stopped
ports:
- "127.0.0.1:9090:9090"
volumes:
- prometheus_data:/prometheus
- ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--web.enable-lifecycle'
networks:
- uploader_network
grafana:
image: grafana/grafana:latest
container_name: my-grafana
restart: unless-stopped
ports:
- "127.0.0.1:3001:3000"
volumes:
- grafana_data:/var/lib/grafana
- ./monitoring/grafana:/etc/grafana/provisioning:ro
environment:
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
- GF_USERS_ALLOW_SIGN_UP=false
- GF_SERVER_DOMAIN=${MY_NETWORK_DOMAIN}
networks:
- uploader_network
# Log Management
loki:
image: grafana/loki:latest
container_name: my-loki
restart: unless-stopped
ports:
- "127.0.0.1:3100:3100"
volumes:
- loki_data:/loki
- ./monitoring/loki.yml:/etc/loki/local-config.yaml:ro
command: -config.file=/etc/loki/local-config.yaml
networks:
- uploader_network
# Log Collection
promtail:
image: grafana/promtail:latest
container_name: my-promtail
restart: unless-stopped
volumes:
- /var/log:/var/log:ro
- ./monitoring/promtail.yml:/etc/promtail/config.yml:ro
- app_logs:/var/log/app:ro
- nginx_logs:/var/log/nginx:ro
command: -config.file=/etc/promtail/config.yml
networks:
- uploader_network
# =============================================================================
# CONVERTER MODULE BUILD (собирается, но не запускается постоянно)
# =============================================================================
# Этот сервис используется только для сборки image
# Основное приложение будет создавать контейнеры on-demand
converter-build:
build:
context: ./modules/converter-module
dockerfile: Dockerfile
image: my-converter:latest
container_name: my-converter-build
profiles: ["build-only"] # Запускается только при сборке
volumes:
- converter_shared:/shared/converter
networks:
- uploader_network
volumes:
# Application data
app_data:
driver: local
app_logs:
driver: local
# Database
postgres_data:
driver: local
redis_data:
driver: local
# Modules
converter_shared: # Общая папка для on-demand converter заданий
driver: local
web2_uploads:
driver: local
# Infrastructure
nginx_logs:
driver: local
# Monitoring
prometheus_data:
driver: local
grafana_data:
driver: local
loki_data:
driver: local
networks:
uploader_network:
driver: bridge
ipam:
config:
- subnet: 172.20.0.0/16
+254
View File
@@ -0,0 +1,254 @@
version: '3.8'
services:
# PostgreSQL Database
postgres:
image: postgres:15-alpine
container_name: uploader-bot-postgres
restart: unless-stopped
environment:
POSTGRES_DB: ${POSTGRES_DB:-my_uploader_db}
POSTGRES_USER: ${POSTGRES_USER:-my_user}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-secure_password}
POSTGRES_INITDB_ARGS: "--encoding=UTF-8 --lc-collate=C --lc-ctype=C"
volumes:
- postgres_data:/var/lib/postgresql/data
- ./scripts/init-db.sql:/docker-entrypoint-initdb.d/01-init.sql:ro
ports:
- "5432:5432"
networks:
- uploader_network
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-my_user} -d ${POSTGRES_DB:-my_uploader_db}"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
# Redis Cache
redis:
image: redis:7-alpine
container_name: uploader-bot-redis
restart: unless-stopped
command: >
redis-server
--appendonly yes
--maxmemory 512mb
--maxmemory-policy allkeys-lru
--save 900 1
--save 300 10
--save 60 10000
volumes:
- redis_data:/data
ports:
- "6379:6379"
networks:
- uploader_network
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 3s
retries: 5
start_period: 10s
# Main Application (MY Uploader Bot)
app:
build:
context: .
dockerfile: Dockerfile
container_name: uploader-bot-app
command: python -m app
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
# Database
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db}
POSTGRES_HOST: postgres
POSTGRES_PORT: 5432
POSTGRES_DB: ${POSTGRES_DB:-my_uploader_db}
POSTGRES_USER: ${POSTGRES_USER:-my_user}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-secure_password}
# Redis
REDIS_URL: redis://redis:6379/0
REDIS_HOST: redis
REDIS_PORT: 6379
# Application
NODE_ENV: ${NODE_ENV:-production}
DEBUG: ${DEBUG:-false}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
# MY Network
MY_NETWORK_NODE_ID: ${MY_NETWORK_NODE_ID}
MY_NETWORK_PORT: ${MY_NETWORK_PORT:-15100}
MY_NETWORK_HOST: ${MY_NETWORK_HOST:-0.0.0.0}
MY_NETWORK_DOMAIN: ${MY_NETWORK_DOMAIN}
MY_NETWORK_SSL_ENABLED: ${MY_NETWORK_SSL_ENABLED:-true}
# API Settings
API_HOST: ${API_HOST:-0.0.0.0}
API_PORT: ${API_PORT:-15100}
API_WORKERS: ${API_WORKERS:-4}
MAX_UPLOAD_SIZE: ${MAX_UPLOAD_SIZE:-100MB}
# Security
SECRET_KEY: ${SECRET_KEY}
JWT_SECRET: ${JWT_SECRET}
ENCRYPTION_KEY: ${ENCRYPTION_KEY}
# Converter (on-demand)
CONVERTER_DOCKER_IMAGE: ${CONVERTER_DOCKER_IMAGE:-my-converter:latest}
CONVERTER_SHARED_PATH: ${CONVERTER_SHARED_PATH:-/shared/converter}
CONVERTER_MAX_PARALLEL: ${CONVERTER_MAX_PARALLEL:-3}
CONVERTER_TIMEOUT: ${CONVERTER_TIMEOUT:-300}
ports:
- "15100:15100"
volumes:
- app_data:/app/data
- app_logs:/app/logs
- /var/run/docker.sock:/var/run/docker.sock # For on-demand converter containers
- converter_shared:/shared/converter
networks:
- uploader_network
healthcheck:
test: ["CMD", "python", "-c", "import requests; requests.get('http://localhost:15100/health')"]
interval: 30s
timeout: 10s
retries: 3
start_period: 60s
# Indexer Service
indexer:
build:
context: .
dockerfile: Dockerfile
container_name: uploader-bot-indexer
restart: unless-stopped
command: python -m app indexer
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db}
REDIS_URL: redis://redis:6379/0
LOG_LEVEL: ${LOG_LEVEL:-INFO}
SERVICE_NAME: indexer
volumes:
- app_logs:/app/logs
- app_data:/app/data
networks:
- uploader_network
# TON Daemon Service
ton_daemon:
build:
context: .
dockerfile: Dockerfile
container_name: uploader-bot-ton-daemon
command: python -m app ton_daemon
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db}
REDIS_URL: redis://redis:6379/0
LOG_LEVEL: ${LOG_LEVEL:-INFO}
SERVICE_NAME: ton_daemon
volumes:
- app_logs:/app/logs
- app_data:/app/data
networks:
- uploader_network
# License Index Service
license_index:
build:
context: .
dockerfile: Dockerfile
container_name: uploader-bot-license-index
command: python -m app license_index
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db}
REDIS_URL: redis://redis:6379/0
LOG_LEVEL: ${LOG_LEVEL:-INFO}
SERVICE_NAME: license_index
volumes:
- app_logs:/app/logs
- app_data:/app/data
networks:
- uploader_network
# Convert Process Service
convert_process:
build:
context: .
dockerfile: Dockerfile
container_name: uploader-bot-convert-process
command: python -m app convert_process
restart: unless-stopped
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db}
REDIS_URL: redis://redis:6379/0
LOG_LEVEL: ${LOG_LEVEL:-INFO}
SERVICE_NAME: convert_process
volumes:
- app_logs:/app/logs
- app_data:/app/data
- /var/run/docker.sock:/var/run/docker.sock
- converter_shared:/shared/converter
networks:
- uploader_network
# Converter Build (for creating converter image)
converter-build:
build:
context: ./modules/converter-module
dockerfile: Dockerfile
image: my-converter:latest
container_name: uploader-bot-converter-build
profiles: ["build-only"] # Only runs during build
volumes:
- converter_shared:/shared/converter
networks:
- uploader_network
volumes:
postgres_data:
driver: local
redis_data:
driver: local
app_data:
driver: local
app_logs:
driver: local
converter_shared:
driver: local
networks:
uploader_network:
driver: bridge
ipam:
config:
- subnet: 172.20.0.0/16
+5
View File
@@ -0,0 +1,5 @@
SANIC_PORT=8080
TELEGRAM_API_KEY=Paste your telegram api key from @BotFather here
MYSQL_URI=mysql+pymysql://user:password@maria_db:3306
MYSQL_ROOT_PASSWORD=playground
MYSQL_DATABASE=bot_database
+49
View File
@@ -0,0 +1,49 @@
# Existing dependencies from original requirements.txt
sanic==21.9.1
websockets==10.0
sqlalchemy==2.0.23
python-dotenv==1.0.0
pymysql==1.1.0
aiogram==3.13.0
pytonconnect==0.3.0
base58==2.1.1
git+https://github.com/tonfactory/tonsdk.git@3ebbf0b702f48c2519e4c6c425f9514f673b9d48#egg=tonsdk
httpx==0.25.0
docker==7.0.0
pycryptodome==3.20.0
pynacl==1.5.0
aiofiles==23.2.1
pydub==0.25.1
pillow==10.2.0
ffmpeg-python==0.2.0
python-magic==0.4.27
# Additional dependencies for enhanced functionality
# Security and validation
bcrypt==4.1.2
pyjwt==2.8.0
cryptography==41.0.8
email-validator==2.1.0
pydantic==2.5.2
pydantic-settings==2.1.0
# Database and caching
aiomysql==0.2.0
asyncpg==0.29.0
redis==5.0.1
# Monitoring and logging
prometheus-client==0.19.0
structlog==23.2.0
# Development and testing
pytest==7.4.3
pytest-asyncio==0.21.1
black==23.11.0
isort==5.12.0
mypy==1.7.1
bandit==1.7.5
# Optional monitoring (commented out for minimal setup)
# grafana-api==1.0.3
# sentry-sdk==1.38.0
+46
View File
@@ -0,0 +1,46 @@
# Core Framework
sanic==23.12.1
websockets==12.0
# Async Database (PostgreSQL)
sqlalchemy[asyncio]==2.0.23
asyncpg==0.29.0
alembic==1.13.1
# Redis & Caching
redis[hiredis]==5.0.1
aioredis==2.0.1
# Telegram Bot
aiogram==3.13.0
aiohttp==3.9.1
# TON Blockchain
pytonconnect==0.3.0
base58==2.1.1
git+https://github.com/tonfactory/tonsdk.git@3ebbf0b702f48c2519e4c6c425f9514f673b9d48#egg=tonsdk
# HTTP Client
httpx[http2]==0.25.2
# Cryptography
pycryptodome==3.20.0
pynacl==1.5.0
# File Processing
aiofiles==23.2.1
pydub==0.25.1
pillow==10.2.0
ffmpeg-python==0.2.0
python-magic==0.4.27
# Utilities
python-dotenv==1.0.0
docker==7.0.0
# Monitoring & Observability
prometheus-client==0.19.0
structlog==23.2.0
# Validation
pydantic==2.5.2
+46
View File
@@ -0,0 +1,46 @@
# Core Framework
sanic==23.12.1
websockets==12.0
# Async Database
sqlalchemy[asyncio]==2.0.23
asyncpg==0.29.0
alembic==1.13.1
# Redis & Caching
redis[hiredis]==5.0.1
aioredis==2.0.1
# Telegram Bot
aiogram==3.13.0
aiohttp==3.9.1
# TON Blockchain
pytonconnect==0.3.0
base58==2.1.1
git+https://github.com/tonfactory/tonsdk.git@3ebbf0b702f48c2519e4c6c425f9514f673b9d48#egg=tonsdk
# HTTP Client
httpx[http2]==0.25.2
# Cryptography
pycryptodome==3.20.0
pynacl==1.5.0
# File Processing
aiofiles==24.1.0
pydub==0.25.1
pillow==10.2.0
ffmpeg-python==0.2.0
python-magic==0.4.27
# Utilities
python-dotenv==1.0.0
docker==7.0.0
# Monitoring & Observability
prometheus-client==0.19.0
structlog==23.2.0
# Validation
pydantic==2.5.2