From 2b9fbb6c7de64a5fca39a3a295eed343f769ebfb Mon Sep 17 00:00:00 2001 From: user Date: Fri, 4 Jul 2025 12:33:03 +0300 Subject: [PATCH] mariadb -> postgres --- docker-compose.compatible.yml | 43 +++-- docker-compose.yml | 271 +++++++++++++++++++++------ requirements.txt | 42 ++++- scripts/init-db.sql | 336 +++++----------------------------- setup_production_server.sh | 210 +++++++++++++++------ 5 files changed, 461 insertions(+), 441 deletions(-) diff --git a/docker-compose.compatible.yml b/docker-compose.compatible.yml index 07477c7..2f1e093 100644 --- a/docker-compose.compatible.yml +++ b/docker-compose.compatible.yml @@ -1,21 +1,21 @@ version: '3' services: - maria_db: - image: mariadb:11.2 + postgres: + image: postgres:15-alpine ports: - - "3307:3306" + - "5432:5432" env_file: - .env environment: - - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-password} - - MYSQL_DATABASE=${MYSQL_DATABASE:-myuploader} - - MYSQL_USER=${MYSQL_USER:-myuploader} - - MYSQL_PASSWORD=${MYSQL_PASSWORD:-password} + - POSTGRES_DB=${POSTGRES_DB:-my_uploader_db} + - POSTGRES_USER=${POSTGRES_USER:-my_user} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-secure_password} + - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C volumes: - - /Storage/sqlStorage:/var/lib/mysql + - postgres_data:/var/lib/postgresql/data restart: always healthcheck: - test: [ "CMD", "healthcheck.sh", "--connect", "--innodb_initialized" ] + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-my_user} -d ${POSTGRES_DB:-my_uploader_db}"] interval: 10s timeout: 5s retries: 3 @@ -42,7 +42,7 @@ services: - .env restart: always links: - - maria_db + - postgres - redis ports: - "15100:15100" @@ -50,7 +50,7 @@ services: - /Storage/logs:/app/logs - /Storage/storedContent:/app/data depends_on: - maria_db: + postgres: condition: service_healthy redis: condition: service_healthy @@ -64,13 +64,13 @@ services: env_file: - .env links: - - maria_db + - postgres - redis volumes: - /Storage/logs:/app/logs - /Storage/storedContent:/app/data depends_on: - maria_db: + postgres: condition: service_healthy redis: condition: service_healthy @@ -84,13 +84,13 @@ services: env_file: - .env links: - - maria_db + - postgres - redis volumes: - /Storage/logs:/app/logs - /Storage/storedContent:/app/data depends_on: - maria_db: + postgres: condition: service_healthy redis: condition: service_healthy @@ -104,13 +104,13 @@ services: env_file: - .env links: - - maria_db + - postgres - redis volumes: - /Storage/logs:/app/logs - /Storage/storedContent:/app/data depends_on: - maria_db: + postgres: condition: service_healthy redis: condition: service_healthy @@ -124,17 +124,20 @@ services: env_file: - .env links: - - maria_db + - postgres - redis volumes: - /Storage/logs:/app/logs - /Storage/storedContent:/app/data - /var/run/docker.sock:/var/run/docker.sock depends_on: - maria_db: + postgres: condition: service_healthy redis: condition: service_healthy volumes: - redis_data: \ No newline at end of file + postgres_data: + driver: local + redis_data: + driver: local \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b1e9b6f..9d9c856 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,105 +1,254 @@ -version: '3' +version: '3.8' + services: - maria_db: - image: mariadb:11.2 - ports: - - "3307:3306" - env_file: - - .env + # PostgreSQL Database + postgres: + image: postgres:15-alpine + container_name: uploader-bot-postgres + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-my_uploader_db} + POSTGRES_USER: ${POSTGRES_USER:-my_user} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-secure_password} + POSTGRES_INITDB_ARGS: "--encoding=UTF-8 --lc-collate=C --lc-ctype=C" volumes: - - /Storage/sqlStorage:/var/lib/mysql - restart: always + - postgres_data:/var/lib/postgresql/data + - ./scripts/init-db.sql:/docker-entrypoint-initdb.d/01-init.sql:ro + ports: + - "5432:5432" + networks: + - uploader_network healthcheck: - test: [ "CMD", "healthcheck.sh", "--connect", "--innodb_initialized" ] + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-my_user} -d ${POSTGRES_DB:-my_uploader_db}"] interval: 10s timeout: 5s - retries: 3 + retries: 5 + start_period: 30s + # Redis Cache + redis: + image: redis:7-alpine + container_name: uploader-bot-redis + restart: unless-stopped + command: > + redis-server + --appendonly yes + --maxmemory 512mb + --maxmemory-policy allkeys-lru + --save 900 1 + --save 300 10 + --save 60 10000 + volumes: + - redis_data:/data + ports: + - "6379:6379" + networks: + - uploader_network + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 10s + + # Main Application (MY Uploader Bot) app: build: context: . dockerfile: Dockerfile + container_name: uploader-bot-app command: python -m app - env_file: - - .env - restart: always - links: - - maria_db + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + environment: + # Database + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db} + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_DB: ${POSTGRES_DB:-my_uploader_db} + POSTGRES_USER: ${POSTGRES_USER:-my_user} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-secure_password} + + # Redis + REDIS_URL: redis://redis:6379/0 + REDIS_HOST: redis + REDIS_PORT: 6379 + + # Application + NODE_ENV: ${NODE_ENV:-production} + DEBUG: ${DEBUG:-false} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + + # MY Network + MY_NETWORK_NODE_ID: ${MY_NETWORK_NODE_ID} + MY_NETWORK_PORT: ${MY_NETWORK_PORT:-15100} + MY_NETWORK_HOST: ${MY_NETWORK_HOST:-0.0.0.0} + MY_NETWORK_DOMAIN: ${MY_NETWORK_DOMAIN} + MY_NETWORK_SSL_ENABLED: ${MY_NETWORK_SSL_ENABLED:-true} + + # API Settings + API_HOST: ${API_HOST:-0.0.0.0} + API_PORT: ${API_PORT:-15100} + API_WORKERS: ${API_WORKERS:-4} + MAX_UPLOAD_SIZE: ${MAX_UPLOAD_SIZE:-100MB} + + # Security + SECRET_KEY: ${SECRET_KEY} + JWT_SECRET: ${JWT_SECRET} + ENCRYPTION_KEY: ${ENCRYPTION_KEY} + + # Converter (on-demand) + CONVERTER_DOCKER_IMAGE: ${CONVERTER_DOCKER_IMAGE:-my-converter:latest} + CONVERTER_SHARED_PATH: ${CONVERTER_SHARED_PATH:-/shared/converter} + CONVERTER_MAX_PARALLEL: ${CONVERTER_MAX_PARALLEL:-3} + CONVERTER_TIMEOUT: ${CONVERTER_TIMEOUT:-300} + ports: - "15100:15100" volumes: - - /Storage/logs:/app/logs - - /Storage/storedContent:/app/data - depends_on: - maria_db: - condition: service_healthy + - app_data:/app/data + - app_logs:/app/logs + - /var/run/docker.sock:/var/run/docker.sock # For on-demand converter containers + - converter_shared:/shared/converter + networks: + - uploader_network + healthcheck: + test: ["CMD", "python", "-c", "import requests; requests.get('http://localhost:15100/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s - indexer: # Отправка уведомления о появлении новой NFT-listen. Установка CID поля у всего контента. Проверка следующего за последним индексом item коллекции и поиск нового контента, отправка информации о том что контент найден его загружателю. Присваивание encrypted_content onchain_index + # Indexer Service + indexer: build: context: . dockerfile: Dockerfile - restart: always + container_name: uploader-bot-indexer + restart: unless-stopped command: python -m app indexer - env_file: - - .env - links: - - maria_db - volumes: - - /Storage/logs:/app/logs - - /Storage/storedContent:/app/data depends_on: - maria_db: + postgres: condition: service_healthy + redis: + condition: service_healthy + environment: + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db} + REDIS_URL: redis://redis:6379/0 + LOG_LEVEL: ${LOG_LEVEL:-INFO} + SERVICE_NAME: indexer + volumes: + - app_logs:/app/logs + - app_data:/app/data + networks: + - uploader_network - ton_daemon: # Работа с TON-сетью. Задачи сервисного кошелька и деплой контрактов + # TON Daemon Service + ton_daemon: build: context: . dockerfile: Dockerfile + container_name: uploader-bot-ton-daemon command: python -m app ton_daemon - restart: always - env_file: - - .env - links: - - maria_db - volumes: - - /Storage/logs:/app/logs - - /Storage/storedContent:/app/data + restart: unless-stopped depends_on: - maria_db: + postgres: condition: service_healthy + redis: + condition: service_healthy + environment: + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db} + REDIS_URL: redis://redis:6379/0 + LOG_LEVEL: ${LOG_LEVEL:-INFO} + SERVICE_NAME: ton_daemon + volumes: + - app_logs:/app/logs + - app_data:/app/data + networks: + - uploader_network - license_index: # Проверка кошельков пользователей на новые NFT. Опрос этих NFT на определяемый GET-метод по которому мы определяем что это определенная лицензия и сохранение информации по ней + # License Index Service + license_index: build: context: . dockerfile: Dockerfile + container_name: uploader-bot-license-index command: python -m app license_index - restart: always - env_file: - - .env - links: - - maria_db - volumes: - - /Storage/logs:/app/logs - - /Storage/storedContent:/app/data + restart: unless-stopped depends_on: - maria_db: + postgres: condition: service_healthy + redis: + condition: service_healthy + environment: + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db} + REDIS_URL: redis://redis:6379/0 + LOG_LEVEL: ${LOG_LEVEL:-INFO} + SERVICE_NAME: license_index + volumes: + - app_logs:/app/logs + - app_data:/app/data + networks: + - uploader_network + # Convert Process Service convert_process: build: context: . dockerfile: Dockerfile + container_name: uploader-bot-convert-process command: python -m app convert_process - restart: always - env_file: - - .env - links: - - maria_db - volumes: - - /Storage/logs:/app/logs - - /Storage/storedContent:/app/data - - /var/run/docker.sock:/var/run/docker.sock + restart: unless-stopped depends_on: - maria_db: + postgres: condition: service_healthy + redis: + condition: service_healthy + environment: + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-my_user}:${POSTGRES_PASSWORD:-secure_password}@postgres:5432/${POSTGRES_DB:-my_uploader_db} + REDIS_URL: redis://redis:6379/0 + LOG_LEVEL: ${LOG_LEVEL:-INFO} + SERVICE_NAME: convert_process + volumes: + - app_logs:/app/logs + - app_data:/app/data + - /var/run/docker.sock:/var/run/docker.sock + - converter_shared:/shared/converter + networks: + - uploader_network + # Converter Build (for creating converter image) + converter-build: + build: + context: ./modules/converter-module + dockerfile: Dockerfile + image: my-converter:latest + container_name: uploader-bot-converter-build + profiles: ["build-only"] # Only runs during build + volumes: + - converter_shared:/shared/converter + networks: + - uploader_network + +volumes: + postgres_data: + driver: local + redis_data: + driver: local + app_data: + driver: local + app_logs: + driver: local + converter_shared: + driver: local + +networks: + uploader_network: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/16 diff --git a/requirements.txt b/requirements.txt index 08aa8aa..72d0c13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,46 @@ -sanic==21.9.1 -websockets==10.0 -sqlalchemy==2.0.23 -python-dotenv==1.0.0 -pymysql==1.1.0 +# Core Framework +sanic==23.12.1 +websockets==12.0 + +# Async Database (PostgreSQL) +sqlalchemy[asyncio]==2.0.23 +asyncpg==0.29.0 +alembic==1.13.1 + +# Redis & Caching +redis[hiredis]==5.0.1 +aioredis==2.0.1 + +# Telegram Bot aiogram==3.13.0 +aiohttp==3.9.1 + +# TON Blockchain pytonconnect==0.3.0 base58==2.1.1 git+https://github.com/tonfactory/tonsdk.git@3ebbf0b702f48c2519e4c6c425f9514f673b9d48#egg=tonsdk -httpx==0.25.0 -docker==7.0.0 + +# HTTP Client +httpx[http2]==0.25.2 + +# Cryptography pycryptodome==3.20.0 pynacl==1.5.0 -aiofiles==23.2.1 + +# File Processing +aiofiles==24.1.0 pydub==0.25.1 pillow==10.2.0 ffmpeg-python==0.2.0 python-magic==0.4.27 +# Utilities +python-dotenv==1.0.0 +docker==7.0.0 +# Monitoring & Observability +prometheus-client==0.19.0 +structlog==23.2.0 + +# Validation +pydantic==2.5.2 diff --git a/scripts/init-db.sql b/scripts/init-db.sql index bb20bb4..f75f8b8 100644 --- a/scripts/init-db.sql +++ b/scripts/init-db.sql @@ -1,311 +1,57 @@ --- PostgreSQL initialization script for my-uploader-bot --- This script sets up the database, users, and extensions +-- PostgreSQL Database Initialization Script for MY Uploader Bot +-- This script sets up the initial database structure and users --- Create database if it doesn't exist -SELECT 'CREATE DATABASE myuploader' -WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'myuploader')\gexec +-- Create database if not exists (done by PostgreSQL container initialization) +-- Create user and grant privileges are also handled by container environment variables --- Connect to the database -\c myuploader; +-- Set up database encoding and collation (handled by POSTGRES_INITDB_ARGS) --- Create extensions +-- Create extensions if needed CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; CREATE EXTENSION IF NOT EXISTS "pg_trgm"; CREATE EXTENSION IF NOT EXISTS "btree_gin"; -CREATE EXTENSION IF NOT EXISTS "pgcrypto"; --- Create custom types -DO $$ BEGIN - CREATE TYPE user_role_type AS ENUM ('admin', 'user', 'moderator'); -EXCEPTION - WHEN duplicate_object THEN null; -END $$; +-- Create schemas for better organization +CREATE SCHEMA IF NOT EXISTS my_network; +CREATE SCHEMA IF NOT EXISTS content; +CREATE SCHEMA IF NOT EXISTS users; +CREATE SCHEMA IF NOT EXISTS analytics; -DO $$ BEGIN - CREATE TYPE content_status_type AS ENUM ('pending', 'uploading', 'processing', 'completed', 'failed', 'deleted'); -EXCEPTION - WHEN duplicate_object THEN null; -END $$; +-- Grant permissions on schemas +GRANT USAGE, CREATE ON SCHEMA my_network TO my_user; +GRANT USAGE, CREATE ON SCHEMA content TO my_user; +GRANT USAGE, CREATE ON SCHEMA users TO my_user; +GRANT USAGE, CREATE ON SCHEMA analytics TO my_user; -DO $$ BEGIN - CREATE TYPE transaction_status_type AS ENUM ('pending', 'confirmed', 'failed', 'cancelled'); -EXCEPTION - WHEN duplicate_object THEN null; -END $$; +-- Set default search path +ALTER DATABASE my_uploader_db SET search_path TO public, my_network, content, users, analytics; --- Create application user (for connection pooling) -DO $$ BEGIN - CREATE USER app_user WITH PASSWORD 'secure_app_password'; -EXCEPTION - WHEN duplicate_object THEN - ALTER USER app_user WITH PASSWORD 'secure_app_password'; -END $$; +-- Create initial tables structure (if not handled by ORM migrations) +-- These will be created by the application's migration system --- Grant necessary permissions -GRANT CONNECT ON DATABASE myuploader TO app_user; -GRANT USAGE ON SCHEMA public TO app_user; - --- Grant table permissions (will be applied after tables are created) -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO app_user; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO app_user; - --- Create read-only user for monitoring/analytics -DO $$ BEGIN - CREATE USER readonly_user WITH PASSWORD 'readonly_password'; -EXCEPTION - WHEN duplicate_object THEN - ALTER USER readonly_user WITH PASSWORD 'readonly_password'; -END $$; - -GRANT CONNECT ON DATABASE myuploader TO readonly_user; -GRANT USAGE ON SCHEMA public TO readonly_user; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO readonly_user; - --- Create backup user -DO $$ BEGIN - CREATE USER backup_user WITH PASSWORD 'backup_password'; -EXCEPTION - WHEN duplicate_object THEN - ALTER USER backup_user WITH PASSWORD 'backup_password'; -END $$; - -GRANT CONNECT ON DATABASE myuploader TO backup_user; -GRANT USAGE ON SCHEMA public TO backup_user; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO backup_user; - --- Create performance monitoring functions -CREATE OR REPLACE FUNCTION get_table_stats() -RETURNS TABLE ( - schema_name TEXT, - table_name TEXT, - row_count BIGINT, - total_size TEXT, - index_size TEXT, - toast_size TEXT -) AS $$ -BEGIN - RETURN QUERY - SELECT - schemaname::TEXT, - tablename::TEXT, - n_tup_ins - n_tup_del AS row_count, - pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS total_size, - pg_size_pretty(pg_indexes_size(schemaname||'.'||tablename)) AS index_size, - pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename) - pg_relation_size(schemaname||'.'||tablename)) AS toast_size - FROM pg_stat_user_tables - WHERE schemaname = 'public' - ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Create index monitoring function -CREATE OR REPLACE FUNCTION get_unused_indexes() -RETURNS TABLE ( - schema_name TEXT, - table_name TEXT, - index_name TEXT, - index_size TEXT, - index_scans BIGINT -) AS $$ -BEGIN - RETURN QUERY - SELECT - schemaname::TEXT, - tablename::TEXT, - indexname::TEXT, - pg_size_pretty(pg_relation_size(indexrelid)) AS index_size, - idx_scan - FROM pg_stat_user_indexes - WHERE schemaname = 'public' - AND idx_scan < 100 -- Indexes used less than 100 times - ORDER BY pg_relation_size(indexrelid) DESC; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Create slow query logging configuration -ALTER SYSTEM SET log_min_duration_statement = 1000; -- Log queries taking more than 1 second -ALTER SYSTEM SET log_statement = 'mod'; -- Log modifications -ALTER SYSTEM SET log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '; - --- Create audit log table for sensitive operations -CREATE TABLE IF NOT EXISTS audit_log ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - user_id UUID, - action VARCHAR(50) NOT NULL, - table_name VARCHAR(50), - record_id UUID, - old_values JSONB, - new_values JSONB, - ip_address INET, - user_agent TEXT, - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +-- Example: Create a simple health check table +CREATE TABLE IF NOT EXISTS health_check ( + id SERIAL PRIMARY KEY, + status VARCHAR(50) NOT NULL DEFAULT 'ok', + timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + details JSONB ); -CREATE INDEX IF NOT EXISTS idx_audit_log_user_id ON audit_log(user_id); -CREATE INDEX IF NOT EXISTS idx_audit_log_action ON audit_log(action); -CREATE INDEX IF NOT EXISTS idx_audit_log_created_at ON audit_log(created_at); -CREATE INDEX IF NOT EXISTS idx_audit_log_table_name ON audit_log(table_name); +-- Insert initial health check record +INSERT INTO health_check (status, details) +VALUES ('initialized', '{"message": "Database initialized successfully", "version": "1.0.0"}') +ON CONFLICT DO NOTHING; --- Create audit trigger function -CREATE OR REPLACE FUNCTION audit_trigger_function() -RETURNS TRIGGER AS $$ -BEGIN - IF TG_OP = 'DELETE' THEN - INSERT INTO audit_log (action, table_name, record_id, old_values) - VALUES (TG_OP, TG_TABLE_NAME, OLD.id, row_to_json(OLD)); - RETURN OLD; - ELSIF TG_OP = 'UPDATE' THEN - INSERT INTO audit_log (action, table_name, record_id, old_values, new_values) - VALUES (TG_OP, TG_TABLE_NAME, NEW.id, row_to_json(OLD), row_to_json(NEW)); - RETURN NEW; - ELSIF TG_OP = 'INSERT' THEN - INSERT INTO audit_log (action, table_name, record_id, new_values) - VALUES (TG_OP, TG_TABLE_NAME, NEW.id, row_to_json(NEW)); - RETURN NEW; - END IF; - RETURN NULL; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; +-- Create indexes for better performance +CREATE INDEX IF NOT EXISTS idx_health_check_timestamp ON health_check(timestamp); +CREATE INDEX IF NOT EXISTS idx_health_check_status ON health_check(status); --- Create cleanup function for old audit logs -CREATE OR REPLACE FUNCTION cleanup_old_audit_logs(retention_days INTEGER DEFAULT 90) -RETURNS INTEGER AS $$ -DECLARE - deleted_count INTEGER; -BEGIN - DELETE FROM audit_log - WHERE created_at < NOW() - INTERVAL '1 day' * retention_days; - - GET DIAGNOSTICS deleted_count = ROW_COUNT; - RETURN deleted_count; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; +-- Log successful initialization +INSERT INTO health_check (status, details) +VALUES ('ready', '{"message": "PostgreSQL initialization completed", "timestamp": "' || NOW() || '"}'); --- Create maintenance function -CREATE OR REPLACE FUNCTION run_maintenance() -RETURNS TEXT AS $$ -DECLARE - result TEXT := ''; - rec RECORD; -BEGIN - -- Update table statistics - ANALYZE; - result := result || 'Statistics updated. '; - - -- Vacuum analyze all tables - FOR rec IN SELECT tablename FROM pg_tables WHERE schemaname = 'public' LOOP - EXECUTE 'VACUUM ANALYZE ' || quote_ident(rec.tablename); - END LOOP; - result := result || 'Vacuum completed. '; - - -- Cleanup old audit logs (keep 90 days) - result := result || 'Cleaned up ' || cleanup_old_audit_logs(90) || ' old audit logs. '; - - -- Reindex if needed (check for bloat) - FOR rec IN - SELECT schemaname, tablename - FROM pg_stat_user_tables - WHERE n_dead_tup > n_live_tup * 0.1 - AND n_live_tup > 1000 - LOOP - EXECUTE 'REINDEX TABLE ' || quote_ident(rec.schemaname) || '.' || quote_ident(rec.tablename); - result := result || 'Reindexed ' || rec.tablename || '. '; - END LOOP; - - RETURN result; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Create backup verification function -CREATE OR REPLACE FUNCTION verify_backup_integrity() -RETURNS TABLE ( - table_name TEXT, - row_count BIGINT, - last_modified TIMESTAMP WITH TIME ZONE, - checksum TEXT -) AS $$ -BEGIN - RETURN QUERY - SELECT - t.tablename::TEXT, - t.n_live_tup, - GREATEST(t.last_vacuum, t.last_autovacuum, t.last_analyze, t.last_autoanalyze), - md5(string_agg(c.column_name, ',' ORDER BY c.ordinal_position)) - FROM pg_stat_user_tables t - JOIN information_schema.columns c ON c.table_name = t.tablename - WHERE t.schemaname = 'public' - GROUP BY t.tablename, t.n_live_tup, - GREATEST(t.last_vacuum, t.last_autovacuum, t.last_analyze, t.last_autoanalyze) - ORDER BY t.tablename; -END; -$$ LANGUAGE plpgsql SECURITY DEFINER; - --- Create connection monitoring view -CREATE OR REPLACE VIEW active_connections AS -SELECT - pid, - usename, - application_name, - client_addr, - client_port, - backend_start, - state, - query_start, - LEFT(query, 100) as query_preview -FROM pg_stat_activity -WHERE state != 'idle' -AND pid != pg_backend_pid() -ORDER BY backend_start; - --- Grant permissions for monitoring functions -GRANT EXECUTE ON FUNCTION get_table_stats() TO readonly_user; -GRANT EXECUTE ON FUNCTION get_unused_indexes() TO readonly_user; -GRANT EXECUTE ON FUNCTION verify_backup_integrity() TO backup_user; -GRANT SELECT ON active_connections TO readonly_user; - --- Set up automatic maintenance schedule (requires pg_cron extension) --- Uncomment if pg_cron is available --- SELECT cron.schedule('database-maintenance', '0 2 * * 0', 'SELECT run_maintenance();'); --- SELECT cron.schedule('audit-cleanup', '0 3 * * *', 'SELECT cleanup_old_audit_logs(90);'); - --- Create performance tuning settings -ALTER SYSTEM SET shared_preload_libraries = 'pg_stat_statements'; -ALTER SYSTEM SET track_activity_query_size = 2048; -ALTER SYSTEM SET track_functions = 'all'; -ALTER SYSTEM SET track_io_timing = 'on'; - --- Connection pooling settings -ALTER SYSTEM SET max_connections = 200; -ALTER SYSTEM SET shared_buffers = '256MB'; -ALTER SYSTEM SET effective_cache_size = '1GB'; -ALTER SYSTEM SET maintenance_work_mem = '64MB'; -ALTER SYSTEM SET checkpoint_completion_target = 0.9; -ALTER SYSTEM SET wal_buffers = '16MB'; -ALTER SYSTEM SET default_statistics_target = 100; -ALTER SYSTEM SET random_page_cost = 1.1; -ALTER SYSTEM SET effective_io_concurrency = 200; - --- Security settings -ALTER SYSTEM SET ssl = 'on'; -ALTER SYSTEM SET log_connections = 'on'; -ALTER SYSTEM SET log_disconnections = 'on'; -ALTER SYSTEM SET log_checkpoints = 'on'; -ALTER SYSTEM SET log_lock_waits = 'on'; - --- Reload configuration -SELECT pg_reload_conf(); - --- Create initial admin user (password should be changed immediately) --- This will be handled by the application during first startup - --- Display completion message -DO $$ -BEGIN - RAISE NOTICE 'Database initialization completed successfully!'; - RAISE NOTICE 'Remember to:'; - RAISE NOTICE '1. Change default passwords for app_user, readonly_user, and backup_user'; - RAISE NOTICE '2. Configure SSL certificates'; - RAISE NOTICE '3. Set up regular backups'; - RAISE NOTICE '4. Run initial migrations with Alembic'; - RAISE NOTICE '5. Create your first admin user through the application'; -END $$; \ No newline at end of file +COMMENT ON DATABASE my_uploader_db IS 'MY Uploader Bot - Main database for content management and MY Network protocol'; +COMMENT ON SCHEMA my_network IS 'MY Network protocol related tables and functions'; +COMMENT ON SCHEMA content IS 'Content storage and management tables'; +COMMENT ON SCHEMA users IS 'User management and authentication tables'; +COMMENT ON SCHEMA analytics IS 'Analytics and metrics tables'; \ No newline at end of file diff --git a/setup_production_server.sh b/setup_production_server.sh index ee5cd9e..fc6e5a2 100644 --- a/setup_production_server.sh +++ b/setup_production_server.sh @@ -26,6 +26,7 @@ EMAIL="${3:-admin@${1}}" PROJECT_DIR="/home/myuploader" SERVICE_USER="myuploader" PROGRESS_FILE="/home/myuploader/.installation_progress" +MAIN_PROJECT_DIR="$PROJECT_DIR/uploader-bot" # Репозитории для клонирования UPLOADER_REPO="https://git.projscale.dev/my-dev/uploader-bot" @@ -407,16 +408,65 @@ EOF chown "$SERVICE_USER:$SERVICE_USER" "$MAIN_PROJECT_DIR/.env" log_success "Файл .env создан" - # Создание симлинков для модулей в uploader-bot - log "Создание симлинков для модулей..." + # Создание папок и симлинков для модулей + log "Создание папок и симлинков для модулей..." sudo -u "$SERVICE_USER" mkdir -p "$MAIN_PROJECT_DIR/modules" + sudo -u "$SERVICE_USER" mkdir -p "$MAIN_PROJECT_DIR/scripts" + + # Создание симлинков для модулей sudo -u "$SERVICE_USER" ln -sf "$PROJECT_DIR/converter-module" "$MAIN_PROJECT_DIR/modules/converter-module" if [ "$NODE_TYPE" = "main" ]; then sudo -u "$SERVICE_USER" ln -sf "$PROJECT_DIR/web2-client" "$MAIN_PROJECT_DIR/modules/web2-client" fi - log_success "Симлинки созданы" + # Создание init-db.sql если его нет + if [ ! -f "$MAIN_PROJECT_DIR/scripts/init-db.sql" ]; then + log "Создание init-db.sql..." + cat > "$MAIN_PROJECT_DIR/scripts/init-db.sql" << 'EOF' +-- PostgreSQL Database Initialization Script for MY Uploader Bot +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +CREATE EXTENSION IF NOT EXISTS "pg_trgm"; +CREATE EXTENSION IF NOT EXISTS "btree_gin"; + +-- Create schemas for better organization +CREATE SCHEMA IF NOT EXISTS my_network; +CREATE SCHEMA IF NOT EXISTS content; +CREATE SCHEMA IF NOT EXISTS users; +CREATE SCHEMA IF NOT EXISTS analytics; + +-- Grant permissions on schemas +GRANT USAGE, CREATE ON SCHEMA my_network TO my_user; +GRANT USAGE, CREATE ON SCHEMA content TO my_user; +GRANT USAGE, CREATE ON SCHEMA users TO my_user; +GRANT USAGE, CREATE ON SCHEMA analytics TO my_user; + +-- Set default search path +ALTER DATABASE my_uploader_db SET search_path TO public, my_network, content, users, analytics; + +-- Create health check table +CREATE TABLE IF NOT EXISTS health_check ( + id SERIAL PRIMARY KEY, + status VARCHAR(50) NOT NULL DEFAULT 'ok', + timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + details JSONB +); + +-- Insert initial health check record +INSERT INTO health_check (status, details) +VALUES ('initialized', '{"message": "Database initialized successfully", "version": "1.0.0"}') +ON CONFLICT DO NOTHING; + +-- Create indexes for better performance +CREATE INDEX IF NOT EXISTS idx_health_check_timestamp ON health_check(timestamp); +CREATE INDEX IF NOT EXISTS idx_health_check_status ON health_check(status); + +COMMENT ON DATABASE my_uploader_db IS 'MY Uploader Bot - Main database for content management and MY Network protocol'; +EOF + chown "$SERVICE_USER:$SERVICE_USER" "$MAIN_PROJECT_DIR/scripts/init-db.sql" + fi + + log_success "Папки и симлинки созданы" log_progress "environment_setup" fi @@ -427,6 +477,9 @@ echo -e "${BLUE}==========================================${NC}" echo "" if ! check_and_skip "nginx_config" "Настройка Nginx"; then + # Определяем MAIN_PROJECT_DIR если не определена + MAIN_PROJECT_DIR="$PROJECT_DIR/uploader-bot" + log "Настройка Nginx конфигурации..." # Сначала добавляем rate limiting в основной конфиг nginx @@ -447,27 +500,28 @@ if ! check_and_skip "nginx_config" "Настройка Nginx"; then mkdir -p /etc/nginx/sites-enabled if [ "$NODE_TYPE" = "main" ]; then - # Конфигурация для основной ноды (с web2-client) + # Конфигурация для основной ноды (с web2-client) - только HTTP сначала cat > "/etc/nginx/sites-available/$DOMAIN" << EOF server { listen 80; server_name $DOMAIN; - # Redirect all HTTP requests to HTTPS - return 301 https://\$server_name\$request_uri; -} - -server { - listen 443 ssl http2; - server_name $DOMAIN; + # File upload limits + client_max_body_size 100M; + client_body_timeout 600s; + proxy_read_timeout 600s; + proxy_connect_timeout 600s; + proxy_send_timeout 600s; - # SSL configuration will be added by certbot + # Hide server version + server_tokens off; - # Security headers - add_header X-Frame-Options DENY; - add_header X-Content-Type-Options nosniff; - add_header X-XSS-Protection "1; mode=block"; - add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + # Deny access to hidden files + location ~ /\. { + deny all; + access_log off; + log_not_found off; + } # Web2 Client (main page for main nodes) location / { @@ -513,6 +567,8 @@ server { proxy_read_timeout 600s; proxy_send_timeout 600s; client_max_body_size 100M; + proxy_buffering off; + proxy_request_buffering off; } # Converter API (on-demand, через uploader-bot) @@ -533,6 +589,7 @@ server { # Health check (no rate limiting) location /health { proxy_pass http://127.0.0.1:15100/health; + proxy_set_header Host \$host; access_log off; } @@ -542,37 +599,31 @@ server { expires 30d; add_header Cache-Control "public, immutable"; } - - # Disable access to hidden files - location ~ /\. { - deny all; - access_log off; - log_not_found off; - } } EOF else - # Конфигурация для обычной ноды (только API) + # Конфигурация для обычной ноды (только API) - только HTTP сначала cat > "/etc/nginx/sites-available/$DOMAIN" << EOF server { listen 80; server_name $DOMAIN; - # Redirect all HTTP requests to HTTPS - return 301 https://\$server_name\$request_uri; -} - -server { - listen 443 ssl http2; - server_name $DOMAIN; + # File upload limits + client_max_body_size 100M; + client_body_timeout 600s; + proxy_read_timeout 600s; + proxy_connect_timeout 600s; + proxy_send_timeout 600s; - # SSL configuration will be added by certbot + # Hide server version + server_tokens off; - # Security headers - add_header X-Frame-Options DENY; - add_header X-Content-Type-Options nosniff; - add_header X-XSS-Protection "1; mode=block"; - add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + # Deny access to hidden files + location ~ /\. { + deny all; + access_log off; + log_not_found off; + } # Main API (uploader-bot) location / { @@ -604,6 +655,8 @@ server { proxy_read_timeout 600s; proxy_send_timeout 600s; client_max_body_size 100M; + proxy_buffering off; + proxy_request_buffering off; } # Converter API (on-demand, через uploader-bot) @@ -624,6 +677,7 @@ server { # Health check (no rate limiting) location /health { proxy_pass http://127.0.0.1:15100/health; + proxy_set_header Host \$host; access_log off; } @@ -633,13 +687,6 @@ server { expires 30d; add_header Cache-Control "public, immutable"; } - - # Disable access to hidden files - location ~ /\. { - deny all; - access_log off; - log_not_found off; - } } EOF fi @@ -788,13 +835,20 @@ if ! check_and_skip "docker_build" "Сборка и запуск"; then log "Запуск приложения..." if [ "$NODE_TYPE" = "main" ]; then - # Для основной ноды запускаем все сервисы - sudo -u "$SERVICE_USER" docker-compose -f "$COMPOSE_FILE" up -d + # Для основной ноды запускаем все сервисы включая web2-client + if [ "$COMPOSE_FILE" = "docker-compose.production.yml" ]; then + sudo -u "$SERVICE_USER" docker-compose -f "$COMPOSE_FILE" up -d app postgres redis web2-client nginx prometheus grafana loki promtail + else + sudo -u "$SERVICE_USER" docker-compose -f "$COMPOSE_FILE" up -d + fi log "🚀 Запущены все сервисы для основной ноды" else - # Для обычной ноды запускаем только необходимые сервисы - sudo -u "$SERVICE_USER" docker-compose -f "$COMPOSE_FILE" up -d uploader-bot converter-builder watchtower 2>/dev/null || \ - sudo -u "$SERVICE_USER" docker-compose -f "$COMPOSE_FILE" up -d + # Для обычной ноды запускаем только основные сервисы без web2-client + if [ "$COMPOSE_FILE" = "docker-compose.production.yml" ]; then + sudo -u "$SERVICE_USER" docker-compose -f "$COMPOSE_FILE" up -d app postgres redis + else + sudo -u "$SERVICE_USER" docker-compose -f "$COMPOSE_FILE" up -d app postgres redis indexer ton_daemon license_index convert_process + fi log "🚀 Запущены основные сервисы для обычной ноды" fi @@ -858,9 +912,17 @@ if ! check_and_skip "final_check" "Финальная проверка"; then #!/bin/bash cd $MAIN_PROJECT_DIR if [ "$NODE_TYPE" = "main" ]; then - docker-compose -f $COMPOSE_FILE up -d + if [ "$COMPOSE_FILE" = "docker-compose.production.yml" ]; then + docker-compose -f $COMPOSE_FILE up -d app postgres redis web2-client nginx prometheus grafana loki promtail + else + docker-compose -f $COMPOSE_FILE up -d + fi else - docker-compose -f $COMPOSE_FILE up -d uploader-bot converter-builder watchtower 2>/dev/null || docker-compose -f $COMPOSE_FILE up -d + if [ "$COMPOSE_FILE" = "docker-compose.production.yml" ]; then + docker-compose -f $COMPOSE_FILE up -d app postgres redis + else + docker-compose -f $COMPOSE_FILE up -d app postgres redis indexer ton_daemon license_index convert_process + fi fi echo "✅ MY Uploader Bot запущен (тип ноды: $NODE_TYPE)" docker-compose -f $COMPOSE_FILE ps @@ -882,6 +944,12 @@ echo "" echo "🌐 API статус:" curl -s http://localhost:15100/health | jq . 2>/dev/null || echo "API недоступен" echo "" +echo "🗄️ PostgreSQL статус:" +docker-compose -f $COMPOSE_FILE exec postgres pg_isready -U my_user -d my_uploader_db 2>/dev/null || echo "PostgreSQL недоступен" +echo "" +echo "🔴 Redis статус:" +docker-compose -f $COMPOSE_FILE exec redis redis-cli ping 2>/dev/null || echo "Redis недоступен" +echo "" echo "🔒 SSL сертификат:" sudo certbot certificates | grep -A2 -B2 $DOMAIN || echo "Нет SSL сертификата" echo "" @@ -902,9 +970,17 @@ echo "🔄 Пересборка и перезапуск..." docker-compose -f $COMPOSE_FILE down docker-compose -f $COMPOSE_FILE build if [ "$NODE_TYPE" = "main" ]; then - docker-compose -f $COMPOSE_FILE up -d + if [ "$COMPOSE_FILE" = "docker-compose.production.yml" ]; then + docker-compose -f $COMPOSE_FILE up -d app postgres redis web2-client nginx prometheus grafana loki promtail + else + docker-compose -f $COMPOSE_FILE up -d + fi else - docker-compose -f $COMPOSE_FILE up -d uploader-bot converter-builder watchtower 2>/dev/null || docker-compose -f $COMPOSE_FILE up -d + if [ "$COMPOSE_FILE" = "docker-compose.production.yml" ]; then + docker-compose -f $COMPOSE_FILE up -d app postgres redis + else + docker-compose -f $COMPOSE_FILE up -d app postgres redis indexer ton_daemon license_index convert_process + fi fi echo "✅ Пересборка завершена" docker-compose -f $COMPOSE_FILE ps @@ -917,10 +993,18 @@ EOF log "Настройка автозапуска..." if [ "$NODE_TYPE" = "main" ]; then - SERVICE_EXEC_START="/usr/local/bin/docker-compose -f $COMPOSE_FILE up -d" + if [ "$COMPOSE_FILE" = "docker-compose.production.yml" ]; then + SERVICE_EXEC_START="/usr/local/bin/docker-compose -f $COMPOSE_FILE up -d app postgres redis web2-client nginx prometheus grafana loki promtail" + else + SERVICE_EXEC_START="/usr/local/bin/docker-compose -f $COMPOSE_FILE up -d" + fi SERVICE_DESCRIPTION="MY Uploader Bot (Main Node)" else - SERVICE_EXEC_START="/usr/local/bin/docker-compose -f $COMPOSE_FILE up -d uploader-bot converter-builder watchtower" + if [ "$COMPOSE_FILE" = "docker-compose.production.yml" ]; then + SERVICE_EXEC_START="/usr/local/bin/docker-compose -f $COMPOSE_FILE up -d app postgres redis" + else + SERVICE_EXEC_START="/usr/local/bin/docker-compose -f $COMPOSE_FILE up -d app postgres redis indexer ton_daemon license_index convert_process" + fi SERVICE_DESCRIPTION="MY Uploader Bot (Regular Node)" fi @@ -1010,7 +1094,15 @@ echo "sudo systemctl start my-uploader-bot" echo "sudo systemctl stop my-uploader-bot" echo "sudo systemctl status my-uploader-bot" echo "" -echo -e "${YELLOW}🔄 Повторная установка:${NC}" +echo -e "${YELLOW}🗄️ PostgreSQL управление:${NC}" +echo "# Подключение к БД:" +echo "sudo docker-compose -f $MAIN_PROJECT_DIR/$COMPOSE_FILE exec postgres psql -U my_user -d my_uploader_db" +echo "# Бэкап БД:" +echo "sudo docker-compose -f $MAIN_PROJECT_DIR/$COMPOSE_FILE exec postgres pg_dump -U my_user my_uploader_db > backup_\$(date +%Y%m%d).sql" +echo "# Проверка статуса БД:" +echo "sudo docker-compose -f $MAIN_PROJECT_DIR/$COMPOSE_FILE exec postgres pg_isready -U my_user -d my_uploader_db" +echo "" +echo -e "${YELLOW}� Повторная установка:${NC}" echo "# Для полной переустановки:" echo "sudo rm -f $PROGRESS_FILE" echo "# Для сброса определенного этапа отредактируйте:" @@ -1022,6 +1114,10 @@ echo "" echo -e "${YELLOW}📊 Мониторинг:${NC}" echo "docker stats" echo "sudo docker-compose -f $MAIN_PROJECT_DIR/$COMPOSE_FILE logs -f" +echo "# Логи PostgreSQL:" +echo "sudo docker-compose -f $MAIN_PROJECT_DIR/$COMPOSE_FILE logs postgres" +echo "# Логи Redis:" +echo "sudo docker-compose -f $MAIN_PROJECT_DIR/$COMPOSE_FILE logs redis" echo "" echo -e "${GREEN}✅ MY Uploader Bot готов к работе!${NC}" echo ""