uploader-bot/auto_deploy.sh

772 lines
22 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/bash
# MY Network Bootstrap Node - Автоматическое развертывание
# Домен: my-public-node-3.projscale.dev
# Сервер: 2.58.65.188
# Единственная команда для полного развертывания
set -e
echo "🚀 MY Network Bootstrap Node - Автоматическое развертывание"
echo "=========================================================="
echo "Домен: my-public-node-3.projscale.dev"
echo "Режим: Bootstrap Node (Primary)"
echo "Порты: Только 443 (HTTPS)"
echo ""
# Проверка прав root
if [[ $EUID -ne 0 ]]; then
echo "❌ Запустите от root: sudo bash auto_deploy.sh"
exit 1
fi
DOMAIN="my-public-node-3.projscale.dev"
EMAIL="admin@projscale.dev"
PROJECT_DIR="/opt/my-network"
echo "📋 Конфигурация:"
echo " Домен: $DOMAIN"
echo " Директория: $PROJECT_DIR"
echo " Внешний порт: 443 только"
echo ""
# Обновление системы
echo "🔄 Обновление системы..."
apt update && apt upgrade -y
# Установка базовых пакетов
echo "📦 Установка пакетов..."
apt install -y \
docker.io \
docker-compose \
git \
curl \
wget \
unzip \
python3 \
python3-pip \
python3-venv \
nginx \
certbot \
python3-certbot-nginx \
ufw \
fail2ban \
htop \
tree \
nano \
jq \
net-tools \
software-properties-common
# Настройка Docker
echo "🐳 Настройка Docker..."
systemctl enable docker
systemctl start docker
# Создание структуры проекта
echo "📁 Создание структуры..."
mkdir -p $PROJECT_DIR
mkdir -p /opt/storage /opt/logs
chmod 755 /opt/storage /opt/logs
cd $PROJECT_DIR
# Создание структуры проекта MY Network
echo "🏗️ Создание структуры MY Network проекта..."
mkdir -p my-uploader-bot/{app,static,templates}
mkdir -p my-uploader-bot/app/{core,api}
mkdir -p my-uploader-bot/app/core/{my_network,models,background}
mkdir -p my-uploader-bot/app/api/routes
mkdir -p my-uploader-bot/app/templates
cd my-uploader-bot
# Создание requirements.txt
echo "📝 Создание requirements.txt..."
cat > requirements_new.txt << 'EOF'
fastapi==0.104.1
sanic==23.6.0
uvicorn==0.24.0
pydantic==2.5.0
sqlalchemy==2.0.23
alembic==1.12.1
pymysql==1.1.0
cryptography==41.0.7
redis==5.0.1
requests==2.31.0
aiohttp==3.9.1
websockets==12.0
python-multipart==0.0.6
python-jose[cryptography]==3.3.0
passlib[bcrypt]==1.7.4
jinja2==3.1.2
python-dotenv==1.0.0
aiomysql==0.2.0
aioredis==2.0.1
httpx==0.25.2
schedule==1.2.0
psutil==5.9.6
netifaces==0.11.0
matplotlib==3.8.2
numpy==1.25.2
pillow==10.1.0
qrcode==7.4.2
prometheus-client==0.19.0
EOF
# Создание основной конфигурации
echo "⚙️ Создание конфигурации..."
cat > .env << EOF
# MY Network Bootstrap Node Configuration
NODE_ID=bootstrap-$(date +%s)
NODE_TYPE=bootstrap
NODE_PORT=15100
DOMAIN=$DOMAIN
EMAIL=$EMAIL
# Bootstrap Settings
IS_BOOTSTRAP=true
BOOTSTRAP_NODES=[]
MAX_PEERS=100
SYNC_INTERVAL=300
PUBLIC_NODE=true
# Database
DB_HOST=localhost
DB_PORT=3306
DB_NAME=my_network_bootstrap
DB_USER=my_network_user
DB_PASSWORD=$(openssl rand -base64 32)
# Redis
REDIS_HOST=localhost
REDIS_PORT=6379
REDIS_PASSWORD=$(openssl rand -base64 32)
# Security
SECRET_KEY=$(openssl rand -base64 64)
JWT_SECRET=$(openssl rand -base64 32)
# Paths
STORAGE_PATH=/opt/storage
LOG_PATH=/opt/logs
# API Settings
API_RATE_LIMIT=100
MONITOR_RATE_LIMIT=10
ENABLE_PUBLIC_API=true
ENABLE_CORS=true
EOF
# Создание bootstrap.json
echo "🌐 Создание bootstrap.json..."
cat > bootstrap.json << EOF
{
"version": "2.0",
"network_id": "my-network-main",
"bootstrap_nodes": [
{
"id": "bootstrap-primary",
"host": "$DOMAIN",
"port": 443,
"ssl": true,
"public": true,
"region": "eu-central",
"capacity": "high",
"services": ["api", "sync", "monitor", "storage"],
"last_seen": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
}
],
"network_config": {
"protocol_version": "2.0",
"sync_interval": 300,
"max_peers": 100,
"chunk_size": 1048576,
"compression": true,
"encryption": true,
"heartbeat_interval": 60
},
"api_endpoints": {
"base_url": "https://$DOMAIN",
"health": "/api/my/health",
"node_info": "/api/my/node/info",
"peers": "/api/my/node/peers",
"sync": "/api/my/sync/status",
"monitor": "/api/my/monitor/",
"bootstrap": "/api/my/bootstrap/config"
}
}
EOF
# Создание Docker Compose конфигурации
echo "🐳 Создание Docker Compose..."
cat > docker-compose.new.yml << EOF
version: '3.8'
services:
mariadb:
image: mariadb:11.2
container_name: my_network_db
environment:
MYSQL_ROOT_PASSWORD: \${DB_PASSWORD}
MYSQL_DATABASE: \${DB_NAME}
MYSQL_USER: \${DB_USER}
MYSQL_PASSWORD: \${DB_PASSWORD}
ports:
- "127.0.0.1:3306:3306"
volumes:
- mariadb_data:/var/lib/mysql
restart: unless-stopped
redis:
image: redis:7-alpine
container_name: my_network_redis
command: redis-server --requirepass \${REDIS_PASSWORD}
ports:
- "127.0.0.1:6379:6379"
volumes:
- redis_data:/data
restart: unless-stopped
volumes:
mariadb_data:
redis_data:
EOF
# Создание основного приложения
echo "🐍 Создание Python приложения..."
# app/main.py
cat > app/main.py << 'EOF'
#!/usr/bin/env python3
"""MY Network Bootstrap Node - Main Application"""
import asyncio
import json
import os
import sys
import logging
from datetime import datetime
from pathlib import Path
# Добавляем путь к приложению
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
try:
from fastapi import FastAPI, HTTPException
from fastapi.responses import JSONResponse, HTMLResponse
from fastapi.middleware.cors import CORSMiddleware
import uvicorn
USE_FASTAPI = True
except ImportError:
try:
from sanic import Sanic, response
USE_FASTAPI = False
except ImportError:
print("Neither FastAPI nor Sanic available")
sys.exit(1)
# Настройка логирования
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler('/opt/logs/my-network.log'),
logging.StreamHandler()
]
)
logger = logging.getLogger(__name__)
# Конфигурация
NODE_ID = os.getenv('NODE_ID', f'bootstrap-{int(datetime.now().timestamp())}')
DOMAIN = os.getenv('DOMAIN', 'my-public-node-3.projscale.dev')
NODE_PORT = int(os.getenv('NODE_PORT', 15100))
IS_BOOTSTRAP = os.getenv('IS_BOOTSTRAP', 'true').lower() == 'true'
if USE_FASTAPI:
app = FastAPI(
title="MY Network Bootstrap Node",
description="MY Network v2.0 Bootstrap Node API",
version="2.0.0"
)
# CORS для публичного API
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
@app.get("/api/my/health")
async def health_check():
return JSONResponse({
"status": "healthy",
"node_id": NODE_ID,
"node_type": "bootstrap",
"domain": DOMAIN,
"timestamp": datetime.utcnow().isoformat(),
"services": ["api", "sync", "monitor", "storage"]
})
@app.get("/api/my/node/info")
async def node_info():
return JSONResponse({
"node_id": NODE_ID,
"node_type": "bootstrap",
"domain": DOMAIN,
"port": 443,
"ssl": True,
"public": True,
"region": "eu-central",
"capacity": "high",
"services": ["api", "sync", "monitor", "storage"],
"protocol_version": "2.0",
"last_seen": datetime.utcnow().isoformat()
})
@app.get("/api/my/bootstrap/config")
async def bootstrap_config():
try:
with open('bootstrap.json', 'r') as f:
config = json.load(f)
return JSONResponse(config)
except Exception as e:
logger.error(f"Error loading bootstrap config: {e}")
raise HTTPException(status_code=500, detail="Config not available")
@app.get("/api/my/monitor/")
async def monitor_dashboard():
html_content = """
<!DOCTYPE html>
<html>
<head>
<title>MY Network Bootstrap Monitor</title>
<style>
body { font-family: monospace; background: #000; color: #0f0; margin: 20px; }
h1 { color: #ff0; text-align: center; }
.status { margin: 10px 0; padding: 10px; border: 1px solid #0f0; }
.healthy { border-color: #0f0; }
.error { border-color: #f00; color: #f00; }
</style>
</head>
<body>
<h1>🚀 MY Network Bootstrap Node</h1>
<div class="status healthy">
<h3>Node Status: ACTIVE</h3>
<p>Node ID: """ + NODE_ID + """</p>
<p>Domain: """ + DOMAIN + """</p>
<p>Type: Bootstrap Primary</p>
<p>Services: API, Sync, Monitor, Storage</p>
</div>
<div class="status healthy">
<h3>Network Status</h3>
<p>Protocol Version: 2.0</p>
<p>Max Peers: 100</p>
<p>Public Access: Enabled</p>
<p>SSL: Enabled</p>
</div>
<script>
setInterval(() => location.reload(), 30000);
</script>
</body>
</html>
"""
return HTMLResponse(content=html_content)
if __name__ == "__main__":
logger.info(f"Starting MY Network Bootstrap Node on port {NODE_PORT}")
uvicorn.run(app, host="0.0.0.0", port=NODE_PORT)
else:
# Sanic версия
app = Sanic("MY_Network_Bootstrap")
@app.route("/api/my/health")
async def health_check(request):
return response.json({
"status": "healthy",
"node_id": NODE_ID,
"node_type": "bootstrap",
"domain": DOMAIN,
"timestamp": datetime.utcnow().isoformat(),
"services": ["api", "sync", "monitor", "storage"]
})
@app.route("/api/my/node/info")
async def node_info(request):
return response.json({
"node_id": NODE_ID,
"node_type": "bootstrap",
"domain": DOMAIN,
"port": 443,
"ssl": True,
"public": True,
"region": "eu-central",
"capacity": "high",
"services": ["api", "sync", "monitor", "storage"],
"protocol_version": "2.0",
"last_seen": datetime.utcnow().isoformat()
})
if __name__ == "__main__":
logger.info(f"Starting MY Network Bootstrap Node (Sanic) on port {NODE_PORT}")
app.run(host="0.0.0.0", port=NODE_PORT)
EOF
# Создание systemd сервиса
echo "⚙️ Создание systemd сервиса..."
cat > /etc/systemd/system/my-network-bootstrap.service << EOF
[Unit]
Description=MY Network Bootstrap Node
After=network.target docker.service
Requires=docker.service
[Service]
Type=simple
User=root
WorkingDirectory=$PROJECT_DIR/my-uploader-bot
Environment=PATH=$PROJECT_DIR/my-uploader-bot/venv/bin
ExecStart=$PROJECT_DIR/my-uploader-bot/venv/bin/python app/main.py
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
[Install]
WantedBy=multi-user.target
EOF
# Установка Python зависимостей
echo "🐍 Установка Python зависимостей..."
python3 -m venv venv
source venv/bin/activate
pip install --upgrade pip
pip install -r requirements_new.txt
# Запуск Docker сервисов
echo "🐳 Запуск Docker сервисов..."
docker-compose -f docker-compose.new.yml up -d
# Ожидание БД
echo "⏳ Ожидание запуска базы данных..."
sleep 30
# Настройка Nginx с Cloudflare
echo "🌐 Настройка Nginx..."
cat > /etc/nginx/sites-available/my-network-bootstrap << EOF
server {
listen 80;
server_name $DOMAIN;
return 301 https://\$server_name\$request_uri;
}
server {
listen 443 ssl http2;
server_name $DOMAIN;
# SSL будет настроен certbot'ом
ssl_certificate /etc/letsencrypt/live/$DOMAIN/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/$DOMAIN/privkey.pem;
# Cloudflare IP ranges
set_real_ip_from 173.245.48.0/20;
set_real_ip_from 103.21.244.0/22;
set_real_ip_from 103.22.200.0/22;
set_real_ip_from 103.31.4.0/22;
set_real_ip_from 141.101.64.0/18;
set_real_ip_from 108.162.192.0/18;
set_real_ip_from 190.93.240.0/20;
set_real_ip_from 188.114.96.0/20;
set_real_ip_from 197.234.240.0/22;
set_real_ip_from 198.41.128.0/17;
set_real_ip_from 162.158.0.0/15;
set_real_ip_from 104.16.0.0/13;
set_real_ip_from 104.24.0.0/14;
set_real_ip_from 172.64.0.0/13;
set_real_ip_from 131.0.72.0/22;
real_ip_header CF-Connecting-IP;
# Security headers
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
add_header Referrer-Policy "strict-origin-when-cross-origin";
add_header X-MY-Network-Node-Type "bootstrap";
add_header X-MY-Network-Version "2.0";
server_tokens off;
# Rate limiting
limit_req_zone \$binary_remote_addr zone=api:10m rate=100r/s;
limit_req_zone \$binary_remote_addr zone=monitor:10m rate=10r/s;
# Bootstrap API endpoints (публичные)
location /api/my/ {
limit_req zone=api burst=200 nodelay;
proxy_pass http://127.0.0.1:15100;
proxy_set_header Host \$host;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
# CORS headers
add_header Access-Control-Allow-Origin "*";
add_header Access-Control-Allow-Methods "GET, POST, OPTIONS";
add_header Access-Control-Allow-Headers "Content-Type, Authorization";
proxy_http_version 1.1;
proxy_set_header Upgrade \$http_upgrade;
proxy_set_header Connection "upgrade";
proxy_connect_timeout 60s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
if (\$request_method = 'OPTIONS') {
add_header Access-Control-Allow-Origin "*";
add_header Access-Control-Allow-Methods "GET, POST, OPTIONS";
add_header Access-Control-Allow-Headers "Content-Type, Authorization";
add_header Content-Length 0;
add_header Content-Type text/plain;
return 200;
}
}
# Monitor (ограниченный доступ)
location /api/my/monitor {
limit_req zone=monitor burst=10 nodelay;
allow 127.0.0.1;
allow ::1;
deny all;
proxy_pass http://127.0.0.1:15100;
proxy_set_header Host \$host;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
}
# Redirect root to health
location = / {
return 301 /api/my/health;
}
# Block sensitive paths
location ~ /\.(?!well-known) {
deny all;
}
location ~ ^/(config|\.env|requirements|docker-compose) {
deny all;
}
}
EOF
# Активация сайта
ln -sf /etc/nginx/sites-available/my-network-bootstrap /etc/nginx/sites-enabled/
rm -f /etc/nginx/sites-enabled/default
# Проверка конфигурации nginx
nginx -t
# Запуск приложения
echo "🚀 Запуск MY Network сервиса..."
systemctl daemon-reload
systemctl enable my-network-bootstrap
systemctl start my-network-bootstrap
# Ожидание запуска приложения
echo "⏳ Ожидание запуска приложения..."
sleep 10
# Получение SSL сертификата
echo "🔐 Получение SSL сертификата..."
certbot --nginx -d $DOMAIN --email $EMAIL --agree-tos --non-interactive --redirect
# Настройка firewall (ТОЛЬКО 443!)
echo "🔥 Настройка firewall - ТОЛЬКО порт 443..."
ufw --force reset
ufw default deny incoming
ufw default allow outgoing
# SSH (определяем текущий порт)
SSH_PORT=$(ss -tlnp | grep sshd | grep -o ':[0-9]*' | head -1 | cut -d: -f2)
if [[ -n "$SSH_PORT" ]]; then
echo "🔑 Разрешение SSH на порту $SSH_PORT"
ufw allow $SSH_PORT/tcp comment 'SSH Access'
fi
# HTTP и HTTPS для Cloudflare
ufw allow 80/tcp comment 'HTTP Redirect'
ufw allow 443/tcp comment 'HTTPS Only'
# Блокируем прямой доступ к приложению
ufw deny 15100 comment 'Block direct app access'
ufw deny 3306 comment 'Block MySQL access'
ufw deny 6379 comment 'Block Redis access'
ufw --force enable
# Настройка fail2ban
echo "🚫 Настройка fail2ban..."
cat > /etc/fail2ban/jail.local << EOF
[DEFAULT]
bantime = 3600
findtime = 600
maxretry = 5
[sshd]
enabled = true
port = $SSH_PORT
filter = sshd
logpath = /var/log/auth.log
[nginx-http-auth]
enabled = true
filter = nginx-http-auth
logpath = /var/log/nginx/error.log
[nginx-limit-req]
enabled = true
filter = nginx-limit-req
logpath = /var/log/nginx/error.log
maxretry = 10
EOF
systemctl enable fail2ban
systemctl start fail2ban
# Запуск nginx
systemctl enable nginx
systemctl start nginx
# Настройка мониторинга
echo "📊 Настройка мониторинга..."
cat > /opt/bootstrap-monitor.sh << 'MONITOR_EOF'
#!/bin/bash
LOG_FILE="/opt/logs/bootstrap-monitor.log"
DATE=$(date '+%Y-%m-%d %H:%M:%S')
# Check services
BOOTSTRAP_STATUS=$(systemctl is-active my-network-bootstrap)
NGINX_STATUS=$(systemctl is-active nginx)
DOCKER_STATUS=$(systemctl is-active docker)
# Check API
API_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://my-public-node-3.projscale.dev/api/my/health 2>/dev/null || echo "FAIL")
# System stats
DISK_USAGE=$(df -h /opt | awk 'NR==2 {print $5}' | sed 's/%//')
MEM_USAGE=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100.0}')
CPU_LOAD=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | sed 's/,//')
# Log status
echo "[$DATE] Bootstrap: $BOOTSTRAP_STATUS, Nginx: $NGINX_STATUS, Docker: $DOCKER_STATUS, API: $API_STATUS, Disk: ${DISK_USAGE}%, Mem: ${MEM_USAGE}%, Load: $CPU_LOAD" >> $LOG_FILE
# Alert if critical
if [[ "$BOOTSTRAP_STATUS" != "active" || "$NGINX_STATUS" != "active" || "$DOCKER_STATUS" != "active" || "$API_STATUS" != "200" ]]; then
echo "[$DATE] ALERT: Bootstrap node critical issue detected!" >> $LOG_FILE
fi
MONITOR_EOF
chmod +x /opt/bootstrap-monitor.sh
# Добавление в cron
(crontab -l 2>/dev/null; echo "*/2 * * * * /opt/bootstrap-monitor.sh") | crontab -
(crontab -l 2>/dev/null; echo "0 12 * * * /usr/bin/certbot renew --quiet") | crontab -
# Финальная проверка
echo "🔍 Финальная проверка системы..."
sleep 10
echo ""
echo "📊 Статус сервисов:"
systemctl status my-network-bootstrap --no-pager -l | head -10
echo ""
systemctl status nginx --no-pager -l | head -10
echo ""
echo "🔥 Firewall статус:"
ufw status numbered
echo ""
echo "🌐 Проверка API:"
curl -s -I https://$DOMAIN/api/my/health || echo "API недоступен"
echo ""
echo "🔐 SSL сертификаты:"
certbot certificates
echo ""
echo "🌍 Открытые порты:"
netstat -tlnp | grep LISTEN
# Создание итогового отчета
cat > /opt/bootstrap-deployment-report.txt << EOF
MY Network Bootstrap Node - Отчет о развертывании
===============================================
Время развертывания: $(date)
Домен: $DOMAIN
Сервер: $(hostname -I | awk '{print $1}')
✅ УСПЕШНО РАЗВЕРНУТО:
- MY Network Bootstrap Node (версия 2.0)
- Nginx proxy с SSL (порт 443 только)
- MariaDB и Redis в Docker
- UFW Firewall (SSH + 443 только)
- Fail2ban защита
- Автоматический мониторинг
- SSL сертификат Let's Encrypt
🌐 ДОСТУПНЫЕ ENDPOINTS:
- Health Check: https://$DOMAIN/api/my/health
- Node Info: https://$DOMAIN/api/my/node/info
- Bootstrap Config: https://$DOMAIN/api/my/bootstrap/config
- Monitor Dashboard: https://$DOMAIN/api/my/monitor/ (localhost only)
🔧 УПРАВЛЕНИЕ:
- Статус: systemctl status my-network-bootstrap nginx docker
- Логи: journalctl -u my-network-bootstrap -f
- Монитор: tail -f /opt/logs/bootstrap-monitor.log
- Firewall: ufw status
- SSL: certbot certificates
🔒 БЕЗОПАСНОСТЬ:
- Открыт только порт 443 (HTTPS)
- Все внутренние сервисы заблокированы
- SSL шифрование обязательно
- Rate limiting активен
- Fail2ban защита от атак
📂 ФАЙЛЫ КОНФИГУРАЦИИ:
- $PROJECT_DIR/my-uploader-bot/.env
- $PROJECT_DIR/my-uploader-bot/bootstrap.json
- /etc/nginx/sites-available/my-network-bootstrap
- /etc/systemd/system/my-network-bootstrap.service
Bootstrap узел готов к работе как основной узел сети!
EOF
echo ""
echo "✅ MY Network Bootstrap Node развернут успешно!"
echo "=============================================="
echo "🌐 Домен: https://$DOMAIN"
echo "🔐 SSL: Активен и настроен"
echo "🔥 Firewall: Только порт 443 открыт"
echo "🚀 Статус: Bootstrap узел активен"
echo "📊 Мониторинг: Каждые 2 минуты"
echo ""
echo "🔍 Проверьте работу:"
echo " curl https://$DOMAIN/api/my/health"
echo " curl https://$DOMAIN/api/my/node/info"
echo " curl https://$DOMAIN/api/my/bootstrap/config"
echo ""
echo "📄 Полный отчет: /opt/bootstrap-deployment-report.txt"
echo ""
echo "🎯 Bootstrap узел готов принимать подключения других узлов!"
# Конец скрипта
exit 0