diff --git a/docker-compose.yml b/docker-compose.yml index 2589c79..fd1d7d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -188,6 +188,9 @@ services: - ${TUSD_DATA_DIR_HOST:-./data/tusd}:/data - /var/run/docker.sock:/var/run/docker.sock # required by conversion pipeline restart: unless-stopped + cpus: ${CONVERT_PROCESS_CPUS:-0.5} + mem_limit: ${CONVERT_PROCESS_MEM:-512m} + cpuset: ${CONVERT_CPUSET:-0} networks: [mynet] backend-convert-v3: @@ -212,6 +215,9 @@ services: - ${TUSD_DATA_DIR_HOST:-./data/tusd}:/data - /var/run/docker.sock:/var/run/docker.sock restart: unless-stopped + cpus: ${CONVERT_V3_CPUS:-1} + mem_limit: ${CONVERT_V3_MEM:-2048m} + cpuset: ${CONVERT_CPUSET:-0} networks: [mynet] backend-index-scout-v3: diff --git a/nginx.conf b/nginx.conf index f956b55..6d9c8b8 100644 --- a/nginx.conf +++ b/nginx.conf @@ -202,7 +202,7 @@ server { add_header Access-Control-Allow-Origin $cors_origin always; add_header Access-Control-Allow-Credentials "true" always; add_header Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS, HEAD" always; - add_header Access-Control-Allow-Headers "Origin, Cache-Control, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-admin-token" always; + add_header Access-Control-Allow-Headers "Origin, Cache-Control, Content-Type, Accept, Authorization, Referer, User-Agent, Sec-Fetch-Dest, Sec-Fetch-Mode, Sec-Fetch-Site, x-admin-token, Tus-Resumable, tus-resumable, Upload-Length, upload-length, Upload-Offset, upload-offset, Upload-Metadata, upload-metadata, Upload-Defer-Length, upload-defer-length, Upload-Concat, upload-concat, x-file-name, x-last-chunk, x-chunk-start, x-upload-id, x-request-id" always; add_header Vary "Origin" always; # Вебсокеты (на будущее) @@ -221,4 +221,4 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Request-Id $req_id; } -} \ No newline at end of file +} diff --git a/start.sh b/start.sh index e8b531d..b3f50bd 100755 --- a/start.sh +++ b/start.sh @@ -291,6 +291,89 @@ mkdir -p "$BASE_DIR/postgres-data" "$BASE_DIR/data/ipfs" "$BASE_DIR/data/tusd" " update_env DB_DATA_DIR_HOST "$BASE_DIR/postgres-data" update_env IPFS_DATA_DIR_HOST "$BASE_DIR/data/ipfs" update_env TUSD_DATA_DIR_HOST "$BASE_DIR/data/tusd" + +# Normalise host mount paths so docker run inside containers resolves them correctly +ensure_absolute_host_path() { + local key="$1" default="$2" + local current + current=$(ini_val "$key") + if [[ -z "$current" || "$current" != /* ]]; then + update_env "$key" "$default" + fi +} + +ensure_absolute_host_path BACKEND_DATA_DIR_HOST "$BASE_DIR/dynamicStorage" +ensure_absolute_host_path BACKEND_LOGS_DIR_HOST "$BASE_DIR/app-logs" + +set_env_if_missing() { + local key="$1" value="$2" + local current + current=$(ini_val "$key") + if [[ -z "$current" ]]; then + update_env "$key" "$value" + fi +} + +TOTAL_CPUS=$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || printf '2') +if ! [[ "$TOTAL_CPUS" =~ ^[0-9]+$ ]] || [ "$TOTAL_CPUS" -le 0 ]; then + TOTAL_CPUS=2 +fi +HALF_CPUS_FLOAT=$(python3 - <<'PY' +import os +cpus = int(os.environ.get('TOTAL_CPUS', '2')) +if cpus <= 1: + print('0.5') +else: + value = cpus / 2 + if abs(value - int(value)) < 1e-9: + print(int(value)) + else: + formatted = f"{value:.2f}" + print(formatted.rstrip('0').rstrip('.')) +PY +) +HALF_CPUS_INT=$(( TOTAL_CPUS / 2 )) +if [ "$HALF_CPUS_INT" -le 0 ]; then + HALF_CPUS_INT=1 +fi +if [ "$HALF_CPUS_INT" -ge "$TOTAL_CPUS" ] && [ "$TOTAL_CPUS" -gt 1 ]; then + HALF_CPUS_INT=$((TOTAL_CPUS - 1)) +fi +if [ "$HALF_CPUS_INT" -le 0 ]; then + HALF_CPUS_INT=1 +fi +if [ "$HALF_CPUS_INT" -eq 1 ]; then + CONVERT_CPUSET="0" +else + last=$((HALF_CPUS_INT - 1)) + CONVERT_CPUSET="0-${last}" +fi + +TOTAL_MEM_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null || printf '0') +if ! [[ "$TOTAL_MEM_KB" =~ ^[0-9]+$ ]] || [ "$TOTAL_MEM_KB" -le 0 ]; then + TOTAL_MEM_KB=$((4096 * 1024)) +fi +HALF_MEM_MB=$(( TOTAL_MEM_KB / 2048 )) +if [ "$HALF_MEM_MB" -lt 512 ]; then + HALF_MEM_MB=512 +fi + +set_env_if_missing CONVERT_CPUSET "$CONVERT_CPUSET" +set_env_if_missing CONVERT_V3_CPUS "$HALF_CPUS_FLOAT" +set_env_if_missing CONVERT_V3_MEM "${HALF_MEM_MB}m" +set_env_if_missing CONVERT_PROCESS_CPUS "0.5" +set_env_if_missing CONVERT_PROCESS_MEM "512m" +set_env_if_missing MEDIA_CONVERTER_CPU_LIMIT "$HALF_CPUS_FLOAT" +set_env_if_missing MEDIA_CONVERTER_MEM_LIMIT "${HALF_MEM_MB}m" +set_env_if_missing MEDIA_CONVERTER_CPUSET "$CONVERT_CPUSET" +if ! grep -qE '^CONVERT_V3_MAX_CONCURRENCY=' "$ENV_FILE"; then + if [ "$HALF_CPUS_INT" -gt 2 ]; then + update_env CONVERT_V3_MAX_CONCURRENCY "2" + else + update_env CONVERT_V3_MAX_CONCURRENCY "$HALF_CPUS_INT" + fi +fi + if ! grep -qE '^IPFS_GATEWAY_BIND=' "$ENV_FILE"; then update_env IPFS_GATEWAY_BIND "0.0.0.0" fi @@ -306,6 +389,14 @@ if ! docker compose version >/dev/null 2>&1 && ! docker-compose --version >/dev/ exit 1 fi +MEDIA_CONVERTER_CONTEXT="$BASE_DIR/converter-module/converter" +if [[ -d "$MEDIA_CONVERTER_CONTEXT" ]]; then + echo "Building media_converter image from $MEDIA_CONVERTER_CONTEXT ..." + docker build -t media_converter:latest "$MEDIA_CONVERTER_CONTEXT" +else + echo "Warning: converter-module directory not found; skipping media_converter build" >&2 +fi + set -x COMPOSE_FILE_PATH="$SCRIPT_DIR/docker-compose.yml" COMPOSE_PROJECT=$(python3 "$SCRIPT_DIR/compose_name.py" "$BASE_DIR")