configs/start.sh

623 lines
18 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
echo "MY Network Node setup (interactive)"
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
BASE_DIR=$(cd "$SCRIPT_DIR/.." && pwd)
GIT_BASE_URL_DEFAULT="https://git.projscale.dev/my-dev"
GIT_BASE_URL="${GIT_BASE_URL:-$GIT_BASE_URL_DEFAULT}"
ensure_git() {
if ! command -v git >/dev/null 2>&1; then
echo "git is required to clone dependent repositories (uploader-bot, web2-client, converter-module)." >&2
echo "Please install git or clone these repositories manually under $BASE_DIR." >&2
exit 1
fi
}
ensure_repo_required() {
local name="$1"
local dir="$2"
local url="$3"
if [[ -d "$dir" ]]; then
echo "Found $name at $dir"
return
fi
echo "Cloning $name from $url into $dir ..."
git clone "$url" "$dir"
}
ensure_repo_optional() {
local name="$1"
local dir="$2"
local url="$3"
if [[ -d "$dir" ]]; then
echo "Found optional $name at $dir"
return
fi
echo "Cloning optional $name from $url into $dir ..."
if ! git clone "$url" "$dir"; then
echo "Warning: failed to clone optional repository $name from $url; continuing without it." >&2
fi
}
ensure_git
ensure_repo_required "uploader-bot" "$BASE_DIR/uploader-bot" "${GIT_BASE_URL}/uploader-bot.git"
ensure_repo_required "web2-client" "$BASE_DIR/web2-client" "${GIT_BASE_URL}/web2-client.git"
ensure_repo_optional "converter-module" "$BASE_DIR/converter-module" "${GIT_BASE_URL}/converter-module.git"
ENV_FILE="$SCRIPT_DIR/.env"
PROJECT_EXAMPLE="$BASE_DIR/.env.example"
if [[ ! -f "$SCRIPT_DIR/.env.example" && -f "$PROJECT_EXAMPLE" ]]; then
cp "$PROJECT_EXAMPLE" "$SCRIPT_DIR/.env.example"
fi
EXAMPLE_FILE="$SCRIPT_DIR/.env.example"
if [[ ! -f "$ENV_FILE" ]]; then
if [[ -f "$EXAMPLE_FILE" ]]; then
cp "$EXAMPLE_FILE" "$ENV_FILE"
else
echo "No .env or .env.example found; starting with a fresh .env for interactive setup."
touch "$ENV_FILE"
fi
fi
# Safe reader for existing values (does not fail under set -e -o pipefail)
# Returns only the value part after the first "=", empty string if key missing or value is empty.
ini_val() {
local key="$1"
local val
val=$(awk -F'=' -v k="$key" '
$1 == k {
if (NF == 1) {
# Key present but no "=", treat as empty value
print ""
} else {
# Reconstruct everything after the first "="
$1 = ""
sub(/^=/, "", $0)
print $0
}
exit
}
' "$ENV_FILE" 2>/dev/null || true)
echo -n "$val"
}
read -rp "Node privacy (public/private) [public]: " NODE_PRIVACY
NODE_PRIVACY=${NODE_PRIVACY:-public}
if [[ "$NODE_PRIVACY" != "public" && "$NODE_PRIVACY" != "private" ]]; then
echo "Invalid privacy; defaulting to public"
NODE_PRIVACY=public
fi
# Strip leading KEY= or export KEY= that users may paste
sanitize_assignment() {
local key="$1" value="$2"
if [[ $value == export\ * ]]; then
value=${value#export }
fi
local prefix="${key}="
if [[ $value == "$prefix"* ]]; then
value=${value#$prefix}
fi
value="${value#\"}"
value="${value%\"}"
value="${value#\'}"
value="${value%\'}"
# Trim leading/trailing whitespace (for safety we assume env values
# do not intentionally start or end with spaces)
if command -v awk >/dev/null 2>&1; then
value=$(printf '%s\n' "$value" | awk '{$1=$1;print}')
fi
echo "$value"
}
is_valid_port() {
local p="$1"
[[ "$p" =~ ^[0-9]+$ ]] && (( p > 0 && p <= 65535 ))
}
is_positive_int() {
local n="$1"
[[ "$n" =~ ^[0-9]+$ ]] && (( n > 0 ))
}
# Helper to prompt only if missing in .env
ask_or_keep() {
local key="$1"; shift
local prompt="$1"; shift
local def=""
if [[ $# -gt 0 ]]; then
def="$1"
shift
fi
local invalid=""
if [[ $# -gt 0 ]]; then
invalid="$1"
shift
fi
local cur
cur=$(ini_val "$key")
if [[ -n "$cur" && ( -z "$invalid" || "$cur" != "$invalid" ) ]]; then
echo "$key is set in .env; keeping existing value"
eval "$key=\"$cur\""
return
fi
local hint="$def"
if [[ -n "$hint" ]]; then
read -rp "$prompt [${hint}]: " val || true
val=${val:-$hint}
else
read -rp "$prompt: " val || true
fi
val=$(sanitize_assignment "$key" "$val")
eval "$key=\"$val\""
}
# Helper specifically for port values: validates and reprompts until a correct port is provided.
ask_port_or_keep() {
local key="$1"; shift
local prompt="$1"; shift
local def="${1:-}"
local cur
cur=$(ini_val "$key")
cur=$(sanitize_assignment "$key" "$cur")
if is_valid_port "$cur"; then
echo "$key is set in .env; keeping existing value ($cur)"
eval "$key=\"$cur\""
return
elif [[ -n "$cur" ]]; then
echo "Existing value for $key in .env is invalid ('$cur'); you will be asked to enter a valid port."
fi
local hint
hint=$(sanitize_assignment "$key" "$def")
if ! is_valid_port "$hint"; then
hint=""
fi
while true; do
local val
if [[ -n "$hint" ]]; then
read -rp "$prompt (1-65535) [${hint}]: " val || true
val=${val:-$hint}
else
read -rp "$prompt (1-65535): " val || true
fi
val=$(sanitize_assignment "$key" "$val")
if is_valid_port "$val"; then
eval "$key=\"$val\""
break
fi
echo "Invalid port '$val'. Please enter a number between 1 and 65535."
done
}
# Helper for positive integer values (e.g. intervals)
ask_positive_int_or_keep() {
local key="$1"; shift
local prompt="$1"; shift
local def="${1:-}"
local cur
cur=$(ini_val "$key")
cur=$(sanitize_assignment "$key" "$cur")
if is_positive_int "$cur"; then
echo "$key is set in .env; keeping existing value ($cur)"
eval "$key=\"$cur\""
return
elif [[ -n "$cur" ]]; then
echo "Existing value for $key in .env is invalid ('$cur'); you will be asked to enter a valid integer."
fi
local hint
hint=$(sanitize_assignment "$key" "$def")
if ! is_positive_int "$hint"; then
hint="60"
fi
while true; do
local val
if [[ -n "$hint" ]]; then
read -rp "$prompt (integer) [${hint}]: " val || true
val=${val:-$hint}
else
read -rp "$prompt (integer): " val || true
fi
val=$(sanitize_assignment "$key" "$val")
if is_positive_int "$val"; then
eval "$key=\"$val\""
break
fi
echo "Invalid value '$val'. Please enter a positive integer."
done
}
if [[ "$NODE_PRIVACY" == "private" ]]; then
# For private nodes, PUBLIC_HOST optional; only ask if missing
ask_or_keep PUBLIC_HOST "Public host URL (leave empty for private)" ""
else
ask_or_keep PUBLIC_HOST "Public host URL (e.g., https://node.example.com)" "$(ini_val PUBLIC_HOST)"
fi
ask_port_or_keep SANIC_PORT "Internal app port (SANIC_PORT)" "$(ini_val SANIC_PORT)"
ask_port_or_keep BACKEND_HTTP_PORT "Published backend port on host (BACKEND_HTTP_PORT)" "$(ini_val BACKEND_HTTP_PORT)"
ask_or_keep BOOTSTRAP_SEEDS "Bootstrap seeds (comma-separated URLs)" "" "https://my-bootstrap-1.example.com,https://my-bootstrap-2.example.com"
ask_positive_int_or_keep HANDSHAKE_INTERVAL_SEC "Handshake interval seconds" "$(ini_val HANDSHAKE_INTERVAL_SEC)"
ask_or_keep TELEGRAM_API_KEY "Telegram uploader bot token (TELEGRAM_API_KEY)" "" "YOUR_UPLOADER_BOT_TOKEN"
ask_or_keep CLIENT_TELEGRAM_API_KEY "Telegram client bot token (CLIENT_TELEGRAM_API_KEY)" "" "YOUR_CLIENT_BOT_TOKEN"
echo "Applying configuration to $ENV_FILE ..."
# In-place update helper
update_env() {
local key=$1 value=$2
if grep -qE "^${key}=" "$ENV_FILE"; then
sed -i.bak "s|^${key}=.*$|${key}=${value}|" "$ENV_FILE"
rm -f "$ENV_FILE.bak"
else
echo "${key}=${value}" >> "$ENV_FILE"
fi
}
ensure_env_default() {
local key=$1 default=$2
local raw cur
raw=$(ini_val "$key")
cur=$(sanitize_assignment "$key" "$raw")
if [[ -z "$cur" ]]; then
update_env "$key" "$default"
elif [[ "$cur" != "$raw" ]]; then
# Normalise the existing value if it only differed by quotes/whitespace
update_env "$key" "$cur"
fi
}
update_env NODE_PRIVACY "$NODE_PRIVACY"
update_env PUBLIC_HOST "${PUBLIC_HOST:-}"
if [[ -n "${PUBLIC_HOST:-}" ]]; then
update_env PROJECT_HOST "$PUBLIC_HOST"
fi
update_env SANIC_PORT "$SANIC_PORT"
update_env BACKEND_HTTP_PORT "$BACKEND_HTTP_PORT"
update_env BOOTSTRAP_SEEDS "$BOOTSTRAP_SEEDS"
existing_bootstrap_required=$(ini_val BOOTSTRAP_REQUIRED)
if [[ -z "$BOOTSTRAP_SEEDS" ]]; then
echo "No bootstrap seeds provided; disabling mandatory bootstrap."
update_env BOOTSTRAP_REQUIRED 0
else
existing_bootstrap_required=${existing_bootstrap_required:-1}
update_env BOOTSTRAP_REQUIRED "$existing_bootstrap_required"
fi
update_env HANDSHAKE_INTERVAL_SEC "$HANDSHAKE_INTERVAL_SEC"
if [[ -z "$TELEGRAM_API_KEY" || "$TELEGRAM_API_KEY" == "YOUR_UPLOADER_BOT_TOKEN" ]]; then
echo "TELEGRAM_API_KEY must be provided (create a bot via @BotFather)." >&2
exit 1
fi
if [[ -z "$CLIENT_TELEGRAM_API_KEY" || "$CLIENT_TELEGRAM_API_KEY" == "YOUR_CLIENT_BOT_TOKEN" ]]; then
echo "CLIENT_TELEGRAM_API_KEY must be provided (client bot token)." >&2
exit 1
fi
update_env TELEGRAM_API_KEY "$TELEGRAM_API_KEY"
update_env CLIENT_TELEGRAM_API_KEY "$CLIENT_TELEGRAM_API_KEY"
ensure_env_default POSTGRES_DB "mysdb"
ensure_env_default POSTGRES_USER "service"
ensure_env_default POSTGRES_PASSWORD "changeme"
ensure_env_default POSTGRES_HOST "db"
ensure_env_default POSTGRES_PORT "5432"
ensure_env_default POSTGRES_FORWARD_PORT "13580"
pg_user=$(sanitize_assignment POSTGRES_USER "$(ini_val POSTGRES_USER)")
pg_pass=$(sanitize_assignment POSTGRES_PASSWORD "$(ini_val POSTGRES_PASSWORD)")
pg_host=$(sanitize_assignment POSTGRES_HOST "$(ini_val POSTGRES_HOST)")
pg_port=$(sanitize_assignment POSTGRES_PORT "$(ini_val POSTGRES_PORT)")
pg_db=$(sanitize_assignment POSTGRES_DB "$(ini_val POSTGRES_DB)")
update_env DATABASE_URL "postgresql+psycopg2://${pg_user}:${pg_pass}@${pg_host}:${pg_port}/${pg_db}"
ensure_env_default FRONTEND_HTTP_PORT "13300"
ensure_env_default VITE_SENTRY_DSN ""
ensure_env_default TUSD_HTTP_PORT "13400"
tusd_port=$(ini_val TUSD_HTTP_PORT)
current_api_base=$(ini_val VITE_API_BASE_URL)
if [[ -z "$current_api_base" || "$current_api_base" == "https://my-public-node-103.projscale.dev/api/v1" ]]; then
if [[ -n "${PUBLIC_HOST:-}" ]]; then
update_env VITE_API_BASE_URL "${PUBLIC_HOST%/}/api/v1"
else
update_env VITE_API_BASE_URL "http://127.0.0.1:${BACKEND_HTTP_PORT}/api/v1"
fi
fi
current_api_storage=$(ini_val VITE_API_BASE_STORAGE_URL)
if [[ -z "$current_api_storage" || "$current_api_storage" == "https://my-public-node-103.projscale.dev/api/v1.5/storage" ]]; then
if [[ -n "${PUBLIC_HOST:-}" ]]; then
update_env VITE_API_BASE_STORAGE_URL "${PUBLIC_HOST%/}/api/v1.5/storage"
else
update_env VITE_API_BASE_STORAGE_URL "http://127.0.0.1:${BACKEND_HTTP_PORT}/api/v1.5/storage"
fi
fi
current_tus_endpoint=$(ini_val VITE_TUS_ENDPOINT)
if [[ -n "${PUBLIC_HOST:-}" ]]; then
update_env VITE_TUS_ENDPOINT "${PUBLIC_HOST%/}/tus/files"
else
if [[ -z "$current_tus_endpoint" || "$current_tus_endpoint" == "http://localhost:1080/files" ]]; then
update_env VITE_TUS_ENDPOINT "http://127.0.0.1:${tusd_port:-13400}/files"
fi
fi
generate_admin_token() {
if command -v openssl >/dev/null 2>&1; then
openssl rand -hex 32 | tr -d '\n'
elif command -v python3 >/dev/null 2>&1; then
python3 - <<'PY'
import os, binascii
print(binascii.hexlify(os.urandom(32)).decode())
PY
elif command -v python >/dev/null 2>&1; then
python - <<'PY'
import os, binascii
print(binascii.hexlify(os.urandom(32)).decode())
PY
else
echo "Need openssl or python to generate ADMIN_API_TOKEN" >&2
exit 1
fi
}
ensure_admin_token() {
local raw cur
raw=$(ini_val ADMIN_API_TOKEN)
cur=$(sanitize_assignment ADMIN_API_TOKEN "$raw")
if [[ -n "$cur" ]]; then
if [[ "$cur" != "$raw" ]]; then
update_env ADMIN_API_TOKEN "$cur"
fi
echo "ADMIN_API_TOKEN is set in .env; keeping existing value"
return
fi
echo "Generating ADMIN_API_TOKEN for admin /admin access ..."
local token
token=$(generate_admin_token)
update_env ADMIN_API_TOKEN "$token"
}
ensure_admin_token
generate_kek() {
if command -v openssl >/dev/null 2>&1; then
openssl rand -base64 32 | tr -d '\n'
elif command -v python3 >/dev/null 2>&1; then
python3 - <<'PY'
import os, base64
print(base64.b64encode(os.urandom(32)).decode())
PY
elif command -v python >/dev/null 2>&1; then
python - <<'PY'
import os, base64
print(base64.b64encode(os.urandom(32)).decode())
PY
else
echo "Need openssl or python to generate CONTENT_KEY_KEK_B64" >&2
exit 1
fi
}
ensure_content_key_kek() {
local current
current=$(ini_val CONTENT_KEY_KEK_B64)
local valid=0
if [[ -n "$current" ]]; then
if command -v python3 >/dev/null 2>&1; then
if python3 - "$current" <<'PY' >/dev/null 2>&1
import base64, sys
try:
raw = base64.b64decode(sys.argv[1], validate=False)
except Exception:
raise SystemExit(1)
if len(raw) == 32:
raise SystemExit(0)
raise SystemExit(1)
PY
then
valid=1
fi
elif command -v python >/dev/null 2>&1; then
if python - "$current" <<'PY' >/dev/null 2>&1
import base64, sys
try:
raw = base64.b64decode(sys.argv[1], validate=False)
except Exception:
raise SystemExit(1)
if len(raw) == 32:
raise SystemExit(0)
raise SystemExit(1)
PY
then
valid=1
fi
fi
if [[ $valid -eq 1 ]]; then
echo "Using existing CONTENT_KEY_KEK_B64 from .env"
update_env CONTENT_KEY_KEK_B64 "$current"
return
else
echo "Existing CONTENT_KEY_KEK_B64 is invalid; generating a new key"
fi
else
echo "Generating CONTENT_KEY_KEK_B64 ..."
fi
local new_kek
new_kek=$(generate_kek)
update_env CONTENT_KEY_KEK_B64 "$new_kek"
}
ensure_content_key_kek
# Ensure IPFS swarm key exists for private swarm by default
SWARM_KEY_FILE_DEFAULT="$SCRIPT_DIR/ipfs/swarm.key"
if [[ ! -f "$SWARM_KEY_FILE_DEFAULT" ]]; then
echo "Generating IPFS private swarm key at $SWARM_KEY_FILE_DEFAULT ..."
mkdir -p "$(dirname "$SWARM_KEY_FILE_DEFAULT")"
if command -v openssl >/dev/null 2>&1; then
KEYHEX=$(openssl rand -hex 32)
else
KEYHEX=$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')
fi
cat > "$SWARM_KEY_FILE_DEFAULT" <<EOF
/key/swarm/psk/1.0.0/
/base16/
$KEYHEX
EOF
fi
update_env IPFS_SWARM_KEY_FILE "$SWARM_KEY_FILE_DEFAULT"
# Ensure data directories exist
mkdir -p "$BASE_DIR/postgres-data" "$BASE_DIR/data/ipfs" "$BASE_DIR/data/tusd" "$BASE_DIR/app-logs" "$BASE_DIR/dynamicStorage"
# tusd writes into /data inside container which is mounted from $BASE_DIR/data/tusd on host.
# On fresh provision this directory is often root-owned (0755) which causes tusd "permission denied".
chmod 0777 "$BASE_DIR/data/tusd" 2>/dev/null || true
update_env DB_DATA_DIR_HOST "$BASE_DIR/postgres-data"
update_env IPFS_DATA_DIR_HOST "$BASE_DIR/data/ipfs"
update_env TUSD_DATA_DIR_HOST "$BASE_DIR/data/tusd"
# Normalise host mount paths so docker run inside containers resolves them correctly
ensure_absolute_host_path() {
local key="$1" default="$2"
local current
current=$(ini_val "$key")
if [[ -z "$current" || "$current" != /* ]]; then
update_env "$key" "$default"
fi
}
ensure_absolute_host_path BACKEND_DATA_DIR_HOST "$BASE_DIR/dynamicStorage"
ensure_absolute_host_path BACKEND_LOGS_DIR_HOST "$BASE_DIR/app-logs"
set_env_if_missing() {
local key="$1" value="$2"
local current
current=$(ini_val "$key")
if [[ -z "$current" ]]; then
update_env "$key" "$value"
fi
}
TOTAL_CPUS=$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || printf '2')
if ! [[ "$TOTAL_CPUS" =~ ^[0-9]+$ ]] || [ "$TOTAL_CPUS" -le 0 ]; then
TOTAL_CPUS=2
fi
HALF_CPUS_FLOAT=$(python3 - <<'PY'
import os
cpus = int(os.environ.get('TOTAL_CPUS', '2'))
if cpus <= 1:
print('0.5')
else:
value = cpus / 2
if abs(value - int(value)) < 1e-9:
print(int(value))
else:
formatted = f"{value:.2f}"
print(formatted.rstrip('0').rstrip('.'))
PY
)
HALF_CPUS_INT=$(( TOTAL_CPUS / 2 ))
if [ "$HALF_CPUS_INT" -le 0 ]; then
HALF_CPUS_INT=1
fi
if [ "$HALF_CPUS_INT" -ge "$TOTAL_CPUS" ] && [ "$TOTAL_CPUS" -gt 1 ]; then
HALF_CPUS_INT=$((TOTAL_CPUS - 1))
fi
if [ "$HALF_CPUS_INT" -le 0 ]; then
HALF_CPUS_INT=1
fi
if [ "$HALF_CPUS_INT" -eq 1 ]; then
CONVERT_CPUSET="0"
else
last=$((HALF_CPUS_INT - 1))
CONVERT_CPUSET="0-${last}"
fi
TOTAL_MEM_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null || printf '0')
if ! [[ "$TOTAL_MEM_KB" =~ ^[0-9]+$ ]] || [ "$TOTAL_MEM_KB" -le 0 ]; then
TOTAL_MEM_KB=$((4096 * 1024))
fi
HALF_MEM_MB=$(( TOTAL_MEM_KB / 2048 ))
if [ "$HALF_MEM_MB" -lt 512 ]; then
HALF_MEM_MB=512
fi
set_env_if_missing CONVERT_CPUSET "$CONVERT_CPUSET"
set_env_if_missing CONVERT_V3_CPUS "$HALF_CPUS_FLOAT"
set_env_if_missing CONVERT_V3_MEM "${HALF_MEM_MB}m"
set_env_if_missing CONVERT_PROCESS_CPUS "0.5"
set_env_if_missing CONVERT_PROCESS_MEM "512m"
set_env_if_missing MEDIA_CONVERTER_CPU_LIMIT "$HALF_CPUS_FLOAT"
set_env_if_missing MEDIA_CONVERTER_MEM_LIMIT "${HALF_MEM_MB}m"
set_env_if_missing MEDIA_CONVERTER_CPUSET "$CONVERT_CPUSET"
if ! grep -qE '^CONVERT_V3_MAX_CONCURRENCY=' "$ENV_FILE"; then
if [ "$HALF_CPUS_INT" -gt 2 ]; then
update_env CONVERT_V3_MAX_CONCURRENCY "2"
else
update_env CONVERT_V3_MAX_CONCURRENCY "$HALF_CPUS_INT"
fi
fi
if ! grep -qE '^IPFS_GATEWAY_BIND=' "$ENV_FILE"; then
update_env IPFS_GATEWAY_BIND "0.0.0.0"
fi
echo "Config written to $ENV_FILE. Starting containers..."
if ! command -v docker >/dev/null 2>&1; then
echo "Docker is required. Please install Docker and retry." >&2
exit 1
fi
if ! docker compose version >/dev/null 2>&1 && ! docker-compose --version >/dev/null 2>&1; then
echo "docker compose (v2) or docker-compose is required." >&2
exit 1
fi
MEDIA_CONVERTER_CONTEXT="$BASE_DIR/converter-module/converter"
if [[ -d "$MEDIA_CONVERTER_CONTEXT" ]]; then
echo "Building media_converter image from $MEDIA_CONVERTER_CONTEXT ..."
docker build -t media_converter:latest "$MEDIA_CONVERTER_CONTEXT"
else
echo "Warning: converter-module directory not found; skipping media_converter build" >&2
fi
set -x
COMPOSE_FILE_PATH="$SCRIPT_DIR/docker-compose.yml"
COMPOSE_PROJECT=$(python3 "$SCRIPT_DIR/compose_name.py" "$BASE_DIR")
if docker compose version >/dev/null 2>&1; then
COMPOSE_BIN=(docker compose)
elif docker-compose --version >/dev/null 2>&1; then
COMPOSE_BIN=(docker-compose)
else
echo "docker compose (v2) or docker-compose is required." >&2
exit 1
fi
COMPOSE_ARGS=(--env-file "$ENV_FILE" -p "$COMPOSE_PROJECT" -f "$COMPOSE_FILE_PATH")
"${COMPOSE_BIN[@]}" "${COMPOSE_ARGS[@]}" down --remove-orphans
"${COMPOSE_BIN[@]}" "${COMPOSE_ARGS[@]}" up -d --build --force-recreate
set +x
echo "Setup complete. Check health: curl -fsS http://127.0.0.1:${BACKEND_HTTP_PORT}/api/system.version"