← back to Pm2 Migration

migrate.sh

209 lines

#!/usr/bin/env bash
# pm2-migration/migrate.sh
# Move PM2 processes from Kamatera (outward-facing-only) to local Mac.
# Per-process loop:
#   1. SSH in, pull `pm2 describe` for the process (cwd, script, env, args)
#   2. rsync the project dir to ~/Projects/<name>/
#   3. Generate a local PM2 ecosystem entry
#   4. Stop on Kamatera (do NOT delete yet — kept as rollback for 24h)
#   5. Start locally, tail for 30s, abort & roll back if it crashes
#   6. After 24h soak, run --finalize to delete from Kamatera
#
# Usage:
#   ./migrate.sh --dry-run                      # show what would happen, no changes
#   ./migrate.sh <process-name>                 # migrate one
#   ./migrate.sh --batch move_local             # migrate the whole bucket
#   ./migrate.sh --finalize <process-name>      # after 24h soak, delete from Kamatera
#   ./migrate.sh --rollback <process-name>      # restart on Kamatera, stop local

set -euo pipefail

KAMATERA_USER="${KAMATERA_USER:-root}"
KAMATERA_HOST="${KAMATERA_HOST:-45.61.58.125}"
# Mirror Kamatera's /root/* layout under ~/kamatera-mirror/* so:
#   - shared node_modules and sibling dirs (../shared/X) resolve via Node's normal lookup
#   - hardcoded /root/* paths in agent code resolve via root-shim.js
MIRROR_ROOT="${MIRROR_ROOT:-/Users/macstudio3/kamatera-mirror}"
KIT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
STATE_DIR="$KIT_DIR/state"
mkdir -p "$STATE_DIR"

DRY_RUN=0
BATCH=""
FINALIZE=""
ROLLBACK=""
TARGETS=()

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run)   DRY_RUN=1; shift ;;
    --batch)     BATCH="$2"; shift 2 ;;
    --finalize)  FINALIZE="$2"; shift 2 ;;
    --rollback)  ROLLBACK="$2"; shift 2 ;;
    --help|-h)   sed -n '2,30p' "$0"; exit 0 ;;
    *)           TARGETS+=("$1"); shift ;;
  esac
done

red()    { printf '\033[31m%s\033[0m\n' "$*"; }
green()  { printf '\033[32m%s\033[0m\n' "$*"; }
yellow() { printf '\033[33m%s\033[0m\n' "$*"; }
blue()   { printf '\033[34m%s\033[0m\n' "$*"; }

run() {
  if [[ $DRY_RUN -eq 1 ]]; then
    yellow "[dry-run] $*"
  else
    eval "$@"
  fi
}

ssh_run() {
  if [[ $DRY_RUN -eq 1 ]]; then
    yellow "[dry-run] ssh $KAMATERA_USER@$KAMATERA_HOST '$*'"
  else
    ssh -o ConnectTimeout=10 "$KAMATERA_USER@$KAMATERA_HOST" "$*"
  fi
}

probe_kamatera() {
  blue "→ probing $KAMATERA_USER@$KAMATERA_HOST"
  ssh -o ConnectTimeout=5 -o BatchMode=yes "$KAMATERA_USER@$KAMATERA_HOST" 'pm2 -v' > /dev/null \
    || { red "SSH/PM2 probe failed. Check ~/.ssh/config and that Kamatera is reachable."; exit 2; }
  green "✓ Kamatera reachable"
}

describe() {
  local name="$1"
  # Keep it simple: just the four fields we need. SSH-escape via single quotes.
  ssh -o ConnectTimeout=10 "$KAMATERA_USER@$KAMATERA_HOST" \
    "pm2 jlist 2>/dev/null | jq -c --arg n '$name' '.[] | select(.name == \$n) | {name:.name, cwd:.pm2_env.pm_cwd, script:.pm2_env.pm_exec_path, interpreter:.pm2_env.exec_interpreter, args:(.pm2_env.args // [])}'"
}

migrate_one() {
  local name="$1"
  local state_file="$STATE_DIR/$name.json"

  blue "════════ $name ════════"

  # Step 1: capture metadata
  blue "→ pulling metadata"
  local meta
  meta="$(describe "$name")"
  if [[ -z "$meta" || "$meta" == "null" ]]; then
    red "✗ $name not found on Kamatera (maybe already moved?)"
    return 1
  fi
  echo "$meta" > "$state_file"
  green "✓ metadata captured → $state_file"

  local cwd script script_dir source_dir
  cwd="$(echo "$meta" | jq -r '.cwd')"
  script="$(echo "$meta" | jq -r '.script')"
  script_dir="$(dirname "$script")"
  # PM2 cwd is often wrong (process started from another dir). Trust the script's parent.
  if [[ "$cwd" != "$script_dir" ]]; then
    yellow "  cwd ($cwd) != script dir ($script_dir) — using script dir"
    source_dir="$script_dir"
  else
    source_dir="$cwd"
  fi
  echo "    pm2 cwd: $cwd"
  echo "    script:  $script"
  echo "    syncing: $source_dir"

  # Step 2: rsync — target mirrors Kamatera path under ~/kamatera-mirror
  # /root/DW-Agents/foo  → ~/kamatera-mirror/DW-Agents/foo
  local local_dir="${MIRROR_ROOT}${source_dir#/root}"
  blue "→ rsyncing $source_dir → $local_dir"
  if [[ -d "$local_dir" ]]; then
    yellow "  $local_dir exists — syncing in place (use --force-overwrite to wipe)"
  fi
  run "mkdir -p '$local_dir'"
  run "rsync -avz --exclude=node_modules --exclude=.git --exclude=logs --exclude='*.log' \
       '$KAMATERA_USER@$KAMATERA_HOST:$source_dir/' '$local_dir/'"

  # Step 3a: rewrite hardcoded :5432/:6379 → tunnel ports (15432/16379)
  blue "→ patching DB connection strings"
  run "bash '$KIT_DIR/patch-connections.sh' '$local_dir'"

  # Step 3b: install deps locally (if package.json present)
  if [[ $DRY_RUN -eq 0 && -f "$local_dir/package.json" ]]; then
    blue "→ npm install in $local_dir"
    (cd "$local_dir" && npm install --silent)
  fi

  # Step 4: append to local ecosystem
  blue "→ appending to ecosystem.config.cjs"
  run "node '$KIT_DIR/append-ecosystem.js' '$name' '$local_dir' '$script'"

  # Step 5: stop on Kamatera (KEEP, don't delete — rollback hatch)
  blue "→ stopping on Kamatera (kept for 24h rollback)"
  ssh_run "pm2 stop '$name' && pm2 save"

  # Step 6: start locally
  blue "→ starting locally via PM2"
  run "pm2 start '$KIT_DIR/ecosystem.config.cjs' --only '$name'"
  run "pm2 save"

  # Step 7: 30s smoke test
  blue "→ 30s smoke test (watching for crash loop)"
  if [[ $DRY_RUN -eq 0 ]]; then
    sleep 30
    local restarts
    restarts="$(pm2 jlist 2>/dev/null | jq -r --arg n "$name" '.[] | select(.name==$n) | .pm2_env.restart_time // 0')"
    if [[ "$restarts" -gt 3 ]]; then
      red "✗ $name crash-looping locally ($restarts restarts in 30s) — rolling back"
      pm2 stop "$name" || true
      ssh_run "pm2 start '$name' && pm2 save"
      return 1
    fi
  fi
  green "✓ $name running locally — 24h soak, then ./migrate.sh --finalize $name"
}

finalize_one() {
  local name="$1"
  blue "→ finalizing $name (delete from Kamatera)"
  ssh_run "pm2 delete '$name' && pm2 save"
  green "✓ $name removed from Kamatera"
}

rollback_one() {
  local name="$1"
  blue "→ rolling back $name"
  run "pm2 stop '$name' && pm2 save"
  ssh_run "pm2 start '$name' && pm2 save"
  green "✓ $name restored on Kamatera, stopped locally"
}

# ─── main ────────────────────────────────────────────────────────────────

probe_kamatera

if [[ -n "$ROLLBACK" ]];  then rollback_one "$ROLLBACK"; exit 0; fi
if [[ -n "$FINALIZE" ]];  then finalize_one "$FINALIZE"; exit 0; fi

if [[ -n "$BATCH" ]]; then
  mapfile -t TARGETS < <(jq -r --arg b "$BATCH" '.buckets[$b].processes[] | if type=="string" then . else .name end' "$KIT_DIR/processes.json")
  yellow "Batch '$BATCH' → ${#TARGETS[@]} processes"
  printf '  - %s\n' "${TARGETS[@]}"
  echo
  read -p "Proceed? (y/N) " -n 1 -r; echo
  [[ "$REPLY" =~ ^[Yy]$ ]] || exit 0
fi

if [[ ${#TARGETS[@]} -eq 0 ]]; then
  red "No targets. Pass a process name, --batch <bucket>, or --help."
  exit 1
fi

ok=0; fail=0
for name in "${TARGETS[@]}"; do
  if migrate_one "$name"; then ok=$((ok+1)); else fail=$((fail+1)); fi
done

echo
green "Done — $ok migrated, $fail failed"
echo "After 24h: ./migrate.sh --finalize <name>  for each successful one"