← back to Wallco Ai

deploy-kamatera.sh

306 lines

#!/usr/bin/env bash
# wallco.ai — deploy to Kamatera
# Usage: ./deploy-kamatera.sh
# Pre-flight: CF zone must exist + NS swapped before running.
set -euo pipefail

REMOTE="root@45.61.58.125"
REMOTE_DIR="/root/public-projects/wallco-ai"
LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)"
SITE="wallco.ai"
PM2_NAME="wallco-ai"
PORT=9905
HEALTH_URL="https://${SITE}/health"

echo "=== wallco.ai deploy to Kamatera ==="
echo "Local:  $LOCAL_DIR"
echo "Remote: $REMOTE:$REMOTE_DIR"
echo ""

# 1. Port collision pre-flight — only fail if a NON-pm2-managed process holds the port
echo "[1/6] Port pre-flight..."
COLLISION=$(ssh "$REMOTE" "ss -tln | grep ':${PORT} '" 2>/dev/null || true)
if [ -n "$COLLISION" ]; then
  PM2_OWNED=$(ssh "$REMOTE" "pm2 jlist 2>/dev/null | python3 -c 'import sys,json;procs=json.load(sys.stdin);hits=[p[\"name\"] for p in procs if str(p.get(\"pm2_env\",{}).get(\"PORT\",\"\"))==\"${PORT}\" or p.get(\"name\")==\"${PM2_NAME}\"];print(\",\".join(hits))' 2>/dev/null" || true)
  if [ -n "$PM2_OWNED" ]; then
    echo "  Port $PORT held by pm2 process(es): $PM2_OWNED — will reload in step 4."
  else
    echo "ERROR: Port $PORT already in use on Kamatera (NOT by pm2):"
    echo "$COLLISION"
    exit 1
  fi
else
  echo "  Port $PORT is free."
fi

# 2. rsync (exclude heavy data dirs, .env, node_modules, git)
echo "[2/6] rsync..."
ssh "$REMOTE" "mkdir -p $REMOTE_DIR/logs $REMOTE_DIR/data/generated $REMOTE_DIR/public/uploads"
rsync -az --delete \
  --exclude='node_modules/' \
  --exclude='.git/' \
  --exclude='.env*' \
  --exclude='*.log' \
  --exclude='data/images/' \
  --exclude='data/generated/' \
  --exclude='data/generated-web/' \
  --exclude='data/generated_pre_seamless_backup/' \
  --exclude='data/generated_ghost_quarantine/' \
  --exclude='data/generated_cactus_quarantine/' \
  --exclude='data/elements' \
  --exclude='data/tif/' \
  --exclude='data/hires/' \
  --exclude='data/rooms/' \
  --exclude='data/spoonflower-pulled/' \
  --exclude='data/pdp-theme.json' \
  --exclude='data/custom-styleguides.json' \
  --exclude='data/cactus-murals/' \
  --exclude='data/cactus-decisions.jsonl' \
  --exclude='data/fix-decisions.jsonl' \
  --exclude='data/ghost-scan-flagged.jsonl' \
  --exclude='data/ghost-scan-results.jsonl' \
  --exclude='data/ghost-scan-triage.jsonl' \
  --exclude='data/ghost-scan-flagged.pre-*.jsonl' \
  --exclude='data/ghost-scan-flagged.post-*.jsonl' \
  --exclude='data/ghost-scan-results.pre-*.jsonl' \
  --exclude='data/ghost-purge.jsonl' \
  --exclude='data/ghost-labels.jsonl' \
  --exclude='data/bad-aesthetic-patterns.jsonl' \
  --exclude='data/bad-aesthetic-patterns.*.jsonl' \
  --exclude='data/fixes-feed.jsonl' \
  --exclude='data/marketplace/events.jsonl' \
  --exclude='data/yolo-findings/' \
  --exclude='data/edges-scan-*' \
  --exclude='data/crontab.backup-*' \
  --exclude='logs/' \
  --exclude='server-admin.js' \
  --exclude='scripts/' \
  --exclude='db/' \
  "$LOCAL_DIR/" "$REMOTE:$REMOTE_DIR/"

# 3. Rsync design images — the generated PNGs we DO want on prod.
#    APPEND-ONLY store: every PNG filename is unique-per-generation
#    (var_<id>_<hue>_<ts>.png / <ts>_<seed>.png), so files are only ever
#    added, never modified. Therefore:
#      --ignore-existing : transfers ONLY new files → fast, won't be skipped.
#      NO --delete       : local data/generated gets pruned for disk space
#                          (Mac2 runs ~96% full); --delete would then wipe
#                          those same images off prod. Never mirror this dir.
echo "[2b/6] rsync generated design images (incremental, append-only)..."
rsync -az --ignore-existing --stats \
  "$LOCAL_DIR/data/generated/" "$REMOTE:$REMOTE_DIR/data/generated/" \
  | grep -E 'Number of files(:| transferred)|Total transferred' || true

# 2c. Ship refresh_designs_snapshot.py specifically. scripts/ is excluded from
#     the main rsync (keeps prod lean), but this one script MUST track local —
#     it carries the generator allowlist + the CATASTROPHE GUARD that stops a
#     prod-side regen (recolor-bake spawns it) from clobbering designs.json with
#     prod's sparse PG (the 2026-05-26 "157 designs" incident). Cross-platform
#     by design (psql_json handles linux sudo-postgres vs mac).
echo "[2c/6] ship guarded snapshot script..."
rsync -az "$LOCAL_DIR/scripts/refresh_designs_snapshot.py" \
  "$REMOTE:$REMOTE_DIR/scripts/refresh_designs_snapshot.py" && echo "  guarded refresh_designs_snapshot.py shipped"

# 2d. Ship runtime-endpoint scripts. scripts/ is excluded from main rsync, but
#     a handful of .py files ARE called via subprocess.spawn from server.js
#     endpoints reachable on the live site — they have to be present on prod
#     for those features to work. List grows narrowly as endpoints add deps;
#     don't unbox scripts/ wholesale.
echo "[2d/6] ship runtime-endpoint scripts..."
for script in scripts/seam-defect-boxes.py scripts/mint-shopify-download-token.js scripts/generate_designs.js scripts/generator_tick.js scripts/seam-heal-feather.py scripts/quantize-no-ghost.py scripts/recolor-tif.py scripts/tif-to-web.py scripts/pilot-build-tifs.py scripts/cron-build-tifs.sh scripts/backfill-rooms.py scripts/pil-room-composite.py scripts/cron-backfill-rooms.sh scripts/settlement_postgen_vision_check.js scripts/settlement_tick_guard.js; do
  if [ -f "$LOCAL_DIR/$script" ]; then
    rsync -az "$LOCAL_DIR/$script" "$REMOTE:$REMOTE_DIR/$script" && echo "  $script shipped"
  fi
done

# 4. npm install on remote
echo "[3/6] npm install..."
ssh "$REMOTE" "cd $REMOTE_DIR && npm install --omit=dev 2>&1 | tail -5"

# 5. pm2 start or reload
echo "[4/6] pm2 startOrReload..."
ssh "$REMOTE" "cd $REMOTE_DIR && pm2 startOrReload ecosystem.config.js --only $PM2_NAME 2>&1 | tail -10 || pm2 start server.js --name $PM2_NAME --cwd $REMOTE_DIR -- --port $PORT"
ssh "$REMOTE" "pm2 save"

# 5b. Orphaned-entrypoint guard (added 2026-06-01 after the wallco-hot-or-not
# incident). `rsync --delete` mirrors the repo onto prod, so deleting a file
# from the repo deletes it from prod too. If a pm2 process still points at a
# script file under THIS deploy dir that no longer exists, it crash-loops
# forever (MODULE_NOT_FOUND, 1800+ restarts last time). This checks ONLY pm2
# processes whose script path is under $REMOTE_DIR — so it never false-positives
# on the box's other ~180 apps — and warns (non-fatal) if any references a
# now-missing file, naming the exact fix.
echo "[4b/6] orphaned-entrypoint guard..."
ssh "$REMOTE" "pm2 jlist 2>/dev/null | DEPLOY_DIR='$REMOTE_DIR' python3 -c \"
import sys, json, os
deploy_dir = os.environ['DEPLOY_DIR'].rstrip('/') + '/'
try: procs = json.load(sys.stdin)
except Exception: procs = []
bad = []
for p in procs:
    sp = (p.get('pm2_env', {}) or {}).get('pm_exec_path', '') or ''
    if sp.startswith(deploy_dir) and not os.path.exists(sp):
        bad.append((p.get('name', '?'), sp))
if bad:
    print('  WARNING: pm2 process(es) point at MISSING files under this deploy dir:')
    for n, sp in bad:
        print('    - ' + n + ' -> ' + sp + '  (will crash-loop)')
    print('  Fix: restore the file from git, OR retire the process: pm2 delete <name> && pm2 save')
else:
    print('  ok - no pm2 process references a deleted file in the deploy dir')
\""

# 6. Health check (wait up to 20s)
echo "[5/6] Health check → $HEALTH_URL"
for i in $(seq 1 10); do
  # `-w "%{http_code}"` always prints 3 digits (000 on conn failure). `|| true`
  # absorbs curl's non-zero exit without appending another "000" (which was
  # producing "HTTP 000000" before). `-L` follows the CF redirect if any.
  STATUS=$(curl -sS -L -o /dev/null -w "%{http_code}" --max-time 8 "$HEALTH_URL" 2>/dev/null || true)
  if [ "$STATUS" = "200" ]; then
    BODY=$(curl -sS -L --max-time 8 "$HEALTH_URL")
    echo "  OK ($STATUS): $BODY"
    break
  fi
  echo "  Attempt $i: HTTP $STATUS — waiting..."
  sleep 2
done
if [ "$STATUS" != "200" ]; then
  echo "ERROR: Health check failed after 20s. Check pm2 logs:"
  ssh "$REMOTE" "pm2 logs $PM2_NAME --nostream --lines 20"
  exit 1
fi

# 5c. Re-link room mockups. The main rsync above ships Mac2's designs.json over
# prod's, which DROPS the prod-side room_mockups fields (data/rooms PNGs are
# rsync-excluded so they survive — only the JSON pointer is lost). Without this,
# every deploy blanks the "See it in a room" on thousands of live PDPs until a
# nightly cron slowly heals it. Re-link is $0/instant (skip-if-exists reads the
# on-disk PNGs, no re-render). Non-fatal (|| true) so it can NEVER break a deploy.
echo "[5c/6] re-link room mockups from on-disk PNGs (heals the designs.json clobber)..."
ssh "$REMOTE" "cd $REMOTE_DIR && python3 scripts/backfill-rooms.py --limit 5000 2>&1 | tail -3 || true"

# 6b. PDP theme smoke test — themes must be FULLY WIRED (Steve standing rule).
# Loads every theme variant + the v11 Wallpaper/Wall Mural toggle against the
# just-deployed site and asserts each control re-renders its dependent DOM.
# rc 0 = pass, rc 3 = environment can't run a browser (skip), else = FAIL.
echo "[5b/6] PDP theme smoke test → https://$SITE"
SMOKE_RC=0
BASE="https://$SITE" node "$LOCAL_DIR/scripts/smoke-test-themes.js" || SMOKE_RC=$?
if [ "$SMOKE_RC" = "0" ]; then
  echo "  OK — all PDP theme smoke checks passed."
elif [ "$SMOKE_RC" = "3" ]; then
  echo "  SKIPPED — no usable browser/Playwright in this environment; theme smoke test not run."
else
  echo "ERROR: PDP theme smoke test FAILED (rc=$SMOKE_RC) — a theme is not fully wired (see output above)."
  exit 1
fi

# 6c. needs-tif admin-asset smoke test — confirm the just-deployed /admin/needs-tif
# actually shipped its features. It's admin-gated (loopback-only) + prod's own DB is
# sparse, so we can't run the full drag-select E2E here (that's scripts/test-needstif.js
# on data-rich Mac2). Instead, fetch the served HTML over the box's loopback (which
# passes the admin gate) and assert the critical markers are present. Pure structural
# "did it ship" check — no browser, no data needed. Functional E2E lives on Mac2.
echo "[5c/6] needs-tif admin-asset smoke test → loopback on $REMOTE"
NT_RC=$(ssh "$REMOTE" "PORT=$PORT bash -s" <<'EOS' 2>&1
H=$(curl -s "http://127.0.0.1:${PORT}/admin/needs-tif")
CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${PORT}/admin/needs-tif")
if [ "$CODE" != "200" ]; then echo "HTTP $CODE (admin route not serving)"; exit 2; fi
MISSING=""
check() { printf '%s' "$H" | grep -qF "$1" || MISSING="$MISSING | $1"; }
check 'class="marquee"'
check 'id="selbar"'
check 'applyMarquee'
check 'Copy IDs'
check 'Group size (largest first)'
check 'Color (by hue)'
check 'id="sparse"'
if [ -n "$MISSING" ]; then echo "missing markers:$MISSING"; exit 3; fi
echo OK
EOS
) && NT_OK=1 || NT_OK=0
if [ "$NT_OK" = "1" ]; then
  echo "  OK — /admin/needs-tif shipped (marquee select + selection bar + sorts + sparse banner present)."
else
  echo "ERROR: needs-tif admin-asset smoke test FAILED — $NT_RC"
  exit 1
fi

# 7. Nginx vhost (create if not present)
echo "[6/6] nginx vhost..."
VHOST_SRC="/etc/nginx/sites-available/$SITE"
VHOST_EXISTS=$(ssh "$REMOTE" "test -f $VHOST_SRC && echo yes || echo no")
if [ "$VHOST_EXISTS" = "no" ]; then
  echo "  Creating nginx vhost..."
  ssh "$REMOTE" "cat > $VHOST_SRC" <<NGINX
server {
    listen 80;
    listen [::]:80;
    server_name $SITE www.$SITE;

    # Cloudflare real IP passthrough
    set_real_ip_from 103.21.244.0/22;
    set_real_ip_from 103.22.200.0/22;
    set_real_ip_from 103.31.4.0/22;
    set_real_ip_from 104.16.0.0/13;
    set_real_ip_from 104.24.0.0/14;
    set_real_ip_from 108.162.192.0/18;
    set_real_ip_from 131.0.72.0/22;
    set_real_ip_from 141.101.64.0/18;
    set_real_ip_from 162.158.0.0/15;
    set_real_ip_from 172.64.0.0/13;
    set_real_ip_from 173.245.48.0/20;
    set_real_ip_from 188.114.96.0/20;
    set_real_ip_from 190.93.240.0/20;
    set_real_ip_from 197.234.240.0/22;
    set_real_ip_from 198.41.128.0/17;
    set_real_ip_from 2400:cb00::/32;
    set_real_ip_from 2606:4700::/32;
    set_real_ip_from 2803:f800::/32;
    set_real_ip_from 2405:b500::/32;
    set_real_ip_from 2405:8100::/32;
    set_real_ip_from 2a06:98c0::/29;
    set_real_ip_from 2c0f:f248::/32;
    real_ip_header CF-Connecting-IP;

    location / {
        proxy_pass http://127.0.0.1:$PORT;
        proxy_http_version 1.1;
        proxy_set_header Upgrade \$http_upgrade;
        proxy_set_header Connection keep-alive;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_cache_bypass \$http_upgrade;

        # Cache-Control: no-store on HTML (per feedback_cloudflare_html_caching.md)
        add_header Cache-Control "no-store, must-revalidate" always;

        # Security headers
        add_header X-Frame-Options SAMEORIGIN always;
        add_header X-Content-Type-Options nosniff always;
        add_header Referrer-Policy strict-origin-when-cross-origin always;
        add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
        add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
    }

    # Allow certbot challenge
    location /.well-known/acme-challenge/ {
        root /var/www/html;
    }
}
NGINX
  ssh "$REMOTE" "ln -sf $VHOST_SRC /etc/nginx/sites-enabled/$SITE 2>/dev/null || true"
  ssh "$REMOTE" "nginx -t && systemctl reload nginx"
  echo "  nginx vhost created and reloaded."
else
  echo "  nginx vhost already exists — skipping."
fi

echo ""
echo "=== Deploy complete ==="
echo "Origin: http://45.61.58.125:$PORT/health"
echo "Next: run certbot + CF zone creation, then verify https://$SITE/health"