`scripts/deploy-fleet.sh`

79 lines
#!/usr/bin/env bash
# deploy-fleet.sh — push dw-domain-fleet to Kamatera, install, pm2 start all 44,
# install nginx vhosts (HTTP only — certbot adds 443 after DNS cutover).
set -euo pipefail

HOST=root@45.61.58.125
REMOTE=/root/public-projects/dw-domain-fleet
LOCAL="$(cd "$(dirname "$0")/.." && pwd)"

echo "==> rsync $LOCAL -> $HOST:$REMOTE"
ssh "$HOST" "mkdir -p $REMOTE"
# RSYNC_EXTRA_EXCLUDES (space-separated paths) — this rsync ships the WORKING TREE,
# not HEAD, so any other agent's uncommitted in-flight file rides along silently. That
# matters on a shared repo: TK-11463's approved deploy would otherwise have pushed
# TK-11470's shared/render.js candidate, which that ticket explicitly holds as
# "production adoption/deployment remains gated". Excluding a path leaves whatever prod
# already has untouched (--exclude also exempts it from --delete). Verify with a
# checksum that prod's copy is the one you want KEPT before excluding it.
EXTRA=()
for x in ${RSYNC_EXTRA_EXCLUDES:-}; do EXTRA+=(--exclude "$x"); echo "    (excluding $x — prod keeps its current copy)"; done
rsync -az --delete \
  --exclude node_modules --exclude .git --exclude 'logs/*' --exclude '.env*' \
  "${EXTRA[@]+"${EXTRA[@]}"}" \
  "$LOCAL/" "$HOST:$REMOTE/"

echo "==> install deps + pm2 start"
ssh "$HOST" "cd $REMOTE && npm install --omit=dev --silent && pm2 start ecosystem.config.js && pm2 save"

echo "==> install nginx vhosts"
# NO-CLOBBER: only install a vhost conf if it doesn't already exist on the box.
# certbot --nginx upgrades each conf in-place with the :443 block + cert; an unconditional
# cp here overwrites that and takes the whole fleet HTTPS-down (000) on every deploy.
# New domains still get their HTTP conf; existing (certbot-managed) confs are preserved.
ssh "$HOST" "for f in $REMOTE/data/nginx/*.conf; do \
    b=\$(basename \$f); \
    [ -e /etc/nginx/sites-available/\$b ] || cp \$f /etc/nginx/sites-available/\$b; \
    ln -sf /etc/nginx/sites-available/\$b /etc/nginx/sites-enabled/\$b; \
  done && nginx -t && systemctl reload nginx"

echo "==> smoke test (origin, via Host header)"
FAIL=0
for d in $(ls "$LOCAL/data/nginx/" | sed 's/.conf//'); do
  # 301 is HEALTHY here: certbot rewrites each vhost to redirect HTTP->HTTPS, so an origin
  # probe over plain HTTP answers 301, not 200. Demanding 200 made every deploy report
  # "43 failures" on a completely healthy fleet — a false alarm that trains everyone to
  # ignore this smoke test, which is how a real outage gets waved through. Accept 2xx/301,
  # and follow the redirect to confirm the app really answers.
  CODE=$(ssh "$HOST" "curl -s -o /dev/null -w '%{http_code}' -H 'Host: $d' http://127.0.0.1/health" || echo 000)
  case "$CODE" in
    200|301|302)
      LIVE=$(curl -s --http1.1 --max-time 12 -o /dev/null -w '%{http_code}' "https://$d/" 2>/dev/null || echo 000)
      if [ "$LIVE" = "200" ]; then echo "  ok   $d (origin $CODE, https $LIVE)"
      else echo "  FAIL $d (origin $CODE but https $LIVE)"; FAIL=$((FAIL+1)); fi ;;
    *) echo "  FAIL $d ($CODE)"; FAIL=$((FAIL+1)) ;;
  esac
done
# ---- GRID GATE (TK-11463) ----------------------------------------------------
# The smoke test above proves each site ANSWERS. It does not prove each site has
# any PRODUCTS. For two days the 8 catalog-serving sites served 200 with a
# completely empty grid (shared/catalog.js's display_variant rule silently went
# from rejecting 14% of the catalog to rejecting 100% of it) and this smoke test
# printed "ok" on every one of them, every deploy, through the whole outage —
# because a zero-product store answers 200 just like a healthy one.
# assert-grids.js measures the thing that actually matters: rendered product
# cards, with the pool size beside it, and it refuses to report PASS on any site
# it could not read. It ships with scripts/test-assert-grids.sh, which injects
# four faults and proves the gate goes red on each. Never pass --test here.
echo "==> grid gate (do the catalog sites actually render products?)"
if node "$LOCAL/scripts/assert-grids.js" --min 12; then
  echo "  grid gate ok"
else
  GRID_RC=$?
  echo "  GRID GATE FAILED (rc=$GRID_RC) — a catalog site is serving an empty or unreadable grid"
  FAIL=$((FAIL+1))
fi

echo "==> deploy complete · $FAIL failures"
exit $FAIL