`test/rotate-dw-admin-full.test.sh`

182 lines
#!/usr/bin/env bash
# Synthetic harness for rotate-dw-admin-full.sh (TK-11480).
#
# Runs the rotation script inside a throwaway sandbox where EVERY external
# effect is a stub: openssl (emits a fixed SYNTHETIC password), psql, ssh,
# sudo, pm2, and a fake secrets-manager cli.js under a fake $HOME. The ssh stub
# executes the remote command locally with /root and /etc paths rewritten into
# the sandbox, so the Kamatera-side DSN rewrite + pm2 restart logic really runs.
# A python3 wrapper records argv so a password passed as an argument is caught.
#
# No real secret is read, no real DB/host/pm2 is touched. Nothing leaves the box.
#
# Usage: test/rotate-dw-admin-full.test.sh [path-to-script]
#   default script = ../rotate-dw-admin-full.sh. Pass an older copy to prove
#   the harness goes RED on the known defects (negative test).
set -uo pipefail

HERE="$(cd "$(dirname "$0")" && pwd)"
SCRIPT="$(cd "$(dirname "${1:-$HERE/../rotate-dw-admin-full.sh}")" && pwd)/$(basename "${1:-rotate-dw-admin-full.sh}")"
[ -f "$SCRIPT" ] || { echo "no script: $SCRIPT" >&2; exit 2; }

SYNTH="SYNTHpw0000000000000000000000Zz9"   # 32 alnum chars, obviously fake
OLD="OLDsynth"
REAL_PY="$(command -v python3)"
PASS=0; FAIL=0
ok(){ PASS=$((PASS+1)); printf '  PASS  %s\n' "$1"; }
bad(){ FAIL=$((FAIL+1)); printf '  FAIL  %s\n' "$1"; }

mk_sandbox(){
  SB="$(mktemp -d "${TMPDIR:-/tmp}/dwrot-test.XXXXXX")"
  mkdir -p "$SB/bin" "$SB/home/Projects/secrets-manager" "$SB/root/public-projects/appA" \
           "$SB/root/public-projects/appB/config" "$SB/root/public-projects/appC" "$SB/root/.pm2" "$SB/etc" "$SB/log"
  : > "$SB/log/argv.log"
  printf 'DATABASE_URL=postgres://dw_admin:%s@127.0.0.1:5432/dw_unified\n' "$OLD" > "$SB/root/public-projects/appA/.env"
  printf 'module.exports={env:{PG:"postgresql://dw_admin:%s@localhost/dw_unified"}}\n' "$OLD" > "$SB/root/public-projects/appB/config/db.js"
  printf 'DATABASE_URL=postgres://other_user:x@127.0.0.1/other\n' > "$SB/root/public-projects/appC/.env"
  printf '{}\n' > "$SB/root/.pm2/dump.pm2"; : > "$SB/etc/environment"

  # every stub appends "<name> <argv...>" to argv.log
  cat > "$SB/bin/_log" <<EOF
#!/bin/bash
printf '%s\n' "\$*" >> "$SB/log/argv.log"
EOF
  cat > "$SB/bin/openssl" <<EOF
#!/bin/bash
"$SB/bin/_log" openssl "\$@"; printf '%s\n' "$SYNTH$SYNTH"
EOF
  cat > "$SB/bin/psql" <<EOF
#!/bin/bash
"$SB/bin/_log" psql "\$@"
cat >> "$SB/log/psql.stdin"
[ -n "\${FAKE_PSQL_FAIL:-}" ] && { echo "psql: FATAL (synthetic)" >&2; exit 2; }
exit 0
EOF
  cat > "$SB/bin/sudo" <<EOF
#!/bin/bash
"$SB/bin/_log" sudo "\$@"
while [ \$# -gt 0 ]; do case "\$1" in -n) shift;; -u) shift 2;; *) break;; esac; done
exec "\$@"
EOF
  # ssh: log, then run the remote command locally with /root and /etc sandboxed
  cat > "$SB/bin/ssh" <<EOF
#!/bin/bash
"$SB/bin/_log" ssh "\$@"
args=(); for a in "\$@"; do case "\$a" in -n|-T) ;; *) args+=("\$a");; esac; done
cmd="\${args[*]:1}"
cmd="\${cmd//\/root\//$SB/root/}"; cmd="\${cmd//\"\/root\"/\"$SB/root\"}"; cmd="\${cmd//\/etc\/environment/$SB/etc/environment}"
echo ssh >> "$SB/log/ssh.count"
exec bash -c "\$cmd"
EOF
  cat > "$SB/bin/pm2" <<EOF
#!/bin/bash
"$SB/bin/_log" pm2 "\$@"
if [ "\$1" = jlist ] && [ -n "\${FAKE_PM2_JLIST_FAIL:-}" ]; then echo "pm2 daemon unreachable (synthetic)" >&2; exit 1; fi
if [ "\$1" = jlist ]; then cat <<J
[{"name":"appA","pm2_env":{"status":"online","pm_cwd":"$SB/root/public-projects/appA"}},
 {"name":"appB","pm2_env":{"status":"online","pm_cwd":"$SB/root/public-projects/appB"}},
 {"name":"appC","pm2_env":{"status":"online","pm_cwd":"$SB/root/public-projects/appC"}},
 {"name":"appD","pm2_env":{"status":"stopped","pm_cwd":"$SB/root/public-projects/appA"}},
 {"name":"rootcwd","pm2_env":{"status":"online","pm_cwd":"$SB/root"}}]
J
fi
exit 0
EOF
  cat > "$SB/bin/python3" <<EOF
#!/bin/bash
"$SB/bin/_log" python3 "\$@"
exec "$REAL_PY" "\$@"
EOF
  cat > "$SB/home/Projects/secrets-manager/cli.js" <<EOF
let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{require('fs').appendFileSync('$SB/log/cli.stdin',s)});
EOF
  chmod +x "$SB/bin/"*
}

run_script(){ # $1 = stdin answers, rest = script args
  local answers="$1"; shift
  printf '%s' "$answers" | env -i PATH="$SB/bin:/usr/bin:/bin:$(dirname "$(command -v node)")" HOME="$SB/home" \
    TMPDIR="$SB" FAKE_PSQL_FAIL="${FAKE_PSQL_FAIL:-}" FAKE_PM2_JLIST_FAIL="${FAKE_PM2_JLIST_FAIL:-}" bash "$SCRIPT" "$@" > "$SB/log/out" 2>&1
  echo $?
}

echo "harness: $SCRIPT"

# ---- T1: full supervised run, all confirms = y ---------------------------------
mk_sandbox
rc=$(run_script $'y\ny\ny\n')
grep -q "$SYNTH" "$SB/log/argv.log" && bad "T1 password appeared in a process argv (defect 1)" \
                                 || ok  "T1 password never in any process argv"
[ "$(grep -c "ALTER ROLE dw_admin PASSWORD 'SCRAM-SHA-256" "$SB/log/psql.stdin" 2>/dev/null)" -ge 2 ] \
  && ok "T1 SCRAM ALTER reached Mac2 + prod psql via stdin" || bad "T1 ALTER did not reach both psql stdins"
grep -q "$SYNTH" "$SB/log/psql.stdin" 2>/dev/null && bad "T1 PLAINTEXT pw reached psql (would land in PG logs)" \
  || ok "T1 no plaintext pw in any SQL sent to Postgres"
ALTER_STMT="$(grep -m1 "ALTER ROLE dw_admin" "$SB/log/psql.stdin")"
grep -q "PG_DW_ADMIN_PASSWORD=$SYNTH" "$SB/log/cli.stdin" 2>/dev/null && ok "T1 registry fan got the new pw on stdin" \
  || bad "T1 registry fan missing"
grep -q "dw_admin:$SYNTH@" "$SB/root/public-projects/appA/.env" && grep -q "dw_admin:$SYNTH@" "$SB/root/public-projects/appB/config/db.js" \
  && ok "T1 remote received the REAL new pw and rewrote both DSNs" \
  || bad "T1 remote DSN rewrite wrong/missing (defect 2: heredoc overrides stdin pipe)"
grep -q 'other_user:x@' "$SB/root/public-projects/appC/.env" && ok "T1 non-dw_admin DSN untouched" || bad "T1 touched a non-dw_admin DSN"
[ -n "$(find "$SB/root/public-projects/appA" -name '*.pre-rot.*.bak')" ] && ok "T1 backup written" || bad "T1 no backup written"
restarts="$(grep '^pm2 restart' "$SB/log/argv.log" || true)"
if echo "$restarts" | grep -q appA && echo "$restarts" | grep -q appB; then ok "T1 affected apps appA+appB restarted"
else bad "T1 affected apps NOT restarted (defect 3: CHG passed as argv, not env)"; fi
echo "$restarts" | grep -qE 'appC|appD|rootcwd' && bad "T1 restarted an unaffected/stopped/root-cwd app" || ok "T1 no unaffected app restarted"
grep -q '^pm2 start' "$SB/log/argv.log" && bad "T1 started split-brain workers without --start-split-brain" || ok "T1 no unrequested worker starts"
[ "$rc" = 0 ] && ok "T1 exit 0" || bad "T1 exit $rc"

# ---- T2: rerun must not clobber the first backup --------------------------------
first_bak="$(find "$SB/root/public-projects/appA" -name '*.pre-rot.*.bak' | head -1)"
printf 'DATABASE_URL=postgres://dw_admin:%s@127.0.0.1:5432/dw_unified\n' "SECONDold" > "$SB/root/public-projects/appA/.env"
sleep 1; run_script $'y\ny\ny\n' >/dev/null
if [ -n "$first_bak" ] && grep -q "dw_admin:$OLD@" "$first_bak"; then ok "T2 original backup preserved on rerun"
else bad "T2 rerun clobbered the original backup"; fi
rm -rf "$SB"

# ---- T3: Mac2 ALTER fails -> must stop before touching prod ---------------------
mk_sandbox
rc=$(FAKE_PSQL_FAIL=1 run_script $'y\ny\ny\n')
[ ! -s "$SB/log/ssh.count" ] && ok "T3 no ssh to prod after local ALTER failure" \
  || bad "T3 continued to prod after local failure (defect 4: no errexit)"
[ "$rc" != 0 ] && ok "T3 non-zero exit ($rc)" || bad "T3 exited 0 despite failure"
grep -q "dw_admin:$OLD@" "$SB/root/public-projects/appA/.env" && ok "T3 prod DSNs untouched" || bad "T3 prod DSNs changed"
rm -rf "$SB"

# ---- T4: dry-run changes nothing -----------------------------------------------
mk_sandbox
rc=$(run_script '' --dry-run)
grep -qE '^(psql|ssh|pm2) ' "$SB/log/argv.log" && bad "T4 dry-run invoked psql/ssh/pm2" || ok "T4 dry-run touched nothing"
grep -q "$SYNTH" "$SB/log/out" && bad "T4 dry-run printed the password" || ok "T4 dry-run never printed the password"
rm -rf "$SB"

# ---- T6: the emitted SCRAM ALTER really authenticates on a REAL throwaway PG --
if command -v initdb >/dev/null && command -v pg_ctl >/dev/null && [ -n "${ALTER_STMT:-}" ]; then
  PGT="$(mktemp -d "${TMPDIR:-/tmp}/dwrot-pg.XXXXXX")"; PORT=$((50000 + RANDOM % 5000))
  if initdb -D "$PGT/data" -U tester --auth=trust >/dev/null 2>&1; then
    # TCP only: macOS caps unix-socket paths at ~103 chars and TMPDIR paths run long.
    printf 'host all tester 127.0.0.1/32 trust\nhost all all 127.0.0.1/32 scram-sha-256\n' > "$PGT/data/pg_hba.conf"
    if pg_ctl -D "$PGT/data" -o "-p $PORT -c unix_socket_directories='' -c listen_addresses=127.0.0.1" -l "$PGT/log" -w start >/dev/null 2>&1 \
       && psql -X -q -h 127.0.0.1 -p "$PORT" -U tester -d postgres -v ON_ERROR_STOP=1 -c "CREATE ROLE dw_admin LOGIN" >/dev/null 2>&1 \
       && printf '%s\n' "$ALTER_STMT" | psql -X -q -h 127.0.0.1 -p "$PORT" -U tester -d postgres -v ON_ERROR_STOP=1 -f - >/dev/null 2>&1; then
    PGPASSWORD="$SYNTH" psql -X -h 127.0.0.1 -p "$PORT" -U dw_admin -d postgres -tAc 'select 1' 2>/dev/null | grep -q 1 \
      && ok "T6 real PG: new pw authenticates via the client-side SCRAM verifier" || bad "T6 real PG: new pw REJECTED"
    PGPASSWORD="WRONGpw" psql -X -h 127.0.0.1 -p "$PORT" -U dw_admin -d postgres -tAc 'select 1' >/dev/null 2>&1 \
      && bad "T6 real PG: WRONG pw accepted (auth not enforced — test invalid)" || ok "T6 real PG: wrong pw rejected (negative control)"
    else bad "T6 NOT MEASURED: throwaway PG did not start / role or ALTER failed ($(tail -1 "$PGT/log" 2>/dev/null))"; fi
    pg_ctl -D "$PGT/data" -m immediate stop >/dev/null 2>&1
  else bad "T6 initdb failed (NOT MEASURED)"; fi
  rm -rf "$PGT"
else bad "T6 NOT MEASURED (no initdb or no captured ALTER)"; fi

# ---- T5: pm2 jlist fails mid-fan -> must exit non-zero, never a false-green DONE
mk_sandbox
rc=$(FAKE_PM2_JLIST_FAIL=1 run_script $'y\ny\ny\n')
[ "$rc" != 0 ] && ok "T5 non-zero exit when pm2 jlist fails ($rc)" || bad "T5 exit 0 with pm2 jlist dead (silent zero-restart)"
grep -q '^DONE' "$SB/log/out" && bad "T5 printed DONE despite no restart targets" || ok "T5 no false-green DONE"
grep -q 'PARTIAL ROTATION' "$SB/log/out" && ok "T5 partial-rotation state reported" || bad "T5 no partial-rotation warning"
rm -rf "$SB"

echo "result: $PASS pass, $FAIL fail"
[ "$FAIL" = 0 ]