`verification/gracie-rollout/rollout.cjs`

80 lines
// Approved TK11438 local Gracie-only migration. No email, publishing or remote ops.
const fs=require('fs'),path=require('path'),crypto=require('crypto'),cp=require('child_process'),assert=require('assert/strict'),vm=require('vm'),os=require('os');
const ROOT='/Users/macstudio3/Projects/gracie-internal',OUT=__dirname,PRIV=OUT+'/private',DUMP='/Users/macstudio3/.pm2/dump.pm2';
const EVIDENCE='/Users/macstudio3/Projects/tk11438-postgres-migration/verification/fleet-classification';
const {Client}=require(ROOT+'/node_modules/pg');
const deps='/Users/macstudio3/.npm-global/lib/node_modules/pm2/node_modules/';
const axon=require(deps+'pm2-axon'),rpc=require(deps+'pm2-axon-rpc');
const mode=process.argv[2];assert(['prepare','apply','verify','observe','rollback'].includes(mode));
const sha=x=>crypto.createHash('sha256').update(x).digest('hex');
const json=p=>JSON.parse(fs.readFileSync(p));
const record=(name,x)=>fs.writeFileSync(OUT+'/'+name+'.json',JSON.stringify({at:new Date().toISOString(),...x},(k,v)=>k==='auth_sha'?'[stored in private rollback evidence]':v,2)+'\n');
const run=(cmd,args,opts={})=>cp.execFileSync(cmd,args,{encoding:'utf8',timeout:15000,maxBuffer:10*1024*1024,...opts});
const git=(...args)=>run('git',args,{cwd:ROOT}).trim();
const rpcCall=(method,opts={})=>new Promise((resolve,reject)=>{const sock=axon.socket('req'),client=new rpc.Client(sock);const t=setTimeout(()=>{sock.close();reject(Error('RPC timeout '+method));},30000);sock.on('error',e=>{clearTimeout(t);sock.close();reject(e);});sock.connect('/Users/macstudio3/.pm2/rpc.sock');client.call(method,opts,(err,data)=>{clearTimeout(t);sock.close();err?reject(Error('RPC failure '+method)):resolve(data);});});
function unique(list){const a=list.filter(x=>x.name==='gracie-internal');assert.equal(a.length,1);const x=a[0],e=x.pm2_env||x;assert.equal(e.pm_cwd,ROOT);assert.equal(e.pm_exec_path,ROOT+'/server.js');return x;}
function fields(e){assert(e.env);return{PG:e.PG,env:{PG:e.env.PG}};}
function setFields(e,v){e.PG=v.PG;e.env.PG=v.env.PG;}
function socketURL(s){const u=new URL(s);assert(['localhost','127.0.0.1','[::1]',''].includes(u.hostname));assert.equal(u.pathname,'/dw_unified');assert(['','5432'].includes(u.port));u.searchParams.set('host','/tmp');return u.toString();}
function atomic(file,bytes,expected){assert.equal(sha(fs.readFileSync(file)),expected,'concurrent file change '+file);const temp=file+'.TK11438-'+process.pid;fs.writeFileSync(temp,bytes,{flag:'wx',mode:fs.statSync(file).mode&0o777});assert.equal(sha(fs.readFileSync(file)),expected);fs.renameSync(temp,file);assert.equal(sha(fs.readFileSync(file)),sha(bytes));}
function patchDump(expected,next){const bytes=fs.readFileSync(DUMP),d=JSON.parse(bytes);assert.deepEqual(fields(unique(d)),expected);setFields(unique(d),next);const restored=JSON.parse(JSON.stringify(d));setFields(unique(restored),expected);assert.deepEqual(restored,JSON.parse(bytes));atomic(DUMP,JSON.stringify(d,null,2),sha(bytes));return{only_gracie_two_fields:true};}
function controls(e){return{auth_sha:sha(e.BASIC_AUTH||''),PGHOST:e.PGHOST||null,nested_PGHOST:e.env?.PGHOST||null,DATA_SOURCE:e.DATA_SOURCE||null,PORT:String(e.PORT||10073),cwd:e.pm_cwd,script:e.pm_exec_path};}
function dbOpts(source,e){let captured;const match=source.match(/let pool = null;\s*function db\(\) \{[\s\S]*?\n\}/);assert(match);const secret=fs.readFileSync('/Users/macstudio3/Projects/secrets-manager/.env','utf8').match(/^DW_ADMIN_DB_PASSWORD=(.+)$/m);const pass=e.DW_ADMIN_DB_PASSWORD||(secret?secret[1].replace(/^["']|["']$/g,'').trim():'');vm.runInNewContext(match[0]+';db();',{process:{platform:process.platform,env:e},PGPASS:pass,Pool:class{constructor(o){captured=o;}}},{timeout:1000});return captured;}
async function withDb(options,fn){const c=new Client({...options,application_name:'TK11438-Gracie-'+mode,connectionTimeoutMillis:4000,options:'-c default_transaction_read_only=on -c statement_timeout=5000'});try{await c.connect();return await fn(c);}finally{await c.end().catch(()=>{});}}
async function dataProof(options){return withDb(options,async c=>({identity:(await c.query("SELECT current_database() database,current_user role,inet_client_addr()::text addr,current_setting('transaction_read_only') readonly,(SELECT oid FROM pg_database WHERE datname=current_database()) database_oid,pg_postmaster_start_time() server_started")).rows[0],requests:(await c.query("SELECT count(*)::int count,max(id)::text last_id FROM vendor_requests WHERE vendor_code='gracie'")).rows[0],schema:(await c.query("SELECT column_name,data_type,is_nullable,column_default FROM information_schema.columns WHERE table_schema='public' AND table_name='vendor_requests' ORDER BY ordinal_position")).rows,indexes:(await c.query("SELECT indexname,indexdef FROM pg_indexes WHERE schemaname='public' AND tablename='vendor_requests' ORDER BY indexname")).rows}));}
function lsof(pid){try{return run('lsof',['-nP','-a','-p',String(pid),'-iTCP']);}catch(e){if(e.status===1)return e.stdout||'';throw e;}}
async function snapshot(name,requests=false){const p=unique(await rpcCall('getMonitorData')),e=p.pm2_env;assert.equal(e.status,'online');const source=fs.readFileSync(ROOT+'/server.js','utf8'),poolSource=fs.readFileSync(ROOT+'/lib/vendor-requests.js','utf8');const auth=e.BASIC_AUTH||source.match(/process\.env\.BASIC_AUTH \|\| '([^']+)'/)[1];
 const checks=[];let count;for(const route of ['/healthz','/api/products',...(requests?['/api/requests']:[])])for(const [label,a]of [['missing',null],['invalid','invalid:invalid'],['valid',auth]]){const r=await fetch('http://127.0.0.1:'+String(e.PORT||10073)+route,{headers:a?{authorization:'Basic '+Buffer.from(a).toString('base64')}:{},signal:AbortSignal.timeout(8000)});const body=await r.text();assert.equal(r.status,route==='/healthz'||label==='valid'?200:401,route+' '+label);if(label==='valid'&&route==='/api/products')count=JSON.parse(body).count;if(label==='valid'&&route==='/api/requests')assert(Array.isArray(JSON.parse(body).requests));checks.push({route,auth:label,status:r.status,body_sha:sha(body),bytes:Buffer.byteLength(body)});}
 const pool=dbOpts(poolSource,e),vendor=await dataProof(pool),catalog=await withDb({connectionString:e.PG},async c=>(await c.query("SELECT current_database() database,current_user role,inet_client_addr()::text addr,current_setting('transaction_read_only') readonly")).rows[0]);
 const tcp=lsof(p.pid).split('\n').filter(Boolean),proof={pid:p.pid,pm_id:e.pm_id,restarts:e.restart_time,controls:controls(e),http:checks,product_count:count,vendor,catalog,vendor_config:{host:pool.host,database:pool.database,user:pool.user,port:pool.port},saved_match:JSON.stringify(fields(e))===JSON.stringify(fields(unique(json(DUMP)))),listeners:tcp.filter(l=>/LISTEN/.test(l)),database_tcp:tcp.filter(l=>/:5432\b/.test(l)),source_hashes:{server:sha(source),vendor:sha(poolSource)},bundle_sha:sha(fs.readFileSync(ROOT+'/data/gracie.jsonl')),email_sends:0};record(name,proof);return proof;
}
function fileSpecs(){const pre=json(EVIDENCE+'/gracie-preflight.json'),shared=json(EVIDENCE+'/shared-modules.json'),entry=shared.modules.find(x=>x.path===ROOT+'/lib/vendor-requests.js');const oldServer=fs.readFileSync(ROOT+'/server.js','utf8'),oldVendor=fs.readFileSync(entry.path,'utf8');assert.equal(sha(oldServer),pre.source.before_sha256);assert.equal(sha(oldVendor),entry.sha256);
 const old="const PG = process.env.PG || 'postgresql://localhost/dw_unified';",next="const PG = process.env.PG || (process.platform === 'darwin' ? 'postgresql://localhost/dw_unified?host=/tmp' : 'postgresql://localhost/dw_unified');";
 const bindOld='app.listen(PORT, () =>',bindNew="app.listen(PORT, '127.0.0.1', () =>";assert.equal(oldServer.split(old).length-1,1);assert.equal(oldServer.split(bindOld).length-1,1);assert.equal(oldVendor.split(shared.proposed_old).length-1,1);
 return[{key:'server',path:ROOT+'/server.js',before:oldServer,after:oldServer.replace(old,next).replace(bindOld,bindNew)},{key:'vendor',path:entry.path,before:oldVendor,after:oldVendor.replace(shared.proposed_old,shared.proposed_new)}];}
function restorationActions(specs,getCurrent){return specs.map(s=>{const current=getCurrent(s.path);assert([s.before_sha,s.after_sha,s.recovery_sha].includes(current),'peer edit blocks rollback');return{...s,current,restore:current!==s.recovery_sha};});}
(async()=>{
 if(mode==='prepare'){
  assert(!fs.existsSync(PRIV),'already prepared');assert.equal(git('status','--porcelain'),'');const p=unique(await rpcCall('getMonitorData')),e=p.pm2_env,before=fields(e);assert.deepEqual(before,fields(unique(json(DUMP))));assert.equal(before.PG,before.env.PG);assert(!e.PGHOST&&!e.env.PGHOST,'unexpected override');const next={PG:socketURL(before.PG),env:{PG:socketURL(before.env.PG)}};const specs=fileSpecs();
  for(const s of specs)run('node',['--check'],{input:s.after});const baseline=await snapshot('baseline',false);assert.equal(baseline.vendor.identity.database,'dw_unified');assert.equal(baseline.vendor.identity.role,'dw_admin');assert(baseline.vendor.schema.length>0,'Existing table required');
  for(const s of specs)s.recovery=s.key==='server'?s.before.replace('app.listen(PORT, () =>',"app.listen(PORT, '127.0.0.1', () =>"):s.before;
  fs.mkdirSync(PRIV,{mode:0o700});fs.writeFileSync(PRIV+'/baseline.runtime.json',JSON.stringify(baseline),{mode:0o600,flag:'wx'});for(const s of specs)for(const side of ['before','after','recovery'])fs.writeFileSync(PRIV+'/'+s.key+'.'+side,s[side],{mode:0o600,flag:'wx'});
  const receipt={head:git('rev-parse','HEAD'),pid:p.pid,pm_id:e.pm_id,before,next,controls:controls(e),specs:specs.map(s=>({key:s.key,path:s.path,before_sha:sha(s.before),after_sha:sha(s.after),recovery_sha:sha(s.recovery)}))};fs.writeFileSync(PRIV+'/receipt.json',JSON.stringify(receipt,null,2),{mode:0o600,flag:'wx'});
  // Rehearse changes and restoration on private copies, and both PG fields only.
  for(const s of receipt.specs){const f=PRIV+'/rehearsal-'+s.key;fs.copyFileSync(PRIV+'/'+s.key+'.before',f);atomic(f,fs.readFileSync(PRIV+'/'+s.key+'.after'),s.before_sha);atomic(f,fs.readFileSync(PRIV+'/'+s.key+'.before'),s.after_sha);assert.equal(sha(fs.readFileSync(f)),s.before_sha);}
  const d=json(DUMP),original=JSON.parse(JSON.stringify(d));setFields(unique(d),next);setFields(unique(d),before);assert.deepEqual(d,original);
  for(const mask of [0,1,2,3])for(const action of restorationActions(receipt.specs,(f)=>{const i=receipt.specs.findIndex(s=>s.path===f),s=receipt.specs[i];return(mask&(1<<i))?s.after_sha:s.before_sha;}))assert.equal(action.restore,action.current!==action.recovery_sha);
  assert(restorationActions(receipt.specs,f=>receipt.specs.find(s=>s.path===f).recovery_sha).every(s=>!s.restore));
  for(const s of receipt.specs){const f=PRIV+'/privacy-rehearsal-'+s.key;fs.copyFileSync(PRIV+'/'+s.key+'.after',f);atomic(f,fs.readFileSync(PRIV+'/'+s.key+'.recovery'),s.after_sha);assert.equal(sha(fs.readFileSync(f)),s.recovery_sha);if(s.key==='server')assert(fs.readFileSync(f,'utf8').includes("app.listen(PORT, '127.0.0.1', () =>"));run('node',['--check'],{input:fs.readFileSync(f,'utf8')});}
  assert.throws(()=>restorationActions(receipt.specs,()=>sha('peer edit')));
  record('preparation',{verdict:'PASS',rollback_rehearsal:'2 exact files plus 2 Gracie dump fields; 4 partial-mutation cases and peer-edit refusal',private_mode:fs.statSync(PRIV).mode&0o777,privacy:'localhost-only bind required by Steve',application_mutations:0,email_sends:0});
 }else if(mode==='apply'){
  const rec=json(PRIV+'/receipt.json');assert.equal(json(OUT+'/preparation.json').verdict,'PASS');assert(!fs.existsSync(OUT+'/mutation-started.json'),'already started; use rollback or verify');assert.equal(git('status','--porcelain'),'');assert.equal(git('rev-parse','HEAD'),rec.head);const p=unique(await rpcCall('getMonitorData'));assert.equal(p.pid,rec.pid);assert.deepEqual(fields(p.pm2_env),rec.before);assert.deepEqual(controls(p.pm2_env),rec.controls);assert.deepEqual(fields(unique(json(DUMP))),rec.before);
  for(const s of rec.specs)assert.equal(sha(fs.readFileSync(s.path)),s.before_sha);record('mutation-started',{scope:'Gracie only; local/private',restart_requested:false});
  for(const s of rec.specs)atomic(s.path,fs.readFileSync(PRIV+'/'+s.key+'.after'),s.before_sha);const dump=patchDump(rec.before,rec.next);record('config-applied',{dump,privacy:'127.0.0.1 listener in source'});
  const errPath=p.pm2_env.pm_err_log_path;record('error-log-baseline',{path:errPath,size:fs.existsSync(errPath)?fs.statSync(errPath).size:0});
  record('restart-requested',{before_pid:p.pid,pm_id:rec.pm_id});await rpcCall('restartProcessId',{id:rec.pm_id,env:{PG:rec.next.PG}});const after=unique(await rpcCall('getMonitorData'));assert.notEqual(after.pid,rec.pid);record('applied',{verdict:'APPLIED_VERIFY_PENDING',pid:after.pid,previous_pid:rec.pid});
 }else if(mode==='verify'||mode==='observe'){
  if(mode==='verify'&&!fs.existsSync(OUT+'/startup-schema.json')){
   const log=json(OUT+'/error-log-baseline.json'),bytes=fs.existsSync(log.path)?fs.readFileSync(log.path):Buffer.alloc(0);assert(bytes.length>=log.size,'error log rotated');const tail=bytes.subarray(log.size).toString();assert(!/ensureSchema failed|\[gracie-internal\] load failed/i.test(tail),'startup failure; do not invoke request GET');
   const started=json(OUT+'/restart-requested.json').at;
   const completed=await withDb({connectionString:'postgresql:///postgres?host=/tmp'},async c=>(await c.query("SELECT pid,backend_start,state,client_addr::text addr FROM pg_stat_activity WHERE datname='dw_unified' AND usename='dw_admin' AND backend_start >= $1 AND client_addr IS NULL AND state='idle' AND query LIKE '%CREATE TABLE IF NOT EXISTS vendor_requests%'",[started])).rows);
   assert(completed.length>0,'await completed socket startup DDL before request GET');record('startup-schema',{verdict:'PASS',socket_idle_ddl_backends:completed,error_log_new_bytes:tail.length,error_log_tail_sha:sha(tail),note:'New idle socket backend after startup DDL and no schema error logged; bounded source/runtime correlation'});
  }
  const rec=json(PRIV+'/receipt.json'),baseline=json(PRIV+'/baseline.runtime.json'),p=await snapshot(mode==='verify'?'after':'monitor',true);assert(p.saved_match);assert.deepEqual(p.controls,baseline.controls);assert.equal(p.product_count,baseline.product_count);assert.equal(p.bundle_sha,baseline.bundle_sha);assert.deepEqual(p.vendor.requests,baseline.vendor.requests);assert.deepEqual(p.vendor.schema,baseline.vendor.schema);assert.deepEqual(p.vendor.indexes,baseline.vendor.indexes);
  assert.equal(p.vendor.identity.addr,null);assert.equal(p.catalog.addr,null);assert.equal(p.vendor.identity.readonly,'on');assert.equal(p.catalog.readonly,'on');assert.deepEqual([p.vendor.identity.database,p.vendor.identity.role,p.vendor.identity.database_oid,new Date(p.vendor.identity.server_started).toISOString()],[baseline.vendor.identity.database,baseline.vendor.identity.role,baseline.vendor.identity.database_oid,new Date(baseline.vendor.identity.server_started).toISOString()]);assert.deepEqual([p.catalog.database,p.catalog.role],[baseline.catalog.database,baseline.catalog.role]);assert.equal(p.vendor_config.host,'/tmp');assert.equal(p.database_tcp.length,0);assert.equal(p.listeners.length,1);assert(/127\.0\.0\.1:10073\s+\(LISTEN\)/.test(p.listeners[0]));
  for(const s of rec.specs)assert.equal(sha(fs.readFileSync(s.path)),s.after_sha);const e=unique(await rpcCall('getMonitorData')).pm2_env;assert.deepEqual(fields(e),rec.next);assert.deepEqual(fields(unique(json(DUMP))),rec.next);
  const bad=new Client({host:OUT+'/missing-socket',database:'dw_unified',connectionTimeoutMillis:1000});let code;try{await bad.connect();}catch(err){code=err.code;}finally{await bad.end().catch(()=>{});}assert.equal(code,'ENOENT');
  const interfaces=Object.entries(os.networkInterfaces()).flatMap(([name,rows])=>rows.filter(x=>x.family==='IPv4'&&!x.internal).map(x=>({name,address:x.address}))),networkChecks=[];
  for(const iface of interfaces){let refused=false;try{await fetch('http://'+iface.address+':10073/healthz',{signal:AbortSignal.timeout(3000)});}catch(err){refused=true;}assert(refused,'Gracie reachable over nonloopback interface');networkChecks.push({...iface,result:'connection failed'});}
  if(mode==='observe'){const after=json(OUT+'/after.json');assert.equal(p.pid,after.pid);assert.equal(p.restarts,after.restarts);}
  record(mode==='verify'?'verification':'observation',{verdict:'PASS',pid:p.pid,auth_checks:p.http.length,product_count:p.product_count,role_database_preserved:true,request_records_unchanged:true,schema_unchanged:true,both_transports_socket:true,process_tcp5432:false,durable_match:true,listener:'127.0.0.1:10073',networkChecks,missing_socket:code,email_sends:0});
 }else if(mode==='rollback'){
  const rec=json(PRIV+'/receipt.json');assert(fs.existsSync(OUT+'/mutation-started.json'));const actions=restorationActions(rec.specs,f=>sha(fs.readFileSync(f)));const current=fields(unique(json(DUMP)));assert([JSON.stringify(rec.before),JSON.stringify(rec.next)].includes(JSON.stringify(current)));const e=unique(await rpcCall('getMonitorData')).pm2_env;assert.deepEqual(controls(e),rec.controls);
  // Privacy persists through recovery: restore transport, retain localhost bind.
  for(const s of actions)if(s.restore)atomic(s.path,fs.readFileSync(PRIV+'/'+s.key+'.recovery'),s.current);
  if(JSON.stringify(current)!==JSON.stringify(rec.before))patchDump(rec.next,rec.before);await rpcCall('restartProcessId',{id:rec.pm_id,env:{PG:rec.before.PG}});record('rollback',{verdict:'TRANSPORT_RESTORED_VERIFY_REQUIRED',privacy:'localhost binding retained per Steve; no reopening wildcard',baseline_semantics:'transport restored; intentional privacy change remains'});
 }
 console.log(JSON.stringify({mode,verdict:'PASS',evidence:OUT}));
})().catch(err=>{record(mode+'-failure',{verdict:'FAIL',code:err.code||err.name,message:String(err.message).replace(/postgres(?:ql)?:\/\/\S+/g,'[REDACTED_URI]')});console.error(JSON.stringify({mode,verdict:'FAIL',code:err.code||err.name,message:String(err.message).replace(/postgres(?:ql)?:\/\/\S+/g,'[REDACTED_URI]')}));process.exitCode=1;});