`data/codex-yoloforever/cycle-20260908T1322Z.m4gNE5/ops-reconstruct-cachetree-v2.py`

118 lines
#!/usr/bin/env python3
"""Preserve failed v1, add index TREE closure, rebuild under same data deadline."""
import datetime,hashlib,json,os,shutil,stat,struct,subprocess,time
from pathlib import Path
B=Path('/Users/macstudio3/Projects/ticket-system/data/codex-yoloforever/cycle-20260908T1322Z.m4gNE5')
R=Path('/private/tmp/ops-TK10928-reconstruct.xcLYt5')
S=Path('/private/tmp/ops-TK10928-rehearsal.ndMwgA/original.git')
RESTORE=R/'restored.git';FAILED=R/'trial.git';TRIAL=R/'trial-cachetree-complete.git'
STORE=R/'trial-superseded-objects-retained'
start=json.loads((B/'ops-reconstruct-data-start.json').read_text())['started_epoch'];deadline=start+580
ENV={k:v for k,v in os.environ.items() if not k.startswith('GIT_')}
ENV.update(GIT_OPTIONAL_LOCKS='0',GIT_CONFIG_NOSYSTEM='1',GIT_CONFIG_GLOBAL='/dev/null',GIT_NO_REPLACE_OBJECTS='1')
FULL=dict(ENV,GIT_OBJECT_DIRECTORY=str(STORE));checks=[];commands=[];result={}
def dump(n,x):
 p=B/('ops-reconstruct-'+n+'.json');p.write_text(json.dumps(x,indent=2)+'\n');return str(p)
def left():
 n=deadline-time.time()
 if n<=0:raise TimeoutError('same original data phase580second internal cap')
 return n
def check(n,c,d=None):
 checks.append({'check':n,'verdict':'PASS' if c else 'FAIL','detail':d});dump('v2-checks-progress',checks)
 if not c:raise ValueError(n+': '+repr(d))
def run(a,inp=None,env=None,out=None):
 e={'argv':[str(x) for x in a],'input_sha256':hashlib.sha256(inp).hexdigest() if inp else None,'stdout_path':str(out) if out else None,'object_directory':(env or ENV).get('GIT_OBJECT_DIRECTORY'),'started_epoch':time.time()};commands.append(e);dump('v2-commands',commands)
 if out:
  with Path(out).open('xb') as f:p=subprocess.run(a,input=inp,stdout=f,stderr=subprocess.PIPE,env=env or ENV,timeout=left())
  data=b''
 else:p=subprocess.run(a,input=inp,capture_output=True,env=env or ENV,timeout=left());data=p.stdout
 e.update(exit=p.returncode,stderr=p.stderr.decode(errors='replace')[:4000],finished_epoch=time.time());dump('v2-commands',commands)
 if p.returncode:raise ValueError(repr(e))
 return data
def git(repo,*a,**kw):return run(['git','--git-dir='+str(repo),'-c','core.commitGraph=false','-c','gc.auto=0',*a],**kw)
def sha(p):
 h=hashlib.sha256()
 with Path(p).open('rb') as f:
  for d in iter(lambda:f.read(1048576),b''):left();h.update(d)
 return h.hexdigest()
def du(p):return int(run(['du','-sk',str(p)]).split()[0])
def inv(raw):return {l.split()[0]:l.split() for l in raw.decode().splitlines()}
def parse_index(data):
 assert data[:4]==b'DIRC' and hashlib.sha1(data[:-20]).digest()==data[-20:]
 version,count=struct.unpack('>II',data[4:12]);assert version==2
 at=12
 for i in range(count):
  base=at;flags=struct.unpack('>H',data[at+60:at+62])[0];assert not flags&0x4000
  nul=data.index(0,at+62);at=base+((nul+1-base+7)//8)*8
 exts=[];trees=[]
 def cache(payload,pos,parent):
  nul=payload.index(0,pos);name=payload[pos:nul].decode();nl=payload.index(10,nul+1)
  entries,children=map(int,payload[nul+1:nl].split());pos=nl+1;path=parent+name
  if entries>=0:
   oid=payload[pos:pos+20].hex();assert len(oid)==40;pos+=20;trees.append({'path':path,'entries':entries,'children':children,'oid':oid})
  for _ in range(children):pos=cache(payload,pos,path+'/')
  return pos
 while at<len(data)-20:
  sig=data[at:at+4].decode();length=struct.unpack('>I',data[at+4:at+8])[0];payload=data[at+8:at+8+length];at+=8+length
  exts.append({'signature':sig,'bytes':length})
  if sig=='TREE':assert cache(payload,0,'')==len(payload)
  else:raise ValueError('Unreviewed index extension '+sig)
 assert at==len(data)-20
 return {'version':version,'entry_count':count,'extensions':exts,'cache_trees':trees,'index_internal_sha1':data[-20:].hex()}
try:
 check('canonical zero-cost guard unchanged',Path('/Users/macstudio3/Projects/ticket-system/config/dtd-cost-mode').read_bytes()==b'ZERO_COST_REQUIRED\n')
 first=json.loads((B/'ops-reconstruct-result.json').read_text());check('v1 real negative connectivity retained',first['status']=='blocked' and 'invalid sha1 pointer in cache-tree' in first['error'])
 idx=parse_index((RESTORE/'index').read_bytes());dump('index-cachetree-proof',idx)
 roots={x['oid'] for x in idx['cache_trees']}
 allold=inv((Path('/Users/macstudio3/Projects/ticket-system/data/codex-yoloforever/cycle-20260908T1221Z.HzXvuQ')/'ops-original-object-inventory.txt').read_bytes())
 check('all cache-tree pointers are known verified source trees',all(o in allold and allold[o][1]=='tree' for o in roots),{'roots':len(roots)})
 cacheclosure=set(git(RESTORE,'rev-list','--objects','--no-object-names','--stdin',inp=('\n'.join(sorted(roots))+'\n').encode()).decode().splitlines())
 initial=set((R/'candidate-retained-object-ids.txt').read_text().splitlines());retained=initial|cacheclosure
 oldprotected=set((R/'protected-closure.txt').read_text().splitlines());protected=oldprotected|cacheclosure
 kept=R/'candidate-v2-retained-object-ids.txt';kept.write_text('\n'.join(sorted(retained))+'\n')
 (R/'protected-v2-object-ids.txt').write_text('\n'.join(sorted(protected))+'\n')
 effect=json.loads((B/'ops-reconstruct-effect.json').read_text());candidates=set(x['git_oid'] for x in json.loads((B/'ops-five-render-proof.json').read_text())['restores'])
 dump('cachetree-closure',{'cache_roots':len(roots),'cache_closure_objects':len(cacheclosure),'protected_before':len(oldprotected),'protected_after':len(protected),'newly_added_protected_objects':sorted(retained-initial),'candidate_blobs_protected':sorted(protected&candidates)})
 TRIAL.mkdir()
 for p in FAILED.rglob('*'):
  rel=p.relative_to(FAILED)
  if rel.parts[0]=='objects':continue
  d=TRIAL/rel
  if p.is_dir():d.mkdir(parents=True,exist_ok=True)
  elif p.is_file():
   d.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,d);os.chmod(d,stat.S_IMODE(p.stat().st_mode));os.utime(d,ns=(p.stat().st_atime_ns,p.stat().st_mtime_ns))
 (TRIAL/'objects/pack').mkdir(parents=True);(TRIAL/'objects/info').mkdir()
 packid=git(FAILED,'pack-objects','--window=0','--threads=2',str(TRIAL/'objects/pack/pack'),inp=kept.read_bytes(),env=FULL).decode().strip()
 pack=TRIAL/'objects/pack'/('pack-'+packid+'.pack');packidx=pack.with_suffix('.idx')
 git(TRIAL,'verify-pack','-v',str(packidx),out=R/'candidate-v2-verify-pack.txt')
 git(TRIAL,'fsck','--connectivity-only','--no-dangling',out=R/'candidate-v2-connectivity.txt')
 check('Git connectivity passes with exact preserved index cache-tree',True)
 raw=git(TRIAL,'cat-file','--batch-all-objects','--batch-check=%(objectname) %(objecttype) %(objectsize) %(objectsize:disk)');(R/'candidate-v2-object-inventory.txt').write_bytes(raw);actual=inv(raw)
 check('actual candidate pack inventory equals entire required closure',set(actual)==retained,{'actual':len(actual),'expected':len(retained)})
 check('all original type and size identities preserved',all(actual[o][:3]==allold[o][:3] for o in set(allold)&retained))
 check('all protected roots cache-trees and descendants present',protected<=retained and roots<=retained)
 check('candidate is self-contained',not (TRIAL/'objects/info/alternates').exists())
 snap=effect['new_snapshot'];old=effect['original_snapshot'];master=git(RESTORE,'rev-parse','refs/heads/master').decode().strip()
 check('full leaf tree matches exact five-path effect proven in v1',git(TRIAL,'ls-tree','-rz','--full-tree',snap)==git(FAILED,'ls-tree','-rz','--full-tree',snap))
 origrefs=git(RESTORE,'for-each-ref','--format=%(objectname) %(refname)').decode().splitlines();newrefs=git(TRIAL,'for-each-ref','--format=%(objectname) %(refname)').decode().splitlines()
 check('every non-snapshot effective ref unchanged',[r for r in origrefs if not r.endswith(' refs/auto-snapshot/latest')]==[r for r in newrefs if not r.endswith(' refs/auto-snapshot/latest')])
 check('master full tree unchanged',git(TRIAL,'ls-tree','-rz','--full-tree',master)==git(RESTORE,'ls-tree','-rz','--full-tree',master))
 original_manifest=json.loads((Path('/Users/macstudio3/Projects/ticket-system/data/codex-yoloforever/cycle-20260908T1221Z.HzXvuQ')/'ops-source-git-manifest.json').read_text())
 metadata=[p for p in original_manifest if not p.startswith('objects/') and p not in ['refs/auto-snapshot/latest','logs/refs/auto-snapshot/latest','packed-refs']]
 check('all protected metadata,index,reflogs,pseudorefs bytes and modes unchanged',all(sha(TRIAL/p)==original_manifest[p]['sha256'] and stat.S_IMODE((TRIAL/p).stat().st_mode)==original_manifest[p]['mode'] for p in metadata),metadata)
 non_snapshot=lambda t:[x for x in t.splitlines(keepends=True) if not x.rstrip().endswith(' refs/auto-snapshot/latest')]
 check('non-snapshot packed-ref lines byte-identical',non_snapshot((TRIAL/'packed-refs').read_text())==non_snapshot((RESTORE/'packed-refs').read_text()))
 kib=du(TRIAL);mb=(kib+512)//1024
 js="import {classifyRepo,KNOWN_LEGIT} from '/Users/macstudio3/.claude/skills/dw-backup-canary/lib.mjs';let mb="+str(mb)+";console.log(JSON.stringify({mb,known:KNOWN_LEGIT['(projects-root)'],classification:classifyRepo({repo:'(projects-root)',mb,remote:null,upstream:null,ahead:0})}))"
 classified=json.loads(run(['node','--input-type=module','-e',js]));check('actual installed canary floor remains3000',classified['known']['floor_mb']==3000)
 archived=sorted(set(allold)-retained);(R/'archived-v2-original-object-inventory.txt').write_text('\n'.join(' '.join(allold[o]) for o in archived)+'\n')
 generated=json.loads((B/'ops-reconstruct-new-object-hashes.json').read_text())
 check('all four generated object Git identities persist',all(o['oid'] in actual and actual[o['oid']][1:3]==[o['type'],str(o['bytes'])] for o in generated))
 immutable=[S,*S.rglob('*')];check('all historical source immutable flags still set',all(p.stat().st_flags & stat.UF_IMMUTABLE for p in immutable))
 total=du(R);original_kib=du(S)
 result={'status':'complete' if mb<2850 and classified['classification']['verdict']=='PASS' else 'blocked','historical_feasibility':mb<2850,'operational_outcomes':0,'original':str(S),'restored':str(RESTORE),'candidate':str(TRIAL),'failed_candidate_retained':str(FAILED),'scratch':str(R),'original_allocated_kib':original_kib,'candidate_allocated_kib':kib,'candidate_canary_mb':mb,'candidate_classification':classified,'below2850':mb<2850,'protected_objects':len(protected),'index_cache_tree_roots':len(roots),'candidate_blobs_protected':sorted(protected&candidates),'objects_original':len(allold),'objects_candidate':len(actual),'objects_archived_outside_candidate':len(archived),'archived_by_type':{t:sum(allold[o][1]==t for o in archived) for t in ['commit','tree','blob','tag']},'archive_inventory':str(R/'archived-v2-original-object-inventory.txt'),'generated_objects':generated,'pack_id':packid,'pack_sha256':sha(pack),'pack_index_sha256':sha(packidx),'candidate_snapshot':snap,'original_snapshot':old,'candidate_tree':effect['new_tree'],'master':master,'index_sha256':sha(TRIAL/'index'),'scratch_total_allocated_kib':total,'scratch_plus_immutable_original_allocated_kib':total+original_kib,'current_free_bytes':shutil.disk_usage(R).free,'disk_reclamation_claim':False,'full_fsck_semantics':False,'external_gitlinks_recovered':False,'source_current_identity_claim':False,'source_mutations':False,'checks':checks}
except Exception as ex:result={'status':'blocked','error':str(ex),'error_class':type(ex).__name__,'checks':checks,'scratch':str(R),'source_mutations':False}
finally:
 result['elapsed_combined_data_seconds']=time.time()-start;result['finished_utc']=datetime.datetime.now(datetime.timezone.utc).isoformat();result['commands']=dump('v2-commands',commands);result['all_partial_state_retained']=True
 dump('v2-result',result);print(json.dumps({k:v for k,v in result.items() if k!='checks'}),flush=True)