← back to Ticket System
data/codex-yoloforever/cycle-20260908T1322Z.m4gNE5/ops-reconstruct-runner-v1.py
234 lines
#!/usr/bin/env python3
"""Retained historical Git restoration/reconstruction; no live source writes."""
import hashlib,json,os,re,shutil,stat,subprocess,time,datetime
from pathlib import Path
B=Path('/Users/macstudio3/Projects/ticket-system/data/codex-yoloforever/cycle-20260908T1322Z.m4gNE5')
OLD=Path('/Users/macstudio3/Projects/ticket-system/data/codex-yoloforever/cycle-20260908T1221Z.HzXvuQ')
S=Path('/private/tmp/ops-TK10928-rehearsal.ndMwgA/original.git')
R=Path('/private/tmp/ops-TK10928-reconstruct.xcLYt5')
RESTORE=R/'restored.git';TRIAL=R/'trial.git'
GUARD=Path('/Users/macstudio3/Projects/ticket-system/config/dtd-cost-mode')
ENV={k:v for k,v in os.environ.items() if not k.startswith('GIT_')}
ENV.update(GIT_OPTIONAL_LOCKS='0',GIT_CONFIG_NOSYSTEM='1',GIT_CONFIG_GLOBAL='/dev/null',GIT_NO_REPLACE_OBJECTS='1')
checks=[];commands=[];result={};started=time.monotonic();deadline=started+580
HEX=re.compile(r'^[0-9a-f]{40}$');ZERO='0'*40
def dump(name,x):
p=B/('ops-reconstruct-'+name+'.json')
p.write_text(json.dumps(x,indent=2)+'\n')
return str(p)
def left():
n=deadline-time.monotonic()
if n<=0:raise TimeoutError('580second internal deadline;600second total maximum')
return n
def check(name,yes,detail=None):
checks.append({'check':name,'verdict':'PASS' if yes else 'FAIL','detail':detail})
dump('checks-progress',checks)
if not yes:raise ValueError(name+': '+repr(detail))
def run(argv,inp=None,env=None,out=None,ok=True):
entry={'argv':[str(x) for x in argv],'input_sha256':hashlib.sha256(inp).hexdigest() if inp is not None else None,'stdout_path':str(out) if out else None,'started_epoch':time.time()}
commands.append(entry);dump('commands',commands)
if out:
with Path(out).open('xb') as f:
p=subprocess.run(argv,input=inp,stdout=f,stderr=subprocess.PIPE,env=env or ENV,timeout=left())
data=b''
else:
p=subprocess.run(argv,input=inp,capture_output=True,env=env or ENV,timeout=left());data=p.stdout
entry.update(exit=p.returncode,stderr=p.stderr.decode(errors='replace')[:4000],finished_epoch=time.time());dump('commands',commands)
if ok and p.returncode:raise ValueError('command failed: '+repr(entry))
return data
def git(repo,*args,**kwargs):
return run(['git','--git-dir='+str(repo),'-c','core.commitGraph=false','-c','gc.auto=0',*args],**kwargs)
def sha(path):
h=hashlib.sha256()
with Path(path).open('rb') as f:
for data in iter(lambda:f.read(1024*1024),b''):
left();h.update(data)
return h.hexdigest()
def du(path):return int(run(['du','-sk',str(path)]).split()[0])
def refs(repo):return git(repo,'for-each-ref','--format=%(objectname) %(refname)').decode().splitlines()
def tree(repo,ref):
entries={}
for row in git(repo,'ls-tree','-rz','--full-tree',ref).split(b'\0'):
if row:
meta,path=row.split(b'\t',1);entries[path.decode('utf-8','surrogateescape')]=meta.decode()
return entries
def manifest_copy(target,expected):
target.mkdir()
copied={};receipt=R/(target.stem+'-copy-receipts.jsonl')
with receipt.open('x') as stream:
for rel,old in sorted(expected.items()):
left();source=S/rel;dest=target/rel
ss=source.stat();check_immutable=ss.st_flags & stat.UF_IMMUTABLE
if not check_immutable:raise ValueError('source immutable flag missing: '+rel)
dest.parent.mkdir(parents=True,exist_ok=True)
shutil.copyfile(source,dest)
os.chmod(dest,old['mode']);os.utime(dest,ns=(ss.st_atime_ns,old['mtime_ns']))
ds=dest.stat();actual={'sha256':sha(dest),'bytes':ds.st_size,'mode':stat.S_IMODE(ds.st_mode),'mtime_ns':ds.st_mtime_ns}
if actual!=old:raise ValueError('copy identity mismatch: '+rel)
if ds.st_ino==ss.st_ino and ds.st_dev==ss.st_dev:raise ValueError('shared source inode: '+rel)
if ds.st_nlink!=1:raise ValueError('unexpected hardlink: '+rel)
if ds.st_flags & stat.UF_IMMUTABLE:raise ValueError('destination unexpectedly immutable')
copied[rel]=actual
stream.write(json.dumps({'path':rel,**actual})+'\n');stream.flush()
os.fsync(stream.fileno())
# Preserve every source directory, including empty directories, without flags.
for p in S.rglob('*'):
if p.is_dir():
d=target/p.relative_to(S);d.mkdir(parents=True,exist_ok=True);os.chmod(d,stat.S_IMODE(p.stat().st_mode))
actual_paths={str(p.relative_to(target)) for p in target.rglob('*') if p.is_file()}
check('independent full copy exact '+target.name,actual_paths==set(expected) and copied==expected,{'files':len(copied),'bytes':sum(x['bytes'] for x in copied.values()),'receipt':str(receipt)})
return copied
try:
check('canonical zero-cost guard',GUARD.read_bytes()==b'ZERO_COST_REQUIRED\n')
oldmanifest=json.loads((OLD/'ops-source-git-manifest.json').read_text())
source_rows={r.split()[0]:r for r in (OLD/'ops-original-object-inventory.txt').read_text().splitlines()}
pins=json.loads((B/'ops-object-pins.json').read_text())
prior=json.loads((B/'ops-object-result.json').read_text())
check('reuse previously accepted full object integrity pins',prior['verdict']=='PASS' and prior['remaining']==0 and sha(OLD/'ops-original-object-inventory.txt')==pins['inventory_sha256'] and sha(OLD/'ops-source-git-manifest.json')==pins['manifest_sha256'])
render=json.loads((B/'ops-five-render-proof.json').read_text())
candidates={x['path']:x['git_oid'] for x in render['restores']}
check('exact five candidate set',len(candidates)==5 and render['verdict']=='PASS')
size=du(S);free=shutil.disk_usage(R).free
check('headroom for two copies plus rebuilt store and5GB',free>size*1024*4+5*1024**3,{'free_bytes':free,'source_kib':size})
dump('data-start',{'started_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'started_epoch':time.time(),'upper_bound_seconds':600,'internal_deadline_seconds':580,'max_workers':2,'scratch':str(R),'source':str(S)})
source_refs=refs(S);snap=git(S,'rev-parse','refs/auto-snapshot/latest').decode().strip();master=git(S,'rev-parse','refs/heads/master').decode().strip()
snapshot=tree(S,snap);master_tree=tree(S,master)
check('five original snapshot Git identities',all(snapshot.get(p,'').endswith(o) for p,o in candidates.items()))
restored_manifest=manifest_copy(RESTORE,oldmanifest)
check('restored refs exact',refs(RESTORE)==source_refs,source_refs)
check('restored metadata identity including index refs reflogs pseudorefs',all(restored_manifest[p]==x for p,x in oldmanifest.items() if not p.startswith('objects/')))
dump('full-restore',{'verdict':'PASS','source':str(S),'restored':str(RESTORE),'files':len(restored_manifest),'manifest':str(OLD/'ops-source-git-manifest.json'),'refs':source_refs,'index_sha256':restored_manifest['index']['sha256'],'shared_source_inodes':0,'immutable_source_flags_unchanged':True,'full_fsck_semantics':False})
manifest_copy(TRIAL,oldmanifest)
print('Two full independent copies verified; constructing exact five-path snapshot',flush=True)
indexraw=git(RESTORE,'ls-files','--stage','-z');(R/'protected-index-entries.bin').write_bytes(indexraw)
protected=set();external_index=[]
protected_evidence={}
for line in source_refs:
oid,name=line.split()
if name!='refs/auto-snapshot/latest':protected.add(oid);protected_evidence.setdefault(oid,[]).append(name)
protected.add(git(RESTORE,'rev-parse','HEAD').decode().strip())
for p in (RESTORE/'logs').rglob('*'):
if p.is_file() and str(p.relative_to(RESTORE/'logs'))!='refs/auto-snapshot/latest':
for row in p.read_text().splitlines():
for oid in row.split()[:2]:
if oid!=ZERO:protected.add(oid);protected_evidence.setdefault(oid,[]).append(str(p.relative_to(RESTORE)))
for p in RESTORE.iterdir():
if p.is_file() and re.fullmatch('[A-Z_]+',p.name) and p.name!='COMMIT_EDITMSG':
for row in p.read_text(errors='strict').splitlines():
tokens=row.split()
if tokens and HEX.fullmatch(tokens[0]) and tokens[0]!=ZERO:
protected.add(tokens[0]);protected_evidence.setdefault(tokens[0],[]).append(p.name)
packed=RESTORE/'packed-refs'
if packed.exists():
for row in packed.read_text().splitlines():
if row and row[0] not in '#^':
oid,name=row.split()
if name!='refs/auto-snapshot/latest':protected.add(oid);protected_evidence.setdefault(oid,[]).append('packed:'+name)
for row in indexraw.split(b'\0'):
if row:
meta,path=row.split(b'\t',1);mode,oid,stage=meta.decode().split()
if mode=='160000':external_index.append({'path':path.decode(),'oid':oid})
elif oid!=ZERO:protected.add(oid);protected_evidence.setdefault(oid,[]).append('index:'+path.decode())
check('all protected roots present in pinned source inventory',protected<=set(source_rows),sorted(protected-set(source_rows)))
(R/'protected-roots.txt').write_text('\n'.join(sorted(protected))+'\n')
closure_raw=git(RESTORE,'rev-list','--objects','--no-object-names','--stdin',inp=('\n'.join(sorted(protected))+'\n').encode())
protected_closure=set(closure_raw.decode().splitlines());(R/'protected-closure.txt').write_bytes(closure_raw)
dump('protected',{'roots':len(protected),'closure_objects':len(protected_closure),'sources':protected_evidence,'external_index_gitlinks':external_index,'candidate_blobs_protected':sorted(set(candidates.values())&protected_closure)})
newidx=R/'snapshot.index';ienv=dict(ENV,GIT_INDEX_FILE=str(newidx))
git(TRIAL,'read-tree','--empty',env=ienv)
kept=[]
for path,meta in snapshot.items():
if path not in candidates:
mode,kind,oid=meta.split();kept.append((mode+' '+oid+'\t'+path).encode('utf-8','surrogateescape')+b'\0')
git(TRIAL,'update-index','-z','--index-info',inp=b''.join(kept),env=ienv)
newtree=git(TRIAL,'write-tree',env=ienv).decode().strip()
cenv=dict(ENV,GIT_AUTHOR_NAME='isolated-rehearsal',GIT_AUTHOR_EMAIL='steve@designerwallcoverings.com',GIT_COMMITTER_NAME='isolated-rehearsal',GIT_COMMITTER_EMAIL='steve@designerwallcoverings.com')
newsnap=git(TRIAL,'commit-tree',newtree,'-m','TK-10928 isolated exact five render exclusion trial',env=cenv).decode().strip()
git(TRIAL,'update-ref','refs/auto-snapshot/latest',newsnap,snap)
# The capture has a hidden older packed value for the same snapshot ref.
# Rebase only that snapshot entry too; preserve every non-snapshot line.
packed_preimage=(TRIAL/'packed-refs').read_text() if (TRIAL/'packed-refs').exists() else ''
packed_lines=[];packed_snapshot_changes=[]
for line in packed_preimage.splitlines(keepends=True):
parts=line.strip().split()
if len(parts)==2 and parts[1]=='refs/auto-snapshot/latest':
packed_snapshot_changes.append({'before':parts[0],'after':newsnap,'ref':parts[1]})
packed_lines.append(newsnap+' '+parts[1]+'\n')
else:packed_lines.append(line)
if packed_snapshot_changes:
(R/'candidate-packed-refs.preimage').write_text(packed_preimage)
(TRIAL/'packed-refs').write_text(''.join(packed_lines))
dump('packed-snapshot-effect',packed_snapshot_changes)
nexttree=tree(TRIAL,newsnap)
check('snapshot exact five-path-only leaf effect',nexttree=={p:x for p,x in snapshot.items() if p not in candidates},{'original_entries':len(snapshot),'candidate_entries':len(nexttree),'removed':list(candidates)})
check('master tree/ref and index preserved before packing',tree(TRIAL,master)==master_tree and git(TRIAL,'rev-parse','refs/heads/master').decode().strip()==master and sha(TRIAL/'index')==oldmanifest['index']['sha256'])
retained=set(git(TRIAL,'rev-list','--objects','--no-object-names','--stdin',inp=('\n'.join(sorted(protected|{newsnap}))+'\n').encode()).decode().splitlines())
check('entire protected closure retained',protected_closure<=retained)
retained_file=R/'candidate-retained-object-ids.txt';retained_file.write_text('\n'.join(sorted(retained))+'\n')
prepack_raw=git(TRIAL,'cat-file','--batch-all-objects','--batch-check=%(objectname) %(objecttype) %(objectsize) %(objectsize:disk)')
(R/'trial-prepack-object-inventory.txt').write_bytes(prepack_raw)
generated=set(x.split()[0] for x in prepack_raw.decode().splitlines())-set(source_rows)
newchecks=[]
for oid in sorted(generated):
kind=git(TRIAL,'cat-file','-t',oid).decode().strip();payload=git(TRIAL,'cat-file',kind,oid)
digest=hashlib.sha1((kind+' '+str(len(payload))+'\0').encode()+payload).hexdigest()
check('new object Git hash '+oid,digest==oid)
newchecks.append({'oid':oid,'type':kind,'bytes':len(payload),'hash':digest})
dump('new-object-hashes',newchecks)
archived=sorted(set(source_rows)-retained)
(R/'archived-original-object-inventory.txt').write_text('\n'.join(source_rows[o] for o in archived)+'\n')
oldcommits=set(git(RESTORE,'rev-list',snap).decode().splitlines())
archived_commits=sorted(oldcommits-retained)
dump('effect',{'original_snapshot':snap,'new_snapshot':newsnap,'new_tree':newtree,'new_commit_has_no_parent':True,'old_snapshot_commit_chain_count':len(oldcommits),'old_snapshot_commits_outside_candidate':archived_commits,'original_objects':len(source_rows),'retained_objects':len(retained),'generated_objects':sorted(generated),'archived_original_objects_count':len(archived),'archived_original_objects_path':str(R/'archived-original-object-inventory.txt'),'archived_by_type':{t:sum(source_rows[o].split()[1]==t for o in archived) for t in ['commit','tree','blob','tag']},'candidate_blobs_still_required':sorted(set(candidates.values())&retained),'snapshot_gitlink_entries':sum(x.startswith('160000 ') for x in snapshot.values()),'snapshot_exact_removed_paths':list(candidates),'protected_index_sha256':oldmanifest['index']['sha256'],'scope_warning':'Much broader historical-object archival than five blobs; all originals stay in complete undo stores.'})
stage=R/'new-objects';(stage/'pack').mkdir(parents=True);(stage/'info').mkdir()
packid=git(TRIAL,'pack-objects','--window=0','--threads=2',str(stage/'pack/pack'),inp=retained_file.read_bytes()).decode().strip()
# Preserve superseded stores as siblings; no object deletion or cleanup.
os.rename(TRIAL/'objects',R/'trial-superseded-objects-retained')
os.rename(stage,TRIAL/'objects')
log=TRIAL/'logs/refs/auto-snapshot/latest'
if log.exists():os.rename(log,R/'trial-superseded-snapshot-reflog-retained')
pack=TRIAL/'objects/pack'/('pack-'+packid+'.pack');idx=pack.with_suffix('.idx')
print('Candidate store built; verifying exact pack inventory and Git connectivity',flush=True)
git(TRIAL,'verify-pack','-v',str(idx),out=R/'candidate-verify-pack.txt')
git(TRIAL,'fsck','--connectivity-only','--no-dangling',out=R/'candidate-connectivity.txt')
raw=git(TRIAL,'cat-file','--batch-all-objects','--batch-check=%(objectname) %(objecttype) %(objectsize) %(objectsize:disk)')
(R/'candidate-object-inventory.txt').write_bytes(raw)
actual={x.split()[0]:x.split() for x in raw.decode().splitlines()}
check('actual pack inventory exact retained object set',set(actual)==retained,{'expected':len(retained),'actual':len(actual)})
check('all original object type and size identities preserved',all(actual[o][:3]==source_rows[o].split()[:3] for o in retained&set(source_rows)))
check('self-contained candidate no alternate store',not (TRIAL/'objects/info/alternates').exists())
check('candidate tree exact after self-contained reconstruction',tree(TRIAL,newsnap)==nexttree)
check('protected refs/index/master tree unchanged after reconstruction',[(o,n) for o,n in (x.split() for x in refs(TRIAL)) if n!='refs/auto-snapshot/latest']==[(o,n) for o,n in (x.split() for x in source_refs) if n!='refs/auto-snapshot/latest'] and sha(TRIAL/'index')==oldmanifest['index']['sha256'] and tree(TRIAL,master)==master_tree)
protected_metadata=[p for p in oldmanifest if not p.startswith('objects/') and p not in ['refs/auto-snapshot/latest','logs/refs/auto-snapshot/latest','packed-refs']]
check('all protected metadata file bytes and modes unchanged',all((TRIAL/p).is_file() and sha(TRIAL/p)==oldmanifest[p]['sha256'] and stat.S_IMODE((TRIAL/p).stat().st_mode)==oldmanifest[p]['mode'] for p in protected_metadata))
non_snapshot=lambda text:[x for x in text.splitlines(keepends=True) if not x.rstrip().endswith(' refs/auto-snapshot/latest')]
check('non-snapshot packed-ref lines byte-identical',non_snapshot((TRIAL/'packed-refs').read_text())==non_snapshot(packed_preimage))
kib=du(TRIAL);mb=(kib+512)//1024
script="import {classifyRepo,KNOWN_LEGIT} from '/Users/macstudio3/.claude/skills/dw-backup-canary/lib.mjs';let mb="+str(mb)+";console.log(JSON.stringify({mb,known:KNOWN_LEGIT['(projects-root)'],classification:classifyRepo({repo:'(projects-root)',mb,remote:null,upstream:null,ahead:0})}))"
classified=json.loads(run(['node','--input-type=module','-e',script]))
check('actual canary root floor unchanged3000',classified['known']['floor_mb']==3000)
source_entries=[S,*S.rglob('*')]
check('all original immutable flags remain set',all(p.stat().st_flags & stat.UF_IMMUTABLE for p in source_entries))
total=du(R);afterfree=shutil.disk_usage(R).free
result={'status':'complete' if mb<2850 and classified['classification']['verdict']=='PASS' else 'blocked','scope':'historical copy-only feasibility;zero operational outcomes','original':str(S),'restored':str(RESTORE),'trial':str(TRIAL),'scratch':str(R),'original_allocated_kib':size,'candidate_allocated_kib':kib,'candidate_canary_mb':mb,'candidate_classification':classified,'target_below2850':mb<2850,'scratch_total_allocated_kib':total,'scratch_plus_original_allocated_kib':total+size,'free_bytes_before':free,'free_bytes_after':afterfree,'observed_filesystem_free_delta_bytes':afterfree-free,'physical_disk_reclaimed_claim':False,'pack_id':packid,'pack_sha256':sha(pack),'pack_index_sha256':sha(idx),'restored_files':len(oldmanifest),'original_objects':len(source_rows),'candidate_objects':len(actual),'archived_original_objects':len(archived),'protected_objects':len(protected_closure),'candidate_blobs_protected':sorted(set(candidates.values())&protected_closure),'generated_object_ids':sorted(generated),'original_snapshot':snap,'candidate_snapshot':newsnap,'candidate_tree':newtree,'master':master,'index_sha256':oldmanifest['index']['sha256'],'snapshot_entries_before':len(snapshot),'snapshot_entries_after':len(nexttree),'full_fsck_semantics':False,'external_gitlinks_validated':False,'source_current_identity_claim':False,'source_mutations':False,'no_cleanup':True,'checks':checks}
except Exception as ex:
result={'status':'blocked','error':str(ex),'error_class':type(ex).__name__,'scratch':str(R),'retained':'All completed/partial copies and stores remain retained','source_mutations':False,'checks':checks}
finally:
result['elapsed_data_seconds']=time.monotonic()-started
result['finished_utc']=datetime.datetime.now(datetime.timezone.utc).isoformat()
result['commands']=dump('commands',commands)
dump('result',result)
print(json.dumps({k:v for k,v in result.items() if k!='checks'}),flush=True)