`scripts/rerun_remap.py`

79 lines
#!/usr/bin/env python3
"""READ-ONLY re-classification of the existing flagged set with the FIXED family-mapper.

Re-runs the audit verdict for every row in out/reextracted.jsonl WITHOUT re-downloading any
image: it re-derives the declared family from the title with the fixed name_family(), recomputes
the image family from the already-cached real_hex via fam(), and re-runs classify(). This proves
the false-positive collapse from the leather-hijack / sea-green bug. $0, no network, no DB/Shopify.

Imports the ACTUAL shipped logic from reextract2 (name_family/fam/classify) so the test can't
drift from the code it validates.
"""
import sys, os, json
from collections import Counter

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SCRIPTS = os.path.join(ROOT, "scripts")
sys.path.insert(0, SCRIPTS)
sys.argv = [sys.argv[0]]                       # so reextract2 import sees no argv[1]
import reextract2 as R                         # noqa: E402  (importable via __main__ guard)
R.NAME_FAMILY = json.load(open(os.path.join(SCRIPTS, "name_family.json")))

def hex_rgb(h):
    h = h.lstrip("#")
    return int(h[0:2],16), int(h[2:4],16), int(h[4:6],16)

def reclassify(row):
    """Return (declaredFamily, imageFamily, verdict, reason) under the FIXED mapper."""
    df = R.name_family(row.get("declared",""))
    hx = row.get("real_hex") or row.get("cached_hex")
    if not hx:
        return df, row.get("imageFamily"), "ERROR", "no-hex"
    imf, s, l = R.fam(*hex_rgb(hx))
    v, reason = R.classify(df, imf, s, l)
    return df, imf, v, reason

def main():
    src = os.path.join(ROOT, "out", "reextracted.jsonl")
    rows = [json.loads(x) for x in open(src) if x.strip()]
    out_path = os.path.join(ROOT, "out", "reextracted_remapped.jsonl")

    before = Counter(); after = Counter()
    cleared = []          # was HIGH, now not HIGH
    new_high = []         # was not HIGH, now HIGH (regression watch)
    fam_changed = []      # declaredFamily changed
    with open(out_path, "w") as f:
        for r in rows:
            ov = r.get("verdict"); odf = r.get("declaredFamily")
            ndf, nimf, nv, nreason = reclassify(r)
            before[ov] += 1; after[nv] += 1
            r2 = dict(r)
            r2["declaredFamily_old"] = odf; r2["verdict_old"] = ov
            r2["declaredFamily"] = ndf; r2["imageFamily"] = nimf
            r2["verdict"] = nv; r2["reason"] = nreason
            f.write(json.dumps(r2) + "\n")
            if odf != ndf: fam_changed.append((r2, odf, ndf))
            if ov == "HIGH" and nv != "HIGH": cleared.append((r2, nv, nreason))
            if ov != "HIGH" and nv == "HIGH": new_high.append((r2, ov, nreason))

    print("=== VERDICT COUNTS (all", len(rows), "re-extracted rows) ===")
    print("BEFORE:", dict(before))
    print("AFTER :", dict(after))
    print(f"\ndeclaredFamily changed by fix: {len(fam_changed)} rows")
    print(f"HIGH -> non-HIGH (false positives cleared): {len(cleared)}")
    print(f"non-HIGH -> HIGH (new flags, regression watch): {len(new_high)}")

    print("\n=== CLEARED (former Tier-1 HIGH now OK/REVIEW) ===")
    for r, nv, why in sorted(cleared, key=lambda x: x[0].get("dwsku","")):
        print(f"  {r.get('dwsku',''):14} {r.get('declared','')[:42]:42} "
              f"fam {r.get('declaredFamily_old','')}->{r.get('declaredFamily','')}"
              f" img={r.get('imageFamily',''):7} -> {nv:6} ({why})")

    if new_high:
        print("\n=== NEW HIGH (introduced by fix — inspect) ===")
        for r, ov, why in new_high:
            print(f"  {r.get('dwsku',''):14} {r.get('declared','')[:42]:42} -> HIGH ({why})")

if __name__ == "__main__":
    main()