`scan_fast.py`

59 lines
import json, os, subprocess, time
SP="/private/tmp/claude-501/-Users-macstudio3/a14d95c4-e5cf-4423-9f72-c05a33fa0445/scratchpad/covmatrix"
REG=os.path.expanduser("~/Projects/dw-fleet-registry/dw-fleet-registry.json")
feats=json.load(open(f"{SP}/features.json"))
reg=json.load(open(REG))
sites=[s for s in reg["sites"] if "standalone-node" in s.get("servingModes",[])]

# flatten features -> (group, name, patterns)
flat=[]
for grp,items in feats.items():
    for name,pats in items.items():
        flat.append((grp,name,pats))

def scan_repo(repo, patterns):
    # one grep per feature (alternation), recursive, case-insensitive, list files only
    alt="|".join(patterns)
    cmd=["grep","-rilE","--include=*.html","--include=*.js","--include=*.mjs","--include=*.css",
         "--include=*.liquid","--include=*.ejs","","--include=*.htm",
         "--exclude-dir=node_modules","--exclude-dir=.git","--exclude-dir=dist","--exclude-dir=build","--exclude-dir=logs","--exclude-dir=tmp","--exclude-dir=data","--exclude-dir=backups",
         "--exclude=*.bak*","--exclude=package-lock.json","-e",alt,repo]
    try:
        r=subprocess.run(cmd,capture_output=True,text=True,timeout=20)
        hits=[l for l in r.stdout.splitlines() if l.strip()]
        return hits
    except subprocess.TimeoutExpired:
        return None  # NOT-MEASURED

matrix={}   # site -> {feature: bool}
evidence={} # site -> {feature: sample file}
site_meta={}
t0=time.time()
for i,s in enumerate(sites):
    slug=s["slug"]
    repo=(s.get("local") or {}).get("repoPath")
    live=(s.get("live") or {}).get("state","?")
    site_meta[slug]={"repo":repo,"live":live,"domain":s.get("domain"),
                     "hasRepo": bool(repo and os.path.isdir(repo))}
    matrix[slug]={}; evidence[slug]={}
    if not (repo and os.path.isdir(repo)):
        for grp,name,pats in flat: matrix[slug][name]=None  # NOT-MEASURED
        continue
    for grp,name,pats in flat:
        hits=scan_repo(repo,pats)
        if hits is None:
            matrix[slug][name]=None
        else:
            matrix[slug][name]=len(hits)>0
            if hits:
                # store a short evidence path (relative)
                ev=hits[0].replace(repo+"/","")
                evidence[slug][name]=ev
    if (i+1)%15==0: print(f"  scanned {i+1}/{len(sites)} ({time.time()-t0:.0f}s)")

out={"generatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime()),
     "axis_features":[{"group":g,"name":n} for g,n,_ in flat],
     "site_meta":site_meta,"matrix":matrix,"evidence":evidence}
json.dump(out,open(f"{SP}/matrix.json","w"),indent=1)
print(f"DONE {len(sites)} sites x {len(flat)} features in {time.time()-t0:.0f}s -> matrix.json")