import json, urllib.request, urllib.parse, concurrent.futures as cf
UA={"User-Agent":"zennoxa-research/1.0 (public npm census)"}
# reuse the resolved full-tree corpus from the results file
res=json.load(open("/home/paperclip/shared/research-data/npm_install_scripts_results.json"))
# rebuild corpus names: seed + flagged + we need the full 484 — re-resolve quickly from seed direct+tree pkgs we saved? we only saved flagged. Re-derive: use seed + a broad fetch.
SEED=res["seed"]
from collections import deque
seen=set(SEED); q=deque(SEED); CAP=600
_c={}
def man(pkg):
    if pkg in _c: return _c[pkg]
    try:
        u="https://registry.npmjs.org/"+urllib.parse.quote(pkg,safe='@/')
        d=json.load(urllib.request.urlopen(urllib.request.Request(u,headers=UA),timeout=20))
        lat=d.get("dist-tags",{}).get("latest"); m=d.get("versions",{}).get(lat,{}) if lat else {}
    except Exception: m=None
    _c[pkg]=m; return m
while q and len(seen)<=CAP:
    b=[q.popleft() for _ in range(min(64,len(q)))]
    with cf.ThreadPoolExecutor(max_workers=16) as ex:
        for pkg,m in zip(b, ex.map(man,b)):
            if m:
                for dep in (m.get("dependencies") or {}):
                    if dep not in seen and len(seen)<=CAP: seen.add(dep); q.append(dep)
corpus=sorted(seen)
prov=0; fetched=0; nofetch=0; examples=[]
with cf.ThreadPoolExecutor(max_workers=16) as ex:
    for pkg,m in zip(corpus, ex.map(man,corpus)):
        if not m: nofetch+=1; continue
        fetched+=1
        dist=m.get("dist") or {}
        has_prov = "attestations" in dist   # provenance-published versions carry dist.attestations
        if has_prov: prov+=1; 
        if has_prov and len(examples)<12: examples.append(pkg)
print(f"corpus(full tree)={len(corpus)} fetched={fetched}")
print(f"latest version published WITH build provenance (dist.attestations): {prov}  ({100*prov/max(fetched,1):.1f}%)")
print(f"WITHOUT provenance: {fetched-prov}  ({100*(fetched-prov)/max(fetched,1):.1f}%)")
print("examples WITH provenance:", examples[:12])
json.dump({"corpus":len(corpus),"fetched":fetched,"with_provenance":prov,
           "pct_with":round(100*prov/max(fetched,1),1),"pct_without":round(100*(fetched-prov)/max(fetched,1),1)},
          open("/home/paperclip/shared/research-data/npm_provenance_results.json","w"),indent=1)