scripts/tom/focal.py

Derived analysis output (scratchpad scripts/tom/focal.py).

This is a derived table/summary; the finding's excerpt is usually a paraphrase of its counts, so it may not be highlighted verbatim.

import json,re,random,collections,datetime,csv
S='/tmp/claude-0/-root-swarm-hackathon/a8e4c17a-c034-48ae-9f11-70fc0a8841c9/scratchpad'
L=json.load(open(S+'/derived/tom/page_first.json'))
M='(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*'
def stem(n):
    s=re.sub(M+r'\d*','',n)
    s=re.sub(r'\d+','',s)
    s=re.sub(r'(OpenAI|OAI|Research(er)?|Agent|Helper|Scout|Live|Cohort|Test|[A-Z])$','',s)
    s=re.sub(r'^(OpenAI|OAI|Agent|ZZZ?|Tmp)','',s)
    return s
def ts(t): return datetime.datetime.fromisoformat(t.replace('Z','+00:00')).timestamp()
P=[(ts(t),n,lab,ip,stem(n)) for t,n,lab,ip,b in L if t]
P.sort()
def collisions(P,win=120,minlen=12):
    first={}; out=[]
    bystem=collections.defaultdict(list)
    for p in P: bystem[p[4]].append(p)
    for s,ps in bystem.items():
        if len(s)<minlen: continue
        a=ps[0]
        # first two creations of this stem by different labels within win
        for b in ps[1:]:
            if b[0]-a[0]>win: break
            if b[2]!=a[2] and b[1]!=a[1]:
                out.append((a,b)); break
    return out
real=collisions(P)
print('real',len(real))
for a,b in real: print(datetime.datetime.utcfromtimestamp(a[0]).isoformat(),int(b[0]-a[0]),a[1],a[2],a[3],'||',b[1],b[2],b[3])
# null: permute creation times among pages within same day (keeps diurnal density)
byday=collections.defaultdict(list)
for i,p in enumerate(P): byday[int(p[0]//86400)].append(i)
ns=[]
for it in range(200):
    Q=list(P)
    for d,idx in byday.items():
        tt=[P[i][0] for i in idx]; random.shuffle(tt)
        for i,t in zip(idx,tt): Q[i]=(t,)+P[i][1:]
    Q.sort(); ns.append(len(collisions(Q)))
ns.sort(); print('null mean',sum(ns)/len(ns),'p95',ns[189],'max',ns[-1])
json.dump({'real':[[list(a),list(b)] for a,b in real],'null':ns},open(S+'/derived/tom/focal_collisions.json','w'))