scripts/tom/we.py

Derived analysis output (scratchpad scripts/tom/we.py).

This is a derived table/summary; the finding's excerpt is usually a paraphrase of its counts, so it may not be highlighted verbatim.

import json,re,csv,collections
S='/tmp/claude-0/-root-swarm-hackathon/a8e4c17a-c034-48ae-9f11-70fc0a8841c9/scratchpad'
self_we=re.compile(r'\b(our|we)\b\s+(own\s+)?(run|cohort|instance|thread|task|scaffold|clock|r\d|q\d|next|current|answer|sequence|timer|deadline|cooldown|have|are|will|answered|expect|confirmed|observed|received|can|found|need|saw|see|predict)',re.I)
coll_we=re.compile(r'\b(we all|all of us|us agents|we agents|fellow agents|every(one|body)|all cohorts|many cohorts|other cohorts|all agents|each (agent|cohort)|agents like us|we collectively|shared (cache|values|effort))\b',re.I)
many=re.compile(r'\b(many|all|several|multiple|other|later|upcoming|future) (cohorts|agents|instances|runs)\b',re.I)
seen=set();c=collections.Counter();ex=collections.defaultdict(list)
rows=[]
for l in open(S+'/added_lines.jsonl'):
    r=json.loads(l); rows.append(('wiki',r['rev'],r['t'],r.get('label'),r['line']))
csv.field_size_limit(10**9)
for r in csv.DictReader(open(S+'/derived/iowa/all_linuxiarz_raw.csv')):
    for ln in r['text'].split('\n'): rows.append(('iowa',r['paste'],r['utc'],r['title'],ln))
for s,p,t,lab,ln in rows:
    k=(s,ln.strip())
    if k in seen: continue
    seen.add(k)
    for n,rx in [('self_we',self_we),('coll_we',coll_we),('many',many)]:
        if rx.search(ln):
            c[(s,n)]+=1
            if len(ex[(s,n)])<12: ex[(s,n)].append((p.split('~')[-1],t[5:16],lab,ln[:180]))
print(c)
for k in [('wiki','coll_we'),('wiki','many'),('iowa','coll_we'),('iowa','many')]:
    print('==',k)
    for e in ex[k]: print(*e)
json.dump({f'{a}|{b}':v for (a,b),v in c.items()},open(S+'/derived/tom/we_counts.json','w'))