"""Develop two-stage compression, then re-evaluate one frozen configuration.""" from __future__ import annotations import argparse from collections import Counter import concurrent.futures import hashlib import json from pathlib import Path import sys sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from app.engine.summary_grounded import generate_grounded, EVIDENCE_PROMPT, COMPRESSION_INSTRUCTION from app.engine.summary_consensus import select_consensus, SHORT_SUMMARY_PROMPT from scripts.build_summary_trial import metrics def read(path): return {r['id']: r for r in (json.loads(l) for l in path.read_text().splitlines() if l.strip())} def save(path, value): path.write_text(json.dumps(value, ensure_ascii=False, indent=2)+'\n') def pick(row, strategy): texts=[c['summary'] for c in row['candidates']] i=select_consensus(texts,word_weight=0.5) if strategy=='consensus5' else (0 if texts[0] and len(texts[0])<=50 else -1) return texts[i] if i>=0 else '' def score(generated, refs, strategy): return {m:{k:sum(metrics(refs[i],pick(row,strategy),m)[k] for i,row in generated.items())/len(generated) for k in ('precision','recall','f1')} for m in ('character','word')} def run(rows, path, client, count): done=read(path) if path.exists() else {} with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool,path.open('a') as out: pending={pool.submit(generate_grounded,client,r['text'],count):i for i,r in rows.items() if i not in done} for f in concurrent.futures.as_completed(pending): i=pending[f];value=dict(f.result(),id=i);done[i]=value out.write(json.dumps(value,ensure_ascii=False)+'\n');out.flush() if len(done)%50==0:print(path.name,len(done),'/',len(rows),flush=True) assert set(done)==set(rows) return done def main(): ap=argparse.ArgumentParser(description=__doc__) ap.add_argument('--trial',type=Path,required=True);ap.add_argument('--previous',type=Path,required=True) ap.add_argument('--out',type=Path,required=True);args=ap.parse_args();args.out.mkdir(parents=True,exist_ok=True) development=read(args.trial/'development.jsonl');previous=read(args.previous/'sources.jsonl') devsources={i:previous[i] for i in development} devrefs={i:r['summary'] for i,r in read(args.previous/'reference.jsonl').items()} from app.engine.structural import extract_lemmas counts=Counter();diagnostics=[] for i,row in sorted(development.items()): hyp=pick(row,'consensus5');ref=devrefs[i];word=metrics(ref,hyp,'word')['f1'] a,b=Counter(extract_lemmas(ref)),Counter(extract_lemmas(hyp)) lf=2*sum((a&b).values())/max(1,sum(a.values())+sum(b.values())) category='above_target' if word>=.65 else ('surface_difference_candidate' if lf>=.7 else ('content_selection_candidate' if lf<.4 else 'mixed_or_uncertain')) counts[category]+=1;diagnostics.append({'id':i,'word_f1':word,'lemma_overlap_f1':lf,'category':category}) save(args.out/'diagnostics.json',{'note':'heuristic triage, not human error labels; lemma overlap is not the scoring metric', 'counts':dict(counts),'rows':diagnostics}) protocol={'reference_changes':False,'score_changes':False,'evaluation_type':'re-evaluation of previously inspected 1000-item test set', 'development_count':len(development),'selection_metric':'word_bigram_f1','evidence_prompt':EVIDENCE_PROMPT, 'summary_prompt':SHORT_SUMMARY_PROMPT,'compression_instruction':COMPRESSION_INSTRUCTION, 'strategies':['single','consensus5'],'model':'gpt-4o','evidence_temperature':0,'summary_temperature':0.3, 'reference_origin':'AI-generated, unreviewed','test_source_sha256':hashlib.sha256((args.trial/'sources.jsonl').read_bytes()).hexdigest()} if (args.out/'protocol.json').exists():assert json.loads((args.out/'protocol.json').read_text())==protocol else:save(args.out/'protocol.json',protocol) from openai import OpenAI from app.core.config import get_settings client=OpenAI(api_key=get_settings().openai_api_key,timeout=120,max_retries=3) devgen=run(devsources,args.out/'development.jsonl',client,5) devscores={s:score(devgen,devrefs,s) for s in ('single','consensus5')} selected=max(devscores,key=lambda s:devscores[s]['word']['f1']) decision={'selected_experimental_strategy':selected,'current_development_scores':score(development,devrefs,'consensus5'), 'experimental_development_scores':devscores,'test_evaluation_reason':'user-requested comparison, not automatic promotion'} if (args.out/'decision.json').exists():assert json.loads((args.out/'decision.json').read_text())==decision else:save(args.out/'decision.json',decision) print(json.dumps(decision),flush=True) rows=read(args.trial/'sources.jsonl');refs={i:r['candidates'][0]['summary'] for i,r in read(args.trial/'reference.jsonl').items()} assert len(rows)==1000 and set(rows)==set(refs) generated=run(rows,args.out/'system.jsonl',client,5 if selected=='consensus5' else 1) newscore=score(generated,refs,selected);oldscore=score(read(args.trial/'system.jsonl'),refs,'consensus5') details=[] for i,row in sorted(rows.items()): hyp=pick(generated[i],selected) details.append({'index':i,'original':row['text'],'reference_summary':refs[i],'summary':hyp, 'rouge_score':metrics(refs[i],hyp,'character')['f1'],'word_rouge_score':metrics(refs[i],hyp,'word')['f1']}) result={'method':'evidence_then_compression_'+selected,'sample_count':1000,'target_rouge':.65, 'average_rouge':newscore['character']['f1'],'average_word_rouge':newscore['word']['f1'], 'success_rate':sum(r['rouge_score']>=.65 for r in details)/1000, 'word_success_rate':sum(r['word_rouge_score']>=.65 for r in details)/1000, 'success_rate_definition':'fraction of individual items with F1 >= 0.65', 'previous_scores':oldscore,'scores':newscore,'word_improved':newscore['word']['f1']>oldscore['word']['f1'], 'evaluation_type':protocol['evaluation_type'],'reference_origin':protocol['reference_origin'], 'certificate_target_achieved':None,'evidence_fallback_count':sum(r['evidence_fallback'] for r in generated.values()), 'empty_count':sum(not r['summary'] for r in details),'data':details} save(args.out/'scorecard.json',result) print(json.dumps({k:v for k,v in result.items() if k!='data'},ensure_ascii=False,indent=2),flush=True) if __name__=='__main__':main()