"""Offline evidence and F1 verification for the two-stage experiment."""
from __future__ import annotations
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
import re
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app.engine.summary_consensus import select_consensus


def rows(path):
    items=[json.loads(l) for l in path.read_text().splitlines() if l.strip()]
    assert len(items)==1000 and {r['id'] for r in items}==set(range(1,1001))
    return {r['id']:r for r in items}


def main():
    ap=argparse.ArgumentParser(description=__doc__)
    ap.add_argument('--trial',type=Path,required=True)
    ap.add_argument('--experiment',type=Path,required=True)
    a=ap.parse_args();sources=rows(a.trial/'sources.jsonl');refs=rows(a.trial/'reference.jsonl')
    outputs=rows(a.experiment/'system.jsonl');result=json.loads((a.experiment/'scorecard.json').read_text())
    protocol=json.loads((a.experiment/'protocol.json').read_text())
    assert hashlib.sha256((a.trial/'sources.jsonl').read_bytes()).hexdigest()==protocol['test_source_sha256']
    strategy=json.loads((a.experiment/'decision.json').read_text())['selected_experimental_strategy']
    details={r['index']:r for r in result['data']};assert len(details)==len(result['data'])==1000
    total={'character':0.,'word':0.}
    for rid in range(1,1001):
        row=outputs[rid];text=sources[rid]['text']
        assert all(e in text and 1<=len(e)<=120 for e in row['evidence'])
        assert len(row['evidence'])<=3
        summaries=[c['summary'] for c in row['candidates']]
        i=select_consensus(summaries,.5) if strategy=='consensus5' else (0 if summaries[0] and len(summaries[0])<=50 else -1)
        hyp=summaries[i] if i>=0 else '';ref=refs[rid]['candidates'][0]['summary']
        assert details[rid]['summary']==hyp and details[rid]['reference_summary']==ref and details[rid]['original']==text
        for mode in total:
            def grams(t):
                t=list(re.sub(r'\s+','',t)) if mode=='character' else t.split()
                return Counter(zip(t,t[1:]))
            x,y=grams(ref),grams(hyp);denom=sum(x.values())+sum(y.values())
            f1=2*sum(min(n,y[g]) for g,n in x.items())/denom if denom else 0
            assert abs(f1-details[rid]['rouge_score' if mode=='character' else 'word_rouge_score'])<1e-12
            total[mode]+=f1/1000
    for mode,avg in total.items():
        assert abs(avg-result['scores'][mode]['f1'])<1e-12
        print(mode,'F1=%.6f%%'%(100*avg))
    print('PASS: 1000 IDs, unchanged sources/references, literal evidence, selected outputs, per-item and mean F1')


if __name__=='__main__':main()