78 lines
2.7 KiB
Python
78 lines
2.7 KiB
Python
"""지표 집계 + 학습곡선 (H5, 계획서 G 지표)."""
|
|
|
|
import math
|
|
from dataclasses import dataclass
|
|
from typing import List
|
|
|
|
from eval_harness.simulator import EpisodeResult
|
|
|
|
|
|
@dataclass
|
|
class Aggregate:
|
|
n: int
|
|
success_rate: float
|
|
mean_settled_ratio: float # 평균 타결가 / 목표가
|
|
mean_turns: float
|
|
mean_reward: float
|
|
reward_ci95: float # 평균보상 95% 신뢰구간 반폭
|
|
|
|
def as_row(self) -> dict:
|
|
return {
|
|
"n": self.n,
|
|
"success_rate": round(self.success_rate, 3),
|
|
"mean_settled_ratio": round(self.mean_settled_ratio, 3),
|
|
"mean_turns": round(self.mean_turns, 2),
|
|
"mean_reward": round(self.mean_reward, 4),
|
|
"reward_ci95": round(self.reward_ci95, 4),
|
|
}
|
|
|
|
|
|
def aggregate(results: List[EpisodeResult]) -> Aggregate:
|
|
n = len(results)
|
|
if n == 0:
|
|
return Aggregate(0, 0, 0, 0, 0, 0)
|
|
succ = sum(1 for r in results if r.success) / n
|
|
settled = sum(r.settled_ratio for r in results) / n
|
|
turns = sum(r.turns for r in results) / n
|
|
rewards = [r.total_reward for r in results]
|
|
mean_r = sum(rewards) / n
|
|
var = sum((x - mean_r) ** 2 for x in rewards) / n if n > 1 else 0.0
|
|
ci = 1.96 * math.sqrt(var / n) if n > 1 else 0.0
|
|
return Aggregate(n, succ, settled, turns, mean_r, ci)
|
|
|
|
|
|
def learning_curve(results: List[EpisodeResult], buckets: int = 10) -> List[float]:
|
|
"""에피소드를 buckets 구간으로 나눠 구간별 평균 보상 (우상향이면 학습)."""
|
|
n = len(results)
|
|
if n == 0:
|
|
return []
|
|
size = max(1, n // buckets)
|
|
curve = []
|
|
for i in range(0, n, size):
|
|
chunk = results[i:i + size]
|
|
curve.append(round(sum(r.total_reward for r in chunk) / len(chunk), 4))
|
|
return curve
|
|
|
|
|
|
def good_card_hit_rate(first_actions: List[int], good_actions: List[int], last_frac: float = 0.3) -> float:
|
|
"""후반 구간에서 첫 카드가 '좋은 카드'였던 비율 (학습 수렴 지표)."""
|
|
if not first_actions:
|
|
return 0.0
|
|
tail = first_actions[max(0, int(len(first_actions) * (1 - last_frac))):]
|
|
good = set(good_actions)
|
|
return sum(1 for a in tail if a in good) / len(tail)
|
|
|
|
|
|
def hit_curve(first_actions: List[int], good_actions: List[int], buckets: int = 10) -> List[float]:
|
|
"""구간별 '좋은 카드 선택' 비율 (학습 전/후 개선곡선 — 세일즈용)."""
|
|
n = len(first_actions)
|
|
if n == 0:
|
|
return []
|
|
good = set(good_actions)
|
|
size = max(1, n // buckets)
|
|
curve = []
|
|
for i in range(0, n, size):
|
|
chunk = first_actions[i:i + size]
|
|
curve.append(round(sum(1 for a in chunk if a in good) / len(chunk), 3))
|
|
return curve
|