diff --git a/agent/artifacts/card_embeddings.npz b/agent/artifacts/card_embeddings.npz new file mode 100644 index 0000000..620b6e9 Binary files /dev/null and b/agent/artifacts/card_embeddings.npz differ diff --git a/agent/artifacts/dqn_serving.npz b/agent/artifacts/dqn_serving.npz new file mode 100644 index 0000000..a740cf6 Binary files /dev/null and b/agent/artifacts/dqn_serving.npz differ diff --git a/agent/artifacts/feature_dqn_ktcommerce.pt b/agent/artifacts/feature_dqn_ktcommerce.pt new file mode 100644 index 0000000..803532f Binary files /dev/null and b/agent/artifacts/feature_dqn_ktcommerce.pt differ diff --git a/agent/artifacts/retrain_report.json b/agent/artifacts/retrain_report.json new file mode 100644 index 0000000..7e057ed --- /dev/null +++ b/agent/artifacts/retrain_report.json @@ -0,0 +1,14 @@ +{ + "rows": 25, + "episodes": 4, + "skipped": { + "종료행/카드턴 없음(미완결 세션)": 8 + }, + "min_episodes": 1, + "deployed": false, + "ope_candidate": 0.9029104414200676, + "ope_candidate_ess": 1.0, + "ope_current": 0.9029104414200676, + "ope_current_ess": 1.0, + "result": "gate_failed" +} \ No newline at end of file diff --git a/agent/eval_harness/feature_buyer.py b/agent/eval_harness/feature_buyer.py new file mode 100644 index 0000000..5d25733 --- /dev/null +++ b/agent/eval_harness/feature_buyer.py @@ -0,0 +1,104 @@ +"""FeatureBuyer — 카드 '내용(전략)'과 협력사 '프로필'에 반응하는 시뮬 협력사 (Phase 2·3). + +효과를 2축으로 분리한다(성향 조건화가 의미를 가지려면 트레이드오프가 필요): + - 양보력(concession power): 이 카드가 가격을 얼마나 끌어내리는가 + - 수락력(accept power) : 이 카드가 합의(수락) 확률을 얼마나 높이는가 + +전략별 기본 프로필(트레이드오프): + 경쟁(1): 양보력↑↑ 수락력↓ — 세게 깎지만 결렬 위험 + 수용(2): 양보력↓ 수락력↑ + 고수(3): 양보력·수락력 중간 + 협력(4): 양보력↓ 수락력↑↑ — 잘 성사되지만 덜 깎임 + +여기에 협력사 세그먼트 적합도(AFFINITY)가 곱해진다: 전략이 그 협력사에 안 맞으면 둘 다 죽는다. + 소형·경쟁多 → 경쟁압박이 잘 먹힘 / 대형·단독 → 협력이 잘 먹힘(압박 역효과) + +→ '가격 중시' 고객사는 경쟁 카드(많이 깎음, 결렬 감수), '성사 중시' 고객사는 협력 카드가 정답이 +되는 구조. 에이전트는 카드 특징 + 협력사 특징 + 고객사 성향으로 이를 학습해야 한다. +""" + +from dataclasses import dataclass +from typing import Dict, Tuple + +import numpy as np + +from eval_harness.buyer import BuyerResponse, Scenario + +# strategy_type: 1=경쟁, 2=수용, 3=고수, 4=협력 (card.nego_cards) +# 전략별 (양보력, 수락력) 기본 프로필 — 트레이드오프의 원천 +STRATEGY_PROFILE: Dict[int, Tuple[float, float]] = { + 1: (0.90, 0.25), # 경쟁: 세게 깎지만 성사 어려움 + 2: (0.35, 0.70), # 수용 + 3: (0.55, 0.50), # 고수: 중간 + 4: (0.30, 0.90), # 협력: 잘 성사되지만 덜 깎임 +} + +# 세그먼트별 전략 적합도 m ∈ [0,1] — 전략이 그 협력사에 얼마나 '먹히는가' +AFFINITY: Dict[Tuple[str, str], Dict[int, float]] = { + ("small", "multi"): {1: 0.90, 2: 0.45, 3: 0.60, 4: 0.40}, # 소형·경쟁多 → 경쟁압박 + ("small", "single"): {1: 0.35, 2: 0.60, 3: 0.80, 4: 0.55}, # 소형·단독 → 고수/논리 + ("big", "multi"): {1: 0.65, 2: 0.50, 3: 0.70, 4: 0.60}, + ("big", "single"): {1: 0.20, 2: 0.70, 3: 0.50, 4: 0.90}, # 대형·단독 → 협력 (압박 역효과) +} +REVENUE_BIG = 50_000_000 # state config 'high' 경계와 정합 + + +@dataclass +class SupplierProfile: + """협력사 프로필 — 에피소드마다 달라지는 협상 상대. snapshot 필드와 정합.""" + + revenue_amount: float + partner_count: int # 이 품목의 대안 협력사 수 (BATNA. 1:1 채팅이어도 다양) + distribution_code: str + + @property + def segment(self) -> Tuple[str, str]: + size = "big" if self.revenue_amount > REVENUE_BIG else "small" + comp = "multi" if self.partner_count >= 2 else "single" + return (size, comp) + + +def sample_supplier(rng: np.random.Generator) -> SupplierProfile: + """무작위 협력사 생성 (학습 데이터 다양성).""" + return SupplierProfile( + revenue_amount=float(rng.choice([5_000_000, 20_000_000, 80_000_000, 200_000_000])), + partner_count=int(rng.choice([1, 1, 2, 3])), # 단독 비중 높게 + distribution_code=str(rng.choice(["A", "B", "C"])), + ) + + +class FeatureBuyer: + """전략 프로필 x 세그먼트 적합도 기반 협력사 모델. (양보력, 수락력) 2축.""" + + def __init__(self, supplier: SupplierProfile, card_strategy: Dict[str, int], seed: int = 0, + accept_base: float = 0.08, max_turns: int = 5, jitter: float = 0.05): + self.supplier = supplier + self.card_strategy = card_strategy # {card_number: strategy_type} + self.rng = np.random.default_rng(seed) + self.accept_base = accept_base + self.max_turns = max_turns + self.jitter = jitter + # 숨은 하한가(reservation): 앵커의 94~110%. 앵커보다 높으면(약 60%) 가격만으로는 + # 타결 불가 → 수락을 받아내야 함 → 수락력 낮은(경쟁) 카드에 진짜 결렬 위험이 생긴다. + self.floor_ratio = float(self.rng.uniform(0.94, 1.10)) + + def powers(self, card_number: str) -> Tuple[float, float]: + """숨은 (양보력, 수락력). 전략 프로필 × 세그먼트 적합도 + 카드별 결정론적 지터.""" + strat = self.card_strategy.get(card_number, 3) + conc_base, acc_base = STRATEGY_PROFILE.get(strat, (0.5, 0.5)) + m = AFFINITY[self.supplier.segment].get(strat, 0.5) + scale = 0.35 + 0.85 * m # 적합도: 안 맞으면 둘 다 죽음 (0.35~1.2) + j = (hash(card_number) % 1000 / 1000.0 - 0.5) * 2 * self.jitter + c_pow = float(np.clip(conc_base * scale + j, 0.02, 0.98)) + a_pow = float(np.clip(acc_base * scale + j, 0.02, 0.98)) + return c_pow, a_pow + + def respond(self, card_number: str, scenario: Scenario, turn: int, current_price: float) -> BuyerResponse: + c_pow, a_pow = self.powers(card_number) + floor = scenario.anchor_price * self.floor_ratio # 숨은 하한가 (앵커 이하 보장 없음) + concession = (current_price - floor) * (0.10 + 0.55 * c_pow) + new_price = max(floor, current_price - concession) + p_accept = min(0.97, self.accept_base + 0.80 * a_pow + 0.05 * (turn - 1)) + accept = bool(self.rng.random() < p_accept) + walked = (not accept) and (turn >= self.max_turns) + return BuyerResponse(accept=accept, new_price=new_price, walked=walked) diff --git a/agent/negotiation/policies/feature_dqn_policy.py b/agent/negotiation/policies/feature_dqn_policy.py new file mode 100644 index 0000000..6f329b0 --- /dev/null +++ b/agent/negotiation/policies/feature_dqn_policy.py @@ -0,0 +1,132 @@ +"""FeatureDQNPolicy — action-as-feature DQN (Phase 2·3). + +고정 슬롯 Q(s)→[11개] 대신 ScoreNet(상태벡터 + 카드임베딩) → 스칼라 점수. +결정 시 가용 카드 풀을 순회 채점해 argmax → 카드 추가/삭제/새 카드(zero-shot)에 구조 변화 없음. +협력사 특징은 상태벡터에 포함(feature_builder) → '협력사를 입력으로' 달성. + +가변 행동 학습: replay 에 다음 상태의 '가용 카드 임베딩들'을 함께 저장, + target = r + γ · max_{c'∈next_avail} Q(s', c') · (1-done) +""" + +import math +import random +from collections import deque +from typing import Dict, List, Optional, Tuple + +import numpy as np +import torch +import torch.nn as nn + + +class ScoreNet(nn.Module): + """(상태 + 카드임베딩) → 스칼라 점수.""" + + def __init__(self, state_dim: int, card_dim: int, hidden: int = 128): + super().__init__() + self.net = nn.Sequential( + nn.Linear(state_dim + card_dim, hidden), nn.ReLU(), + nn.Linear(hidden, hidden), nn.ReLU(), + nn.Linear(hidden, 1), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: # x: [B, state+card] + return self.net(x).squeeze(-1) # [B] + + +class FeatureDQNPolicy: + name = "feature_dqn" + + def __init__(self, state_dim: int, card_dim: int, device: str = "cpu", + lr: float = 1e-3, gamma: float = 0.95, hidden: int = 128, + eps_start: float = 1.0, eps_end: float = 0.05, eps_decay: int = 6000, + buffer_size: int = 50_000, batch_size: int = 64, target_sync: int = 500): + self.device = device + self.gamma = gamma + self.batch_size = batch_size + self.target_sync = target_sync + self.q = ScoreNet(state_dim, card_dim, hidden).to(device) + self.tgt = ScoreNet(state_dim, card_dim, hidden).to(device) + self.tgt.load_state_dict(self.q.state_dict()) + self.opt = torch.optim.Adam(self.q.parameters(), lr=lr) + self.buf: deque = deque(maxlen=buffer_size) + self.eps_start, self.eps_end, self.eps_decay = eps_start, eps_end, eps_decay + self.steps = 0 + self.greedy = False # 평가 모드(탐색 끔) + + # ---- 탐색 스케줄 ---------------------------------------------------- + def eps(self) -> float: + if self.greedy: + return 0.0 + return self.eps_end + (self.eps_start - self.eps_end) * math.exp(-self.steps / self.eps_decay) + + # ---- 채점/선택 ------------------------------------------------------- + def scores(self, state_feat: np.ndarray, card_embs: np.ndarray) -> np.ndarray: + """가용 카드 K개 일괄 채점. card_embs: [K, card_dim] → [K].""" + k = card_embs.shape[0] + x = np.concatenate([np.repeat(state_feat[None, :], k, axis=0), card_embs], axis=1) + with torch.no_grad(): + return self.q(torch.tensor(x, device=self.device)).cpu().numpy() + + def select(self, state_feat: np.ndarray, card_embs: np.ndarray) -> Tuple[int, float, float]: + """(선택 인덱스, propensity, 선택 점수). 인덱스는 card_embs 행 기준.""" + k = card_embs.shape[0] + sc = self.scores(state_feat, card_embs) + e = self.eps() + if random.random() < e: + i = random.randrange(k) + prop = e / k + else: + i = int(sc.argmax()) + prop = (1.0 - e) + e / k + return i, prop, float(sc[i]) + + # ---- 경험/학습 ------------------------------------------------------- + def remember(self, state_feat: np.ndarray, card_emb: np.ndarray, reward: float, + next_state_feat: Optional[np.ndarray], next_card_embs: Optional[np.ndarray], + done: bool): + self.buf.append((state_feat, card_emb, reward, next_state_feat, next_card_embs, done)) + + def train_step(self) -> Optional[float]: + if len(self.buf) < self.batch_size: + return None + batch = random.sample(self.buf, self.batch_size) + + # Q(s, a_chosen) + xs = np.stack([np.concatenate([s, c]) for s, c, *_ in batch]) + q_sa = self.q(torch.tensor(xs, device=self.device)) + + # target = r + γ·max_{c'} Q_tgt(s', c') — 가변 후보라 후보 전체를 한 번에 forward 후 세그먼트 max + rewards = torch.tensor([b[2] for b in batch], device=self.device, dtype=torch.float32) + dones = torch.tensor([float(b[5]) for b in batch], device=self.device) + next_rows, owner = [], [] + for bi, (_, _, _, s2, cands, done) in enumerate(batch): + if done or s2 is None or cands is None or len(cands) == 0: + continue + for c in cands: + next_rows.append(np.concatenate([s2, c])) + owner.append(bi) + q_next_max = torch.zeros(self.batch_size, device=self.device) + if next_rows: + with torch.no_grad(): + q_all = self.tgt(torch.tensor(np.stack(next_rows), device=self.device)) + owner_t = torch.tensor(owner, device=self.device) + q_next_max = q_next_max.index_reduce_(0, owner_t, q_all, "amax", include_self=False) + target = rewards + self.gamma * q_next_max * (1.0 - dones) + + loss = nn.functional.smooth_l1_loss(q_sa, target) + self.opt.zero_grad() + loss.backward() + self.opt.step() + self.steps += 1 + if self.steps % self.target_sync == 0: + self.tgt.load_state_dict(self.q.state_dict()) + return float(loss) + + # ---- 저장/로드 ------------------------------------------------------- + def save(self, path: str): + torch.save(self.q.state_dict(), path) + + def load(self, path: str): + sd = torch.load(path, map_location=self.device) + self.q.load_state_dict(sd) + self.tgt.load_state_dict(sd) diff --git a/agent/negotiation/policy/dqn_store.py b/agent/negotiation/policy/dqn_store.py new file mode 100644 index 0000000..be90da1 --- /dev/null +++ b/agent/negotiation/policy/dqn_store.py @@ -0,0 +1,118 @@ +"""DQNServingStore — action-as-feature DQN 서빙 (선택 전용, 학습 없음). + +tools/export_dqn_serving.py 가 만든 dqn_serving.npz(ScoreNet 가중치 + 카드특징 392차원)를 +numpy 로 추론한다 — 서빙 컨테이너에 PyTorch 불필요. + +역할 분담(계획서 H 트랙으로 가기 전 파일럿): +- 카드 '선택'만 DQN(greedy). Q-table 학습/영속/experience_logs 로깅 경로는 기존 그대로 유지 + (Q-learning 은 오프폴리시라 DQN 이 고른 행동으로 갱신해도 유효, 로그는 DQN 오프라인 재학습 재료). +- 폴백: 플래그 꺼짐 / 번들 없음 / 가용 카드 전부 특징 미보유(신규 카드) → None 반환, + 호출부(ChatService)가 기존 UCB Q-table 선택으로 진행한다. + +활성화: 환경변수 DQN_SERVING=1 (docker-compose agent environment). +신규 카드 주의: 번들에 없는 카드번호는 후보에서 제외된다 — 카드 추가 시 +tools/build_card_embeddings.py → tools/export_dqn_serving.py 재실행 후 재배포 필요. +""" + +import os +from typing import List, Optional + +import numpy as np + +from common.logger import LOG +from negotiation.policies.base import ActionDecision, PolicyContext +from negotiation.qtable.domain.service.feature_builder import ( + build_state_features, build_tenant_features) + +_HERE = os.path.dirname(os.path.abspath(__file__)) +BUNDLE_PATH = os.path.join(_HERE, "..", "..", "artifacts", "dqn_serving.npz") + + +class _Bundle: + def __init__(self, z): + self.W0, self.b0 = z["W0"], z["b0"] + self.W1, self.b1 = z["W1"], z["b1"] + self.W2, self.b2 = z["W2"], z["b2"] + self.state_dim = int(z["state_dim"]) + self.card_feats = {str(n): z["card_feats"][i] + for i, n in enumerate(z["card_numbers"])} + + def scores(self, state_feat: np.ndarray, card_feats: np.ndarray) -> np.ndarray: + """가용 카드 K개 일괄 채점: [K, state+card] → [K].""" + k = card_feats.shape[0] + x = np.concatenate([np.repeat(state_feat[None, :], k, axis=0), card_feats], axis=1) + h = np.maximum(x @ self.W0.T + self.b0, 0.0) + h = np.maximum(h @ self.W1.T + self.b1, 0.0) + return (h @ self.W2.T + self.b2).squeeze(-1) + + +class DQNServingPolicy: + """UCBQTablePolicy.select 와 동일한 PolicyContext → ActionDecision 계약(선택 전용).""" + + name = "feature_dqn" + _EPS = 0.1 # propensity 근사용 ε (UCB 정책과 동일 관례 — OPE 지지 확보용, 선택은 greedy) + + def __init__(self, bundle: _Bundle, engine): # engine: tenancy.registry.TenantEngine + self._bundle = bundle + self._mapper = engine.mapper + self._tenant_feat = build_tenant_features(engine.config.reward) + + def _available(self, ctx: PolicyContext) -> List[int]: + # UCBQTablePolicy._available 과 동일 규칙 (마스크 → used 제외 → 소진 시 전체 허용) + if ctx.available_mask is not None: + avail = [a for a in range(ctx.action_space_size) if ctx.available_mask[a]] + else: + used = ctx.episode.used_action_ids if ctx.episode else set() + avail = [a for a in range(ctx.action_space_size) if a not in used] + return avail or list(range(ctx.action_space_size)) + + def select(self, ctx: PolicyContext) -> Optional[ActionDecision]: + """카드특징이 있는 가용 카드가 없으면 None → 호출부가 Q-table 로 폴백.""" + candidates = [] # (action_id, card_feat) + for a in self._available(ctx): + num = self._mapper.get_card_id(a) + feat = self._bundle.card_feats.get(num) if num else None + if feat is not None: + candidates.append((a, feat)) + if not candidates: + return None + state_feat = np.concatenate([build_state_features(ctx.snapshot), self._tenant_feat]) + if state_feat.shape[0] != self._bundle.state_dim: + LOG.e_no_callstack( + f"[DQNServing] state_dim 불일치: {state_feat.shape[0]} != {self._bundle.state_dim}") + return None + sc = self._bundle.scores(state_feat, np.stack([f for _, f in candidates])) + i = int(sc.argmax()) + n = len(candidates) + return ActionDecision( + action_id=candidates[i][0], + propensity=(1.0 - self._EPS) + self._EPS / n, + q_value=float(sc[i]), + ucb_score=float(sc[i]), + available_actions=[a for a, _ in candidates], + ) + + +class DQNServingStore: + """번들 lazy 로드 + 캐시. 비활성/부재 시 None (호출부 Q-table 폴백).""" + + _bundle: Optional[_Bundle] = None + _load_failed = False + + @classmethod + def enabled(cls) -> bool: + return os.getenv("DQN_SERVING", "0").lower() in ("1", "true", "yes") + + @classmethod + def policy_for(cls, engine) -> Optional[DQNServingPolicy]: + if not cls.enabled() or cls._load_failed: + return None + if cls._bundle is None: + try: + cls._bundle = _Bundle(np.load(BUNDLE_PATH, allow_pickle=False)) + LOG.i(f"[DQNServing] 번들 로드 완료: 카드 {len(cls._bundle.card_feats)}장") + except Exception as ex: + cls._load_failed = True # 요청마다 재시도하지 않음 + LOG.e_no_callstack(f"[DQNServing] 번들 로드 실패 → Q-table 폴백: {ex}") + return None + return DQNServingPolicy(cls._bundle, engine) diff --git a/agent/negotiation/qtable/domain/service/feature_builder.py b/agent/negotiation/qtable/domain/service/feature_builder.py new file mode 100644 index 0000000..d458986 --- /dev/null +++ b/agent/negotiation/qtable/domain/service/feature_builder.py @@ -0,0 +1,44 @@ +"""build_state_features — snapshot(raw 연속값) → 실수 벡터 (DQN/action-as-feature 용). + +state_calculator.build_state(이산화)와 대비되는 연속 표현. 이산화(등급/162칸)를 하지 않고 +정규화된 raw 값을 그대로 벡터로 내보낸다. 협력사 특징(매출·경쟁사수·유통)이 벡터에 포함되므로 +'협력사를 입력으로'(Phase 3)가 자연스럽게 달성된다. +""" + +import numpy as np + +from negotiation.qtable.domain.model.snapshot import NegotiationSnapshot + +DIST_CLASSES = ("A", "B", "C") +STATE_FEATURE_DIM = 9 # build_state_features 벡터 길이. feature 추가 시 갱신. +TENANT_FEATURE_DIM = 5 # build_tenant_features 벡터 길이. + + +def build_tenant_features(reward_cfg) -> np.ndarray: + """고객사 '성향'을 ID 가 아니라 보상 설정값(내용)으로 벡터화 (Phase 3 고객사 조건화). + + 새 고객사도 tenant.yaml 의 reward 설정만 있으면 즉시 조건화된다 (cold-start 없음). + """ + return np.array([ + reward_cfg.max_weight, # 가격 중시 정도 (W↑ = 가격보상 비중↑) + reward_cfg.success_reward / 2.0, # 성사를 얼마나 크게 치는가 + -reward_cfg.failure_penalty / 2.0, # 결렬을 얼마나 무서워하는가 + reward_cfg.penalty_lambda * 20.0, # 속도 성향 (오래 끌수록 벌점) + reward_cfg.beta, # 앵커 초과달성 보너스 성향 + ], dtype=np.float32) + + +def build_state_features(s: NegotiationSnapshot) -> np.ndarray: + """정규화된 연속 상태 벡터. 등급화 없음 — 990원과 850원이 구별된다.""" + dist_onehot = [1.0 if s.distribution_code == c else 0.0 for c in DIST_CLASSES] + anchor = max(s.anchor_price, 1.0) + target = max(s.target_price, 1.0) + return np.array([ + min(s.revenue_amount, 5e8) / 5e8, # 협력사 매출 (0~1) + *dist_onehot, # 유통 A/B/C + min(s.partner_count, 5) / 5.0, # 대안 협력사 수 (BATNA) + float(np.clip(s.acceptance_ratio, 0.0, 1.0)), # 수용률 + float(np.clip((s.input_price - anchor) / anchor, -1.0, 2.0)), # 앵커 대비 격차 (연속!) + float(np.clip((target - s.input_price) / target, -2.0, 1.0)), # 목표 대비 여유 + min(s.round_number, 10) / 10.0, # 라운드 + ], dtype=np.float32) diff --git a/agent/tools/build_card_embeddings.py b/agent/tools/build_card_embeddings.py new file mode 100644 index 0000000..61d8c40 --- /dev/null +++ b/agent/tools/build_card_embeddings.py @@ -0,0 +1,63 @@ +"""카드 스크립트 → 임베딩 캐시 생성 (action-as-feature 준비, 1회 실행). + +card.nego_cards(11장)의 name+script 를 문장 임베딩으로 변환해 artifacts/card_embeddings.npz 에 저장. +새 카드가 추가되면 이 스크립트를 다시 돌리면 된다(그 카드만 임베딩돼 캐시에 합류). + +실행: + APP_ENV=local python -m tools.build_card_embeddings +출력: + artifacts/card_embeddings.npz (numbers, names, strategy, tone, embeddings[N,384]) +""" + +import asyncio +import os + +import numpy as np + +_HERE = os.path.dirname(os.path.abspath(__file__)) +ARTIFACTS = os.path.join(_HERE, "..", "artifacts") +OUT_PATH = os.path.join(ARTIFACTS, "card_embeddings.npz") + +MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2" # 384차원, 한국어 지원, 로컬/무료 + + +async def load_cards(): + """card.nego_cards 에서 (number, name, script, strategy_type, tone) 로드.""" + import asyncpg + conn = await asyncpg.connect( + host="127.0.0.1", port=5432, user="postgres", password="password", database="negosium_db") + try: + rows = await conn.fetch( + "SELECT number, name, script, strategy_type, tone FROM card.nego_cards " + "WHERE deleted = FALSE ORDER BY number") + return [(r["number"], r["name"], r["script"], r["strategy_type"], r["tone"]) for r in rows] + finally: + await conn.close() + + +def main(): + cards = asyncio.run(load_cards()) + if not cards: + raise SystemExit("card.nego_cards 가 비어있음 — DB 시드 확인 (docker start negosium-pg)") + print(f"카드 {len(cards)}장 로드: {[c[0] for c in cards]}") + + from sentence_transformers import SentenceTransformer + model = SentenceTransformer(MODEL_NAME) + texts = [f"{name}. {script}" for _, name, script, _, _ in cards] + emb = model.encode(texts, normalize_embeddings=True) # [N, 384], 단위벡터 + print(f"임베딩 shape: {emb.shape}") + + os.makedirs(ARTIFACTS, exist_ok=True) + np.savez( + OUT_PATH, + numbers=np.array([c[0] for c in cards]), + names=np.array([c[1] for c in cards]), + strategy=np.array([c[3] for c in cards], dtype=np.int64), + tone=np.array([c[4] for c in cards], dtype=np.int64), + embeddings=emb.astype(np.float32), + ) + print(f"저장: {OUT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/agent/tools/compare_qtable_vs_dqn.py b/agent/tools/compare_qtable_vs_dqn.py new file mode 100644 index 0000000..a96c41e --- /dev/null +++ b/agent/tools/compare_qtable_vs_dqn.py @@ -0,0 +1,251 @@ +"""기존 Q-Table(UCB) vs action-as-feature DQN 공정 비교 — 고객사 성향 조건화 환경 (최종). + +같은 환경(FeatureBuyer 2축 + 협력사·고객사성향 랜덤)에서 동일 에피소드로 학습·평가. + - Q-Table: 이산 state 162칸 + 카드=슬롯. 성향(고객사) 입력 자체가 불가능 → 평균 성향에 수렴 + - DQN : 연속 상태 + 성향 벡터 + 카드 특징(임베딩+전략/톤 one-hot) + +평가 4종: + ① 학습 카드 9장 — 평균보상(진짜 목적함수) + top3 적중(MC 정답 기준) + ② zero-shot 11장 — 안 본 카드 2장 포함 + ③ 새 카드 첫 턴 사용률 — 구조적 차이 + ④ 성향 극단 테스트 — 같은 협력사, 성향만 바꿨을 때 카드를 바꾸는가 + +실행: APP_ENV=local python -m tools.compare_qtable_vs_dqn +""" + +import random + +import numpy as np +import torch + +from eval_harness.buyer import Scenario +from eval_harness.feature_buyer import FeatureBuyer, SupplierProfile, sample_supplier +from negotiation.policies.feature_dqn_policy import FeatureDQNPolicy +from negotiation.policies.qtable_policy import UCBQTablePolicy +from negotiation.policies.base import EpisodeState, PolicyContext, Transition +from negotiation.qtable.domain.model.q_table import QTable +from negotiation.qtable.domain.model.snapshot import NegotiationOutcome +from negotiation.qtable.domain.service.feature_builder import ( + STATE_FEATURE_DIM, TENANT_FEATURE_DIM, build_state_features) +from negotiation.qtable.domain.service.reward_calculator import RewardCalculator +from negotiation.qtable.domain.service.state_calculator import state_index +from tenancy.config_loader import TenantConfigLoader +from tools.train_feature_dqn import ( + ANCHOR, HOLDOUT, MAX_TURNS, TARGET, load_cards, make_snapshot, pref_config, sample_tenant_pref) + + +# ---- 정책 어댑터 ------------------------------------------------------------------ +class DQNAdapter: + name = "feature_dqn" + + def __init__(self, policy, feat): + self.p, self.feat = policy, feat + + def _sf(self, snap, tf): + return np.concatenate([build_state_features(snap), tf]) + + def choose(self, snap, tf, avail, greedy): + self.p.greedy = greedy + i, _, _ = self.p.select(self._sf(snap, tf), np.stack([self.feat[c] for c in avail])) + return avail[i] + + def learn(self, snap, tf, card, reward, next_snap, next_avail, done): + sf = self._sf(snap, tf) + if done or next_snap is None: + self.p.remember(sf, self.feat[card], reward, None, None, True) + else: + self.p.remember(sf, self.feat[card], reward, self._sf(next_snap, tf), + np.stack([self.feat[c] for c in next_avail]), False) + self.p.train_step() + + +class QTableAdapter: + """기존 UCBQTablePolicy. 성향(tf)은 구조상 받을 수 없다 — 이산 state 162칸에 그 축이 없음.""" + + name = "qtable_ucb" + + def __init__(self, all_numbers, state_cfg, lr=0.1, gamma=0.95): + self.numbers = list(all_numbers) + self.a_of = {n: i for i, n in enumerate(self.numbers)} + self.state_cfg = state_cfg + self.qt = QTable(162, len(self.numbers), learning_rate=lr, discount_factor=gamma) + self.pol = UCBQTablePolicy(self.qt) + + def choose(self, snap, tf, avail, greedy): + idx = state_index(snap, self.state_cfg) + if greedy: + q = self.qt.row(idx) + return max(avail, key=lambda c: q[self.a_of[c]]) + mask = np.zeros(len(self.numbers), dtype=bool) + for c in avail: + mask[self.a_of[c]] = True + ctx = PolicyContext(state_index=idx, snapshot=snap, action_space_size=len(self.numbers), + episode=EpisodeState(), available_mask=mask) + return self.numbers[self.pol.select(ctx).action_id] + + def learn(self, snap, tf, card, reward, next_snap, next_avail, done): + idx = state_index(snap, self.state_cfg) + nidx = state_index(next_snap, self.state_cfg) if (next_snap is not None and not done) else None + self.pol.update(Transition(state_index=idx, action_id=self.a_of[card], reward=reward, + next_state_index=nidx, done=done)) + + +class RandomAdapter: + name = "random" + + def __init__(self, seed=0): + self.rng = np.random.default_rng(seed) + + def choose(self, snap, tf, avail, greedy): + return avail[self.rng.integers(len(avail))] + + def learn(self, *a, **k): + pass + + +# ---- 공용 에피소드 ----------------------------------------------------------------- +def run_episode(adapter, sup, tf, pool, strat, rc, seed, learn=True, greedy=False, forced_first=None): + buyer = FeatureBuyer(sup, strat, seed=seed, max_turns=MAX_TURNS) + scenario = Scenario(anchor_price=ANCHOR, target_price=TARGET, revenue_amount=sup.revenue_amount, + distribution_code=sup.distribution_code, partner_count=sup.partner_count) + price0 = TARGET * 1.15 + price, used, total_r, first_card = price0, set(), 0.0, None + + for turn in range(1, MAX_TURNS + 1): + acceptance = max(0.0, (price0 - price) / price0) + snap = make_snapshot(sup, price, turn, acceptance) + avail = [c for c in pool if c not in used] or list(pool) + if turn == 1 and forced_first is not None: + card = forced_first + else: + card = adapter.choose(snap, tf, avail, greedy) + used.add(card) + if first_card is None: + first_card = card + + resp = buyer.respond(card, scenario, turn, price) + price = resp.new_price + done = resp.accept or price <= ANCHOR or turn >= MAX_TURNS + success = resp.accept or price <= ANCHOR + outcome = (NegotiationOutcome.SUCCESS if success + else NegotiationOutcome.FAILURE if done else NegotiationOutcome.ONGOING) + # 채점은 최종 결과 시점만 (중간 턴 0 → γ 부트스트랩으로 전파). + # 진행 중 보상을 누적하면 '질질 끄는 전략'이 부당하게 유리해지는 인공물이 생긴다. + r = rc.calculate(make_snapshot(sup, price, turn, acceptance, outcome)).total if done else 0.0 + total_r += r + + if learn: + if done: + adapter.learn(snap, tf, card, r, None, None, True) + else: + acc2 = max(0.0, (price0 - price) / price0) + nsnap = make_snapshot(sup, price, turn + 1, acc2) + navail = [c for c in pool if c not in used] or list(pool) + adapter.learn(snap, tf, card, r, nsnap, navail, False) + if done: + return total_r, success, price, first_card + return total_r, False, price, first_card + + +# ---- MC 정답 랭킹: 이 (협력사, 성향)에서 진짜 좋은 첫 카드 top-k --------------------- +_rand = RandomAdapter(seed=1) + +def rank_cards_mc(sup, tf, pool, strat, rc, seed, sims=6, k=3): + means = {} + for c in pool: + rs = [run_episode(_rand, sup, tf, pool, strat, rc, seed=seed + 17 * s, + learn=False, greedy=False, forced_first=c)[0] for s in range(sims)] + means[c] = np.mean(rs) + return sorted(means, key=lambda c: -means[c])[:k] + + +# ---- 학습/평가 --------------------------------------------------------------------- +def train(adapter, pool, strat, base_reward, state_cfg, episodes, seed): + rng = np.random.default_rng(seed) + for ep in range(1, episodes + 1): + sup = sample_supplier(rng) + rcfg, tf = sample_tenant_pref(rng, base_reward) + rc = RewardCalculator(rcfg, state_cfg) + run_episode(adapter, sup, tf, pool, strat, rc, seed=seed * 100 + ep, learn=True) + + +def evaluate(adapter, pool, strat, base_reward, state_cfg, n=300, seed0=777, label=""): + from negotiation.qtable.domain.service.feature_builder import build_tenant_features + rng = np.random.default_rng(seed0) + rewards, succ, ratios, hits, holdout_first = [], 0, [], 0, 0 + for i in range(n): + sup = sample_supplier(rng) + rcfg, tf = sample_tenant_pref(rng, base_reward) + rc = RewardCalculator(rcfg, state_cfg) + good = rank_cards_mc(sup, tf, pool, strat, rc, seed=seed0 * 7 + i) + r, ok, price, first = run_episode(adapter, sup, tf, pool, strat, rc, + seed=seed0 * 1000 + i, learn=False, greedy=True) + rewards.append(r); succ += ok; ratios.append(price / TARGET) + hits += (first in good); holdout_first += (first in HOLDOUT) + m, ci = float(np.mean(rewards)), float(1.96 * np.std(rewards) / np.sqrt(n)) + print(f"{label:<14} mean_rwd={m:.4f} ±{ci:.4f} success={succ/n:.3f} " + f"settled/tgt={np.mean(ratios):.3f} top3_hit={hits/n:.3f} 새카드첫턴={holdout_first/n:.3f}") + + +def pref_behavior_test(adapters, pool, strat, base_reward, state_cfg): + """④ 같은 협력사, 성향만 바꿨을 때 카드를 바꾸는가 (greedy). + + 첫 턴은 '일단 깎기'가 공통 정답이라 성향 차이가 잘 안 드러난다. + → 협상 중반(가격이 이미 target 근처, 3턴째) 상태를 함께 프로브: 여기서 + 성사중시는 '마무리(수락 잘 되는) 카드', 가격중시는 '더 깎는 카드'가 갈려야 한다. + """ + from negotiation.qtable.domain.service.feature_builder import build_tenant_features + sups = [SupplierProfile(5_000_000, 3, "A"), # 소형·경쟁多 + SupplierProfile(200_000_000, 1, "A")] # 대형·단독 + probes = [("첫턴", TARGET * 1.15, 1, 0.0), + ("중반(3턴,가격↓)", TARGET * 1.02, 3, 0.11)] + for pr_name, price, turn, acc in probes: + print(f"\n ── 프로브: {pr_name} (price={price:.0f}) ──") + print(f" {'협력사':<13} {'성향':<9} " + " ".join(f"{a.name:<15}" for a in adapters)) + for sup in sups: + row = {} + for p, pname in [(0.05, "성사중시"), (0.95, "가격중시")]: + rcfg = pref_config(base_reward, p) + tf = build_tenant_features(rcfg) + picks = [] + for a in adapters: + snap = make_snapshot(sup, price, turn, acc) + picks.append(a.choose(snap, tf, pool, True)) + seg = f"{sup.segment[0]}·{sup.segment[1]}" + print(f" {seg:<13} {pname:<9} " + " ".join(f"{c}(전략{strat[c]})".ljust(15) for c in picks)) + + +def main(episodes=10000, seed=42): + random.seed(seed); np.random.seed(seed); torch.manual_seed(seed) + numbers, feat, strat = load_cards() + train_pool = [c for c in numbers if c not in HOLDOUT] + tcfg = TenantConfigLoader().load("ktcommerce") + card_dim = feat[numbers[0]].shape[0] + print(f"환경: 2축 FeatureBuyer + 성향 랜덤 · 학습 {episodes}ep · 카드특징 {card_dim}차원 " + f"(임베딩384+전략4+톤4) · 학습 {len(train_pool)}장 / 홀드아웃 {HOLDOUT}") + + qt = QTableAdapter(numbers, tcfg.state) + dqn = DQNAdapter(FeatureDQNPolicy(state_dim=STATE_FEATURE_DIM + TENANT_FEATURE_DIM, + card_dim=card_dim, eps_decay=4000), feat) + + print("\n[학습] qtable_ucb ...") + train(qt, train_pool, strat, tcfg.reward, tcfg.state, episodes, seed) + print("[학습] feature_dqn ...") + train(dqn, train_pool, strat, tcfg.reward, tcfg.state, episodes, seed) + + print("\n=== ① 학습 카드 9장 풀 ===") + evaluate(RandomAdapter(seed), train_pool, strat, tcfg.reward, tcfg.state, label="random") + evaluate(qt, train_pool, strat, tcfg.reward, tcfg.state, label="qtable_ucb") + evaluate(dqn, train_pool, strat, tcfg.reward, tcfg.state, label="feature_dqn") + + print("\n=== ② zero-shot 11장 풀 (안 본 카드 2장 포함) ===") + evaluate(RandomAdapter(seed), numbers, strat, tcfg.reward, tcfg.state, label="random") + evaluate(qt, numbers, strat, tcfg.reward, tcfg.state, label="qtable_ucb") + evaluate(dqn, numbers, strat, tcfg.reward, tcfg.state, label="feature_dqn") + + print("\n=== ④ 성향 극단 테스트 — 같은 협력사, 성향만 바꾸면 카드를 바꾸는가 (11장 풀) ===") + pref_behavior_test([qt, dqn], numbers, strat, tcfg.reward, tcfg.state) + + +if __name__ == "__main__": + main() diff --git a/agent/tools/export_dqn_serving.py b/agent/tools/export_dqn_serving.py new file mode 100644 index 0000000..baaa688 --- /dev/null +++ b/agent/tools/export_dqn_serving.py @@ -0,0 +1,79 @@ +"""feature_dqn 체크포인트(.pt) → 서빙 번들(dqn_serving.npz) export. + +서빙 컨테이너에 PyTorch 를 넣지 않기 위해 ScoreNet(3층 MLP) 가중치와 카드 특징 +(임베딩384 + 전략 one-hot4 + 톤 one-hot4 = 392)을 numpy 번들 하나로 묶는다. +추론은 negotiation.policy.dqn_store 의 numpy forward 가 수행한다. + +실행(호스트, torch 필요): APP_ENV=local python -m tools.export_dqn_serving +산출: agent/artifacts/dqn_serving.npz (.dockerignore 미제외 → 이미지에 포함) +""" + +import os + +import numpy as np +import torch + +from tools.train_feature_dqn import load_cards + +_HERE = os.path.dirname(os.path.abspath(__file__)) +CKPT_PATH = os.path.join(_HERE, "..", "artifacts", "feature_dqn_ktcommerce.pt") +OUT_PATH = os.path.join(_HERE, "..", "artifacts", "dqn_serving.npz") + +STATE_DIM = 14 # build_state_features(9) + build_tenant_features(5) +CARD_DIM = 392 + + +def _np_forward(x, W0, b0, W1, b1, W2, b2): + h = np.maximum(x @ W0.T + b0, 0.0) + h = np.maximum(h @ W1.T + b1, 0.0) + return h @ W2.T + b2 + + +def export_bundle(sd, out_path: str) -> str: + """state_dict → 서빙 번들 npz (원자적 교체: .tmp 작성 후 replace). 반환: 절대경로. + + retrain_from_logs 재학습 배포도 이 함수를 쓴다 — 검증(torch/numpy 일치)은 main() 전용. + """ + W0, b0 = sd["net.0.weight"].numpy(), sd["net.0.bias"].numpy() + W1, b1 = sd["net.2.weight"].numpy(), sd["net.2.bias"].numpy() + W2, b2 = sd["net.4.weight"].numpy(), sd["net.4.bias"].numpy() + assert W0.shape[1] == STATE_DIM + CARD_DIM, f"입력 차원 불일치: {W0.shape[1]}" + numbers, feat, _ = load_cards() + card_feats = np.stack([feat[n] for n in numbers]).astype(np.float32) + tmp = out_path + ".tmp" + with open(tmp, "wb") as f: + np.savez( + f, + W0=W0, b0=b0, W1=W1, b1=b1, W2=W2, b2=b2, + card_numbers=np.array(numbers), card_feats=card_feats, + state_dim=STATE_DIM, card_dim=CARD_DIM, + ) + if os.path.exists(out_path): + os.replace(out_path, out_path + ".prev") # 직전 번들 백업(롤백용) + os.replace(tmp, out_path) + return os.path.abspath(out_path) + + +def main(): + sd = torch.load(CKPT_PATH, map_location="cpu") + # 정합성 검증: torch forward == numpy forward + from negotiation.policies.feature_dqn_policy import ScoreNet + net = ScoreNet(STATE_DIM, CARD_DIM) + net.load_state_dict(sd) + net.eval() + x = np.random.default_rng(0).normal(size=(8, STATE_DIM + CARD_DIM)).astype(np.float32) + with torch.no_grad(): + ref = net(torch.tensor(x)).numpy() + W0, b0 = sd["net.0.weight"].numpy(), sd["net.0.bias"].numpy() + W1, b1 = sd["net.2.weight"].numpy(), sd["net.2.bias"].numpy() + W2, b2 = sd["net.4.weight"].numpy(), sd["net.4.bias"].numpy() + out = _np_forward(x, W0, b0, W1, b1, W2, b2).squeeze(-1) + diff = float(np.abs(ref - out).max()) + assert diff < 1e-4, f"numpy/torch forward 불일치: {diff}" + + path = export_bundle(sd, OUT_PATH) + print(f"[저장] {path} forward 오차 {diff:.2e}") + + +if __name__ == "__main__": + main() diff --git a/agent/tools/probe_serving_dqn.py b/agent/tools/probe_serving_dqn.py new file mode 100644 index 0000000..3290b88 --- /dev/null +++ b/agent/tools/probe_serving_dqn.py @@ -0,0 +1,61 @@ +"""probe_serving_dqn — 서빙 번들(dqn_serving.npz)의 상황별 카드 선택 프로브. + +배포된 모델이 '상황에 맞게' 고르는지 눈으로 확인하는 진단 도구: +협력사 세그먼트 × 고객사 성향 × 협상 국면(가격대)별 선택 카드를 표로 출력한다. +전부 다르길 기대하는 게 아니라, 축을 바꿨을 때 선택이 '움직이는지'를 본다. + +실행: APP_ENV=local python -m tools.probe_serving_dqn (numpy 만 필요, DB 불필요) +""" + +import numpy as np + +from negotiation.qtable.domain.model.snapshot import NegotiationSnapshot +from negotiation.qtable.domain.service.feature_builder import build_state_features, build_tenant_features +from tenancy.config_loader import TenantConfigLoader +from tools.export_dqn_serving import OUT_PATH +from tools.retrain_from_logs import np_scorer_from_bundle +from tools.train_feature_dqn import pref_config + +ANCHOR, TARGET = 495_000.0, 500_000.0 # BUGCHECK 견적과 동일 스케일 + +SUPPLIERS = { + "소형·경쟁多": dict(revenue_amount=5_000_000, partner_count=3, distribution_code="A"), + "소형·단독": dict(revenue_amount=5_000_000, partner_count=1, distribution_code="A"), + "대형·경쟁多": dict(revenue_amount=200_000_000, partner_count=3, distribution_code="A"), + "대형·단독": dict(revenue_amount=200_000_000, partner_count=1, distribution_code="A"), +} +PHASES = { # (라운드, 제시가): 첫턴 높은 가격 / 중반 목표가 근접 / 막판 앵커존 직전 + "첫턴(575k)": (1, 575_000.0), + "중반(510k)": (2, 510_000.0), + "막판(501k)": (3, 501_000.0), +} +PREFS = {"성사중시": 0.1, "가격중시": 0.9} + + +def main(): + score = np_scorer_from_bundle(OUT_PATH) + z = np.load(OUT_PATH, allow_pickle=False) + numbers = [str(n) for n in z["card_numbers"]] + feats = z["card_feats"] + base = TenantConfigLoader().load("ktcommerce").reward + + for phase, (turn, price) in PHASES.items(): + print(f"\n=== {phase} (앵커 {int(ANCHOR):,} / 목표 {int(TARGET):,}) ===") + print(f"{'협력사':<12}" + "".join(f"{p:>16}" for p in PREFS)) + for sup_name, sup in SUPPLIERS.items(): + row = [] + for _, p in PREFS.items(): + tf = build_tenant_features(pref_config(base, p)) + snap = NegotiationSnapshot( + revenue_amount=sup["revenue_amount"], distribution_code=sup["distribution_code"], + partner_count=sup["partner_count"], + acceptance_ratio=max(0.0, (575_000.0 - price) / 575_000.0), + input_price=price, anchor_price=ANCHOR, target_price=TARGET, round_number=turn, + ) + sf = np.concatenate([build_state_features(snap), tf]) + row.append(numbers[int(np.argmax(score(sf, feats)))]) + print(f"{sup_name:<12}" + "".join(f"{c:>16}" for c in row)) + + +if __name__ == "__main__": + main() diff --git a/agent/tools/retrain_from_logs.py b/agent/tools/retrain_from_logs.py new file mode 100644 index 0000000..5ce2480 --- /dev/null +++ b/agent/tools/retrain_from_logs.py @@ -0,0 +1,238 @@ +"""retrain_from_logs — experience_logs 실데이터로 feature_dqn 오프라인 재학습 + OPE 게이트. + +파이프라인: + ① learning.experience_logs 로드(전 테넌트 — 범용 에이전트는 테넌트를 특징으로 조건화하므로 통합 학습) + ② 세션별 에피소드 재구성: 카드턴(done=False) N개 + 종료행(done=True) 1개. + 보상은 학습 규약(최종 결과 시점만 채점)에 맞춰 종료행 reward 만 쓰고 중간턴은 0. + ③ 현재 체크포인트에서 fine-tune (낮은 lr — 시뮬 사전학습 망각 방지) + ④ OPE(SNIPS, 궤적 IS): 후보 모델 vs 현재 서빙 번들. 후보가 못 넘으면 배포하지 않는다. + ⑤ 통과 시 dqn_serving.npz 원자적 교체(직전본 .prev 백업) → `docker compose build agent && up -d agent` 로 배포. + +실행(호스트, torch+DB 필요): + APP_ENV=local python -m tools.retrain_from_logs +환경변수: + MIN_EPISODES(기본 200) 재학습 최소 에피소드 수 — 미달 시 skip (과적합 방지) + EPOCHS(기본 20) / LR(기본 1e-4) / FORCE_DEPLOY=1 (OPE 게이트 무시 — 테스트 전용) + +주의: 서빙이 greedy(탐색 없음)라 로그가 선택 편향됨 — OPE 의 유효표본(ESS)이 작으면 +게이트가 보수적으로 배포를 막는다. 이는 의도된 동작이다(조용한 성능저하 방지). +""" + +import asyncio +import json +import os +from collections import defaultdict + +import numpy as np +import torch + +from common.database.db_session_manager import DB_SESSION_MNG +from common.database.model.models import ExperienceLog +from common.enums import DBType, DBWRType +from negotiation.policies.feature_dqn_policy import FeatureDQNPolicy +from negotiation.qtable.domain.model.snapshot import NegotiationSnapshot +from negotiation.qtable.domain.service.feature_builder import ( + STATE_FEATURE_DIM, TENANT_FEATURE_DIM, build_state_features, build_tenant_features) +from sqlalchemy import select +from tenancy.config_loader import TenantConfigLoader +from tools.export_dqn_serving import CKPT_PATH, OUT_PATH, export_bundle +from tools.train_feature_dqn import load_cards + +_HERE = os.path.dirname(os.path.abspath(__file__)) +RETRAIN_CKPT = os.path.join(_HERE, "..", "artifacts", "feature_dqn_retrained.pt") +REPORT_PATH = os.path.join(_HERE, "..", "artifacts", "retrain_report.json") + +MIN_EPISODES = int(os.getenv("MIN_EPISODES", "200")) +EPOCHS = int(os.getenv("EPOCHS", "20")) +LR = float(os.getenv("LR", "1e-4")) +FORCE_DEPLOY = os.getenv("FORCE_DEPLOY") == "1" +PROPENSITY_FALLBACK = 0.9 # 구로그 propensity 누락 시 (UCB/DQN 모두 greedy≈(1-ε)+ε/n) + + +# ---- ① 로그 로드 ------------------------------------------------------------- +async def fetch_logs(): + def _q(s): + q = (select(ExperienceLog.company_id, ExperienceLog.session_id, ExperienceLog.card_id, + ExperienceLog.reward, ExperienceLog.done, ExperienceLog.snapshot, + ExperienceLog.propensity, ExperienceLog.turn, ExperienceLog.id) + .where(ExperienceLog.is_invalidated == False) # noqa: E712 + .order_by(ExperienceLog.company_id, ExperienceLog.session_id, ExperienceLog.id)) + return DB_SESSION_MNG.execute(s, q) + err, rows = await DB_SESSION_MNG.execute_lambda(DBType.MAIN.value, DBWRType.DB_READ.value, _q) + return rows + + +# ---- ② 에피소드 재구성 -------------------------------------------------------- +def build_episodes(rows, known_cards: set): + """→ [{tenant, steps:[(snapshot, card, propensity)], terminal_reward}], 스킵 사유 카운트.""" + by_session = defaultdict(list) + for r in rows: + if r[1] is not None: + by_session[(r[0], str(r[1]))].append(r) + + episodes, skipped = [], defaultdict(int) + for (company_id, _sid), items in by_session.items(): + selects = [r for r in items if not r[4] and r[5]] # done=False, snapshot 有 + terminals = [r for r in items if r[4] and r[3] is not None] # done=True, reward 有 + if not selects or not terminals: + skipped["종료행/카드턴 없음(미완결 세션)"] += 1 + continue + if any(str(r[2] or "").startswith("AUT|") for r in selects): + skipped["완전 자율 세션(카드 재학습 대상 아님)"] += 1 + continue + if any(r[2] not in known_cards for r in selects): + skipped["임베딩 없는 카드(파일매핑 테넌트 등)"] += 1 + continue + episodes.append(dict( + tenant=company_id, + steps=[(r[5], r[2], r[6] if r[6] else PROPENSITY_FALLBACK) for r in selects], + terminal_reward=float(terminals[-1][3]), + )) + return episodes, skipped + + +def tenant_feat_for(cache: dict, loader: TenantConfigLoader, company_id: str) -> np.ndarray: + """테넌트 보상설정 → 성향 특징. 미온보딩/로드 실패는 _base 폴백.""" + if company_id not in cache: + try: + cfg = loader.load(company_id) + except Exception: + cfg = loader.load("_base") + cache[company_id] = build_tenant_features(cfg.reward) + return cache[company_id] + + +def to_transitions(episodes, feat, tenant_feats): + """학습 규약(train_feature_dqn 과 동일): 중간턴 r=0, 종료턴만 terminal_reward. 다음 후보 = 전체 − 사용분.""" + all_cards = list(feat.keys()) + out = [] + for ep in episodes: + tf = tenant_feats[ep["tenant"]] + used = set() + n = len(ep["steps"]) + for i, (snap_d, card, _p) in enumerate(ep["steps"]): + sf = np.concatenate([build_state_features(NegotiationSnapshot.from_dict(snap_d)), tf]) + used.add(card) + if i == n - 1: + out.append((sf, feat[card], ep["terminal_reward"], None, None, True)) + else: + s2_d = ep["steps"][i + 1][0] + s2 = np.concatenate([build_state_features(NegotiationSnapshot.from_dict(s2_d)), tf]) + cands = [c for c in all_cards if c not in used] or all_cards + out.append((sf, feat[card], 0.0, s2, np.stack([feat[c] for c in cands]), False)) + return out + + +# ---- ④ OPE (SNIPS, 궤적 단위 IS) ---------------------------------------------- +def _greedy_match(score_fn, ep, feat, tf) -> float: + """궤적 IS 가중치: Π 1[greedy(sᵢ)=aᵢ]/pᵢ. 한 턴이라도 불일치면 0.""" + all_cards = list(feat.keys()) + w, used = 1.0, set() + for snap_d, card, p in ep["steps"]: + sf = np.concatenate([build_state_features(NegotiationSnapshot.from_dict(snap_d)), tf]) + cands = [c for c in all_cards if c not in used] or all_cards + sc = score_fn(sf, np.stack([feat[c] for c in cands])) + if cands[int(np.argmax(sc))] != card: + return 0.0 + w /= max(p, 1e-3) + used.add(card) + return w + + +def snips(score_fn, episodes, feat, tenant_feats): + """SNIPS 추정치 + 유효표본크기(ESS). 매치 0건이면 (None, 0).""" + ws, rs = [], [] + for ep in episodes: + w = _greedy_match(score_fn, ep, feat, tenant_feats[ep["tenant"]]) + ws.append(w) + rs.append(ep["terminal_reward"]) + ws, rs = np.array(ws), np.array(rs) + if ws.sum() <= 0: + return None, 0.0 + est = float((ws * rs).sum() / ws.sum()) + ess = float(ws.sum() ** 2 / (ws ** 2).sum()) + return est, ess + + +def np_scorer_from_bundle(path): + """현재 서빙 번들(npz) → score_fn (dqn_store 와 동일 forward).""" + z = np.load(path, allow_pickle=False) + W0, b0, W1, b1, W2, b2 = z["W0"], z["b0"], z["W1"], z["b1"], z["W2"], z["b2"] + + def score(sf, card_feats): + x = np.concatenate([np.repeat(sf[None, :], card_feats.shape[0], axis=0), card_feats], axis=1) + h = np.maximum(x @ W0.T + b0, 0.0) + h = np.maximum(h @ W1.T + b1, 0.0) + return (h @ W2.T + b2).squeeze(-1) + return score + + +# ---- 메인 --------------------------------------------------------------------- +async def run(): + numbers, feat, _ = load_cards() + rows = await fetch_logs() + episodes, skipped = build_episodes(rows, set(numbers)) + print(f"로그 {len(rows)}행 → 에피소드 {len(episodes)}개 (스킵: {dict(skipped) or '없음'})") + + report = dict(rows=len(rows), episodes=len(episodes), skipped=dict(skipped), + min_episodes=MIN_EPISODES, deployed=False) + if len(episodes) < MIN_EPISODES and not FORCE_DEPLOY: + print(f"[skip] 에피소드 {len(episodes)} < MIN_EPISODES {MIN_EPISODES} — 과적합 위험으로 재학습 안 함") + report["result"] = "skipped_insufficient_data" + return report + + loader = TenantConfigLoader() + tenant_feats = {} + for ep in episodes: + tenant_feat_for(tenant_feats, loader, ep["tenant"]) + + # ③ fine-tune (시뮬 사전학습 체크포인트에서 이어서, 낮은 lr) + transitions = to_transitions(episodes, feat, tenant_feats) + batch = min(64, max(8, len(transitions) // 4)) + policy = FeatureDQNPolicy(state_dim=STATE_FEATURE_DIM + TENANT_FEATURE_DIM, + card_dim=feat[numbers[0]].shape[0], lr=LR, batch_size=batch) + if os.path.exists(CKPT_PATH): + policy.load(CKPT_PATH) + print(f"[fine-tune] 시작점: {os.path.basename(CKPT_PATH)} lr={LR} batch={batch}") + policy.buf.extend(transitions) + steps = EPOCHS * max(1, len(transitions) // batch) + losses = [l for _ in range(steps) if (l := policy.train_step()) is not None] + print(f"[fine-tune] {steps} step loss {losses[0]:.4f} → {losses[-1]:.4f}" if losses else "[fine-tune] 스텝 없음") + + # ④ OPE 게이트: 후보 vs 현재 서빙 + def cand_score(sf, cf): + return policy.scores(sf, cf) + cand_est, cand_ess = snips(cand_score, episodes, feat, tenant_feats) + cur_est, cur_ess = (snips(np_scorer_from_bundle(OUT_PATH), episodes, feat, tenant_feats) + if os.path.exists(OUT_PATH) else (None, 0.0)) + print(f"[OPE/SNIPS] 후보 {cand_est} (ESS {cand_ess:.1f}) vs 현재 {cur_est} (ESS {cur_ess:.1f})") + report.update(ope_candidate=cand_est, ope_candidate_ess=cand_ess, + ope_current=cur_est, ope_current_ess=cur_ess) + + min_ess = max(3.0, 0.02 * len(episodes)) + passed = (cand_est is not None and cand_ess >= min_ess + and (cur_est is None or cand_est >= cur_est - 0.01)) + if not passed and not FORCE_DEPLOY: + print(f"[게이트 불통과] 배포하지 않음 (필요 ESS ≥ {min_ess:.1f}). 현재 번들 유지.") + report["result"] = "gate_failed" + return report + + # ⑤ 배포: 후보 저장 + 번들 교체 (.prev 백업) + policy.save(RETRAIN_CKPT) + path = export_bundle(policy.q.state_dict(), OUT_PATH) + print(f"[배포] {path} (직전본 → dqn_serving.npz.prev)") + print(" 적용: docker compose build agent && docker compose up -d agent") + report.update(result="deployed" if passed else "force_deployed", deployed=True, + ckpt=os.path.abspath(RETRAIN_CKPT)) + return report + + +def main(): + report = asyncio.run(run()) + with open(REPORT_PATH, "w", encoding="utf-8") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + print(f"[리포트] {os.path.abspath(REPORT_PATH)}") + + +if __name__ == "__main__": + main() diff --git a/agent/tools/train_feature_dqn.py b/agent/tools/train_feature_dqn.py new file mode 100644 index 0000000..485385e --- /dev/null +++ b/agent/tools/train_feature_dqn.py @@ -0,0 +1,148 @@ +"""action-as-feature DQN 학습 (Phase 2·3) — 공용 환경 헬퍼 + 단독 학습 엔트리. + +카드 특징 = 스크립트 임베딩(384) + 전략 one-hot(4) + 톤 one-hot(4) = 392차원 +상태 특징 = 연속 상태(9) + 고객사 성향(5) = 14차원 ← 협력사·고객사 조건화 +학습 환경 = FeatureBuyer(양보력/수락력 2축) + 에피소드마다 협력사·고객사성향 랜덤 샘플링 + +비교 평가는 tools.compare_qtable_vs_dqn 에서 수행한다. +실행: APP_ENV=local python -m tools.train_feature_dqn +""" + +import os +import random + +import numpy as np +import torch + +from eval_harness.buyer import Scenario +from eval_harness.feature_buyer import FeatureBuyer, sample_supplier +from negotiation.policies.feature_dqn_policy import FeatureDQNPolicy +from negotiation.qtable.domain.model.snapshot import NegotiationOutcome, NegotiationSnapshot +from negotiation.qtable.domain.service.feature_builder import ( + STATE_FEATURE_DIM, TENANT_FEATURE_DIM, build_state_features, build_tenant_features) +from negotiation.qtable.domain.service.reward_calculator import RewardCalculator +from tenancy.config_loader import TenantConfigLoader + +_HERE = os.path.dirname(os.path.abspath(__file__)) +EMB_PATH = os.path.join(_HERE, "..", "artifacts", "card_embeddings.npz") +CKPT_PATH = os.path.join(_HERE, "..", "artifacts", "feature_dqn_ktcommerce.pt") + +# zero-shot 실험용 홀드아웃 (전략 1·4 — 남은 풀에도 같은 전략 존재). +# 서빙용 최종 학습은 전체 풀 사용: FULL_POOL=1 python -m tools.train_feature_dqn +HOLDOUT = [] if os.getenv("FULL_POOL") == "1" else ["NGC-002", "NGC-010"] +ANCHOR, TARGET = 8000.0, 10000.0 +MAX_TURNS = 5 +N_STRATEGY, N_TONE = 4, 4 + + +# ---- 카드 특징: 임베딩 + 전략/톤 one-hot ------------------------------------------ +def load_cards(): + z = np.load(EMB_PATH, allow_pickle=True) + numbers = [str(n) for n in z["numbers"]] + feat, strat = {}, {} + for i, n in enumerate(numbers): + s, t = int(z["strategy"][i]), int(z["tone"][i]) + s_oh = np.eye(N_STRATEGY, dtype=np.float32)[s - 1] + t_oh = np.eye(N_TONE, dtype=np.float32)[t - 1] + feat[n] = np.concatenate([z["embeddings"][i].astype(np.float32), s_oh, t_oh]) + strat[n] = s + return numbers, feat, strat + + +# ---- 고객사 성향: 보상 설정 샘플링 --------------------------------------------------- +def sample_tenant_pref(rng: np.random.Generator, base_cfg): + """p ∈ [0,1]: 0=성사중시(협력 유리) ↔ 1=가격중시(경쟁 유리). 반환: (RewardConfig, tenant_feat).""" + p = float(rng.uniform(0.0, 1.0)) + cfg = base_cfg.model_copy(update=dict( + max_weight=0.25 + 0.60 * p, # 가격보상 비중 + min_weight=(0.25 + 0.60 * p) * 0.7, + success_reward=1.6 - 1.2 * p, # 성사중시일수록 성공보상↑ + failure_penalty=-(1.4 - 1.1 * p), # 성사중시일수록 결렬이 아픔 + beta=0.1 + 0.4 * p, + penalty_lambda=float(rng.uniform(0.005, 0.05)), + )) + return cfg, build_tenant_features(cfg) + + +def pref_config(base_cfg, p: float, lam: float = 0.02): + """평가용: 성향 p 를 고정해 RewardConfig 생성 (극단 테스트).""" + return base_cfg.model_copy(update=dict( + max_weight=0.25 + 0.60 * p, min_weight=(0.25 + 0.60 * p) * 0.7, + success_reward=1.6 - 1.2 * p, failure_penalty=-(1.4 - 1.1 * p), + beta=0.1 + 0.4 * p, penalty_lambda=lam, + )) + + +def make_snapshot(sup, price: float, turn: int, acceptance: float, + outcome=NegotiationOutcome.ONGOING) -> NegotiationSnapshot: + return NegotiationSnapshot( + revenue_amount=sup.revenue_amount, distribution_code=sup.distribution_code, + partner_count=sup.partner_count, acceptance_ratio=acceptance, + input_price=price, anchor_price=ANCHOR, target_price=TARGET, + round_number=turn, outcome=outcome, + ) + + +# ---- 단독 학습 엔트리 (비교는 compare_qtable_vs_dqn) -------------------------------- +def main(episodes=10000, seed=42): + random.seed(seed); np.random.seed(seed); torch.manual_seed(seed) + numbers, feat, strat = load_cards() + train_pool = [c for c in numbers if c not in HOLDOUT] + card_dim = feat[numbers[0]].shape[0] + print(f"카드 {len(numbers)}장 (학습 {len(train_pool)} / 홀드아웃 {HOLDOUT}) card_dim={card_dim}") + + tcfg = TenantConfigLoader().load("ktcommerce") + policy = FeatureDQNPolicy(state_dim=STATE_FEATURE_DIM + TENANT_FEATURE_DIM, + card_dim=card_dim, eps_decay=4000, gamma=0.95) + rng = np.random.default_rng(seed) + + print(f"=== 학습 {episodes} 에피소드 (협력사·성향 랜덤, CPU) ===") + recent = [] + for ep in range(1, episodes + 1): + sup = sample_supplier(rng) + rcfg, tf = sample_tenant_pref(rng, tcfg.reward) + rc = RewardCalculator(rcfg, tcfg.state) + buyer = FeatureBuyer(sup, strat, seed=seed * 100 + ep, max_turns=MAX_TURNS) + scenario = Scenario(anchor_price=ANCHOR, target_price=TARGET, revenue_amount=sup.revenue_amount, + distribution_code=sup.distribution_code, partner_count=sup.partner_count) + price0 = TARGET * 1.15 + price, used, total_r = price0, set(), 0.0 + for turn in range(1, MAX_TURNS + 1): + acceptance = max(0.0, (price0 - price) / price0) + s = make_snapshot(sup, price, turn, acceptance) + sf = np.concatenate([build_state_features(s), tf]) + avail = [c for c in train_pool if c not in used] or list(train_pool) + embs = np.stack([feat[c] for c in avail]) + i, _, _ = policy.select(sf, embs) + card = avail[i]; used.add(card) + resp = buyer.respond(card, scenario, turn, price) + price = resp.new_price + done = resp.accept or price <= ANCHOR or turn >= MAX_TURNS + success = resp.accept or price <= ANCHOR + outcome = (NegotiationOutcome.SUCCESS if success + else NegotiationOutcome.FAILURE if done else NegotiationOutcome.ONGOING) + # 최종 결과 시점만 채점 (중간 0 → γ 부트스트랩) — compare 스크립트와 동일 규칙. + r = rc.calculate(make_snapshot(sup, price, turn, acceptance, outcome)).total if done else 0.0 + total_r += r + if done: + policy.remember(sf, feat[card], r, None, None, True) + else: + acc2 = max(0.0, (price0 - price) / price0) + s2 = make_snapshot(sup, price, turn + 1, acc2) + navail = [c for c in train_pool if c not in used] or list(train_pool) + policy.remember(sf, feat[card], r, np.concatenate([build_state_features(s2), tf]), + np.stack([feat[c] for c in navail]), False) + policy.train_step() + if done: + break + recent.append(total_r) + if ep % 2000 == 0: + print(f" ep {ep:>6} eps={policy.eps():.3f} 최근2000 평균보상={np.mean(recent[-2000:]):.4f}") + + policy.save(CKPT_PATH) + print(f"[저장] {CKPT_PATH}") + return policy + + +if __name__ == "__main__": + main()