minaChoi/o2o-negosium-original
1
0
Fork 0
Code Pull Requests Activity

[feat] agent: 카드 선택 DQN 서빙 전환 (action-as-feature)

Browse Source 
- feature_dqn_policy: ScoreNet(상태+카드특징 → 점수) 학습 정책 (replay+타깃넷)
- feature_builder: 이산화 없는 연속 상태 벡터(9) + 테넌트 성향 벡터(5)
- dqn_store: numpy 전용 서빙(컨테이너 PyTorch 불필요), DQN_SERVING 플래그,
  미지원 테넌트는 Q-table 자동 폴백
- 파이프라인: build_card_embeddings -> train_feature_dqn -> export_dqn_serving(npz)
- retrain_from_logs: 실로그 재학습 + OPE(SNIPS) 게이트, 통과 시에만 번들 교체(.prev 백업)
- probe_serving_dqn / compare_qtable_vs_dqn: 배포 전 행동 점검 도구

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jwkim 2026-07-10 16:56:24 +09:00
parent b368a19f79
commit d4acdd0ac5
14 changed files with 1252 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
agent/artifacts/card_embeddings.npz Normal file
View File

Binary file not shown.

BIN
agent/artifacts/dqn_serving.npz Normal file
View File

Binary file not shown.

BIN
agent/artifacts/feature_dqn_ktcommerce.pt Normal file
View File

Binary file not shown.

14
agent/artifacts/retrain_report.json Normal file
View File

@ -0,0 +1,14 @@
{
"rows": 25,
"episodes": 4,
"skipped": {
"종료행/카드턴 없음(미완결 세션)": 8
},
"min_episodes": 1,
"deployed": false,
"ope_candidate": 0.9029104414200676,
"ope_candidate_ess": 1.0,
"ope_current": 0.9029104414200676,
"ope_current_ess": 1.0,
"result": "gate_failed"
}

104
agent/eval_harness/feature_buyer.py Normal file
View File

@ -0,0 +1,104 @@
"""FeatureBuyer — 카드 '내용(전략)'과 협력사 '프로필'에 반응하는 시뮬 협력사 (Phase 2·3).
효과를 2축으로 분리한다(성향 조건화가 의미를 가지려면 트레이드오프가 필요):
- 양보력(concession power): 이 카드가 가격을 얼마나 끌어내리는가
- 수락력(accept power) : 이 카드가 합의(수락) 확률을 얼마나 높이는가
전략별 기본 프로필(트레이드오프):
경쟁(1): 양보력↑↑ 수락력↓ — 세게 깎지만 결렬 위험
수용(2): 양보력↓ 수락력↑
고수(3): 양보력·수락력 중간
협력(4): 양보력↓ 수락력↑↑ — 잘 성사되지만 덜 깎임
여기에 협력사 세그먼트 적합도(AFFINITY)가 곱해진다: 전략이 그 협력사에 안 맞으면 둘 다 죽는다.
소형·경쟁多 → 경쟁압박이 잘 먹힘 / 대형·단독 → 협력이 잘 먹힘(압박 역효과)
→ '가격 중시' 고객사는 경쟁 카드(많이 깎음, 결렬 감수), '성사 중시' 고객사는 협력 카드가 정답이
되는 구조. 에이전트는 카드 특징 + 협력사 특징 + 고객사 성향으로 이를 학습해야 한다.
"""
from dataclasses import dataclass
from typing import Dict, Tuple
import numpy as np
from eval_harness.buyer import BuyerResponse, Scenario
# strategy_type: 1=경쟁, 2=수용, 3=고수, 4=협력 (card.nego_cards)
# 전략별 (양보력, 수락력) 기본 프로필 — 트레이드오프의 원천
STRATEGY_PROFILE: Dict[int, Tuple[float, float]] = {
1: (0.90, 0.25), # 경쟁: 세게 깎지만 성사 어려움
2: (0.35, 0.70), # 수용
3: (0.55, 0.50), # 고수: 중간
4: (0.30, 0.90), # 협력: 잘 성사되지만 덜 깎임
}
# 세그먼트별 전략 적합도 m ∈ [0,1] — 전략이 그 협력사에 얼마나 '먹히는가'
AFFINITY: Dict[Tuple[str, str], Dict[int, float]] = {
("small", "multi"): {1: 0.90, 2: 0.45, 3: 0.60, 4: 0.40}, # 소형·경쟁多 → 경쟁압박
("small", "single"): {1: 0.35, 2: 0.60, 3: 0.80, 4: 0.55}, # 소형·단독 → 고수/논리
("big", "multi"): {1: 0.65, 2: 0.50, 3: 0.70, 4: 0.60},
("big", "single"): {1: 0.20, 2: 0.70, 3: 0.50, 4: 0.90}, # 대형·단독 → 협력 (압박 역효과)
}
REVENUE_BIG = 50_000_000 # state config 'high' 경계와 정합
@dataclass
class SupplierProfile:
"""협력사 프로필 — 에피소드마다 달라지는 협상 상대. snapshot 필드와 정합."""
revenue_amount: float
partner_count: int # 이 품목의 대안 협력사 수 (BATNA. 1:1 채팅이어도 다양)
distribution_code: str
@property
def segment(self) -> Tuple[str, str]:
size = "big" if self.revenue_amount > REVENUE_BIG else "small"
comp = "multi" if self.partner_count >= 2 else "single"
return (size, comp)
def sample_supplier(rng: np.random.Generator) -> SupplierProfile:
"""무작위 협력사 생성 (학습 데이터 다양성)."""
return SupplierProfile(
revenue_amount=float(rng.choice([5_000_000, 20_000_000, 80_000_000, 200_000_000])),
partner_count=int(rng.choice([1, 1, 2, 3])), # 단독 비중 높게
distribution_code=str(rng.choice(["A", "B", "C"])),
)
class FeatureBuyer:
"""전략 프로필 x 세그먼트 적합도 기반 협력사 모델. (양보력, 수락력) 2축."""
def __init__(self, supplier: SupplierProfile, card_strategy: Dict[str, int], seed: int = 0,
accept_base: float = 0.08, max_turns: int = 5, jitter: float = 0.05):
self.supplier = supplier
self.card_strategy = card_strategy # {card_number: strategy_type}
self.rng = np.random.default_rng(seed)
self.accept_base = accept_base
self.max_turns = max_turns
self.jitter = jitter
# 숨은 하한가(reservation): 앵커의 94~110%. 앵커보다 높으면(약 60%) 가격만으로는
# 타결 불가 → 수락을 받아내야 함 → 수락력 낮은(경쟁) 카드에 진짜 결렬 위험이 생긴다.
self.floor_ratio = float(self.rng.uniform(0.94, 1.10))
def powers(self, card_number: str) -> Tuple[float, float]:
"""숨은 (양보력, 수락력). 전략 프로필 × 세그먼트 적합도 + 카드별 결정론적 지터."""
strat = self.card_strategy.get(card_number, 3)
conc_base, acc_base = STRATEGY_PROFILE.get(strat, (0.5, 0.5))
m = AFFINITY[self.supplier.segment].get(strat, 0.5)
scale = 0.35 + 0.85 * m # 적합도: 안 맞으면 둘 다 죽음 (0.35~1.2)
j = (hash(card_number) % 1000 / 1000.0 - 0.5) * 2 * self.jitter
c_pow = float(np.clip(conc_base * scale + j, 0.02, 0.98))
a_pow = float(np.clip(acc_base * scale + j, 0.02, 0.98))
return c_pow, a_pow
def respond(self, card_number: str, scenario: Scenario, turn: int, current_price: float) -> BuyerResponse:
c_pow, a_pow = self.powers(card_number)
floor = scenario.anchor_price * self.floor_ratio # 숨은 하한가 (앵커 이하 보장 없음)
concession = (current_price - floor) * (0.10 + 0.55 * c_pow)
new_price = max(floor, current_price - concession)
p_accept = min(0.97, self.accept_base + 0.80 * a_pow + 0.05 * (turn - 1))
accept = bool(self.rng.random() < p_accept)
walked = (not accept) and (turn >= self.max_turns)
return BuyerResponse(accept=accept, new_price=new_price, walked=walked)

132
agent/negotiation/policies/feature_dqn_policy.py Normal file
View File

@ -0,0 +1,132 @@
"""FeatureDQNPolicy — action-as-feature DQN (Phase 2·3).
고정 슬롯 Q(s)→[11개] 대신 ScoreNet(상태벡터 + 카드임베딩) → 스칼라 점수.
결정 시 가용 카드 풀을 순회 채점해 argmax → 카드 추가/삭제/새 카드(zero-shot)에 구조 변화 없음.
협력사 특징은 상태벡터에 포함(feature_builder) → '협력사를 입력으로' 달성.
가변 행동 학습: replay 에 다음 상태의 '가용 카드 임베딩들'을 함께 저장,
target = r + γ · max_{c'∈next_avail} Q(s', c') · (1-done)
"""
import math
import random
from collections import deque
from typing import Dict, List, Optional, Tuple
import numpy as np
import torch
import torch.nn as nn
class ScoreNet(nn.Module):
"""(상태 + 카드임베딩) → 스칼라 점수."""
def __init__(self, state_dim: int, card_dim: int, hidden: int = 128):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim + card_dim, hidden), nn.ReLU(),
nn.Linear(hidden, hidden), nn.ReLU(),
nn.Linear(hidden, 1),
)
def forward(self, x: torch.Tensor) -> torch.Tensor: # x: [B, state+card]
return self.net(x).squeeze(-1) # [B]
class FeatureDQNPolicy:
name = "feature_dqn"
def __init__(self, state_dim: int, card_dim: int, device: str = "cpu",
lr: float = 1e-3, gamma: float = 0.95, hidden: int = 128,
eps_start: float = 1.0, eps_end: float = 0.05, eps_decay: int = 6000,
buffer_size: int = 50_000, batch_size: int = 64, target_sync: int = 500):
self.device = device
self.gamma = gamma
self.batch_size = batch_size
self.target_sync = target_sync
self.q = ScoreNet(state_dim, card_dim, hidden).to(device)
self.tgt = ScoreNet(state_dim, card_dim, hidden).to(device)
self.tgt.load_state_dict(self.q.state_dict())
self.opt = torch.optim.Adam(self.q.parameters(), lr=lr)
self.buf: deque = deque(maxlen=buffer_size)
self.eps_start, self.eps_end, self.eps_decay = eps_start, eps_end, eps_decay
self.steps = 0
self.greedy = False # 평가 모드(탐색 끔)
# ---- 탐색 스케줄 ----------------------------------------------------
def eps(self) -> float:
if self.greedy:
return 0.0
return self.eps_end + (self.eps_start - self.eps_end) * math.exp(-self.steps / self.eps_decay)
# ---- 채점/선택 -------------------------------------------------------
def scores(self, state_feat: np.ndarray, card_embs: np.ndarray) -> np.ndarray:
"""가용 카드 K개 일괄 채점. card_embs: [K, card_dim] → [K]."""
k = card_embs.shape[0]
x = np.concatenate([np.repeat(state_feat[None, :], k, axis=0), card_embs], axis=1)
with torch.no_grad():
return self.q(torch.tensor(x, device=self.device)).cpu().numpy()
def select(self, state_feat: np.ndarray, card_embs: np.ndarray) -> Tuple[int, float, float]:
"""(선택 인덱스, propensity, 선택 점수). 인덱스는 card_embs 행 기준."""
k = card_embs.shape[0]
sc = self.scores(state_feat, card_embs)
e = self.eps()
if random.random() < e:
i = random.randrange(k)
prop = e / k
else:
i = int(sc.argmax())
prop = (1.0 - e) + e / k
return i, prop, float(sc[i])
# ---- 경험/학습 -------------------------------------------------------
def remember(self, state_feat: np.ndarray, card_emb: np.ndarray, reward: float,
next_state_feat: Optional[np.ndarray], next_card_embs: Optional[np.ndarray],
done: bool):
self.buf.append((state_feat, card_emb, reward, next_state_feat, next_card_embs, done))
def train_step(self) -> Optional[float]:
if len(self.buf) < self.batch_size:
return None
batch = random.sample(self.buf, self.batch_size)
# Q(s, a_chosen)
xs = np.stack([np.concatenate([s, c]) for s, c, *_ in batch])
q_sa = self.q(torch.tensor(xs, device=self.device))
# target = r + γ·max_{c'} Q_tgt(s', c') — 가변 후보라 후보 전체를 한 번에 forward 후 세그먼트 max
rewards = torch.tensor([b[2] for b in batch], device=self.device, dtype=torch.float32)
dones = torch.tensor([float(b[5]) for b in batch], device=self.device)
next_rows, owner = [], []
for bi, (_, _, _, s2, cands, done) in enumerate(batch):
if done or s2 is None or cands is None or len(cands) == 0:
continue
for c in cands:
next_rows.append(np.concatenate([s2, c]))
owner.append(bi)
q_next_max = torch.zeros(self.batch_size, device=self.device)
if next_rows:
with torch.no_grad():
q_all = self.tgt(torch.tensor(np.stack(next_rows), device=self.device))
owner_t = torch.tensor(owner, device=self.device)
q_next_max = q_next_max.index_reduce_(0, owner_t, q_all, "amax", include_self=False)
target = rewards + self.gamma * q_next_max * (1.0 - dones)
loss = nn.functional.smooth_l1_loss(q_sa, target)
self.opt.zero_grad()
loss.backward()
self.opt.step()
self.steps += 1
if self.steps % self.target_sync == 0:
self.tgt.load_state_dict(self.q.state_dict())
return float(loss)
# ---- 저장/로드 -------------------------------------------------------
def save(self, path: str):
torch.save(self.q.state_dict(), path)
def load(self, path: str):
sd = torch.load(path, map_location=self.device)
self.q.load_state_dict(sd)
self.tgt.load_state_dict(sd)

118
agent/negotiation/policy/dqn_store.py Normal file
View File

@ -0,0 +1,118 @@
"""DQNServingStore — action-as-feature DQN 서빙 (선택 전용, 학습 없음).
tools/export_dqn_serving.py 가 만든 dqn_serving.npz(ScoreNet 가중치 + 카드특징 392차원)를
numpy 로 추론한다 — 서빙 컨테이너에 PyTorch 불필요.
역할 분담(계획서 H 트랙으로 가기 전 파일럿):
- 카드 '선택'만 DQN(greedy). Q-table 학습/영속/experience_logs 로깅 경로는 기존 그대로 유지
(Q-learning 은 오프폴리시라 DQN 이 고른 행동으로 갱신해도 유효, 로그는 DQN 오프라인 재학습 재료).
- 폴백: 플래그 꺼짐 / 번들 없음 / 가용 카드 전부 특징 미보유(신규 카드) → None 반환,
호출부(ChatService)가 기존 UCB Q-table 선택으로 진행한다.
활성화: 환경변수 DQN_SERVING=1 (docker-compose agent environment).
신규 카드 주의: 번들에 없는 카드번호는 후보에서 제외된다 — 카드 추가 시
tools/build_card_embeddings.py → tools/export_dqn_serving.py 재실행 후 재배포 필요.
"""
import os
from typing import List, Optional
import numpy as np
from common.logger import LOG
from negotiation.policies.base import ActionDecision, PolicyContext
from negotiation.qtable.domain.service.feature_builder import (
build_state_features, build_tenant_features)
_HERE = os.path.dirname(os.path.abspath(__file__))
BUNDLE_PATH = os.path.join(_HERE, "..", "..", "artifacts", "dqn_serving.npz")
class _Bundle:
def __init__(self, z):
self.W0, self.b0 = z["W0"], z["b0"]
self.W1, self.b1 = z["W1"], z["b1"]
self.W2, self.b2 = z["W2"], z["b2"]
self.state_dim = int(z["state_dim"])
self.card_feats = {str(n): z["card_feats"][i]
for i, n in enumerate(z["card_numbers"])}
def scores(self, state_feat: np.ndarray, card_feats: np.ndarray) -> np.ndarray:
"""가용 카드 K개 일괄 채점: [K, state+card] → [K]."""
k = card_feats.shape[0]
x = np.concatenate([np.repeat(state_feat[None, :], k, axis=0), card_feats], axis=1)
h = np.maximum(x @ self.W0.T + self.b0, 0.0)
h = np.maximum(h @ self.W1.T + self.b1, 0.0)
return (h @ self.W2.T + self.b2).squeeze(-1)
class DQNServingPolicy:
"""UCBQTablePolicy.select 와 동일한 PolicyContext → ActionDecision 계약(선택 전용)."""
name = "feature_dqn"
_EPS = 0.1 # propensity 근사용 ε (UCB 정책과 동일 관례 — OPE 지지 확보용, 선택은 greedy)
def __init__(self, bundle: _Bundle, engine): # engine: tenancy.registry.TenantEngine
self._bundle = bundle
self._mapper = engine.mapper
self._tenant_feat = build_tenant_features(engine.config.reward)
def _available(self, ctx: PolicyContext) -> List[int]:
# UCBQTablePolicy._available 과 동일 규칙 (마스크 → used 제외 → 소진 시 전체 허용)
if ctx.available_mask is not None:
avail = [a for a in range(ctx.action_space_size) if ctx.available_mask[a]]
else:
used = ctx.episode.used_action_ids if ctx.episode else set()
avail = [a for a in range(ctx.action_space_size) if a not in used]
return avail or list(range(ctx.action_space_size))
def select(self, ctx: PolicyContext) -> Optional[ActionDecision]:
"""카드특징이 있는 가용 카드가 없으면 None → 호출부가 Q-table 로 폴백."""
candidates = [] # (action_id, card_feat)
for a in self._available(ctx):
num = self._mapper.get_card_id(a)
feat = self._bundle.card_feats.get(num) if num else None
if feat is not None:
candidates.append((a, feat))
if not candidates:
return None
state_feat = np.concatenate([build_state_features(ctx.snapshot), self._tenant_feat])
if state_feat.shape[0] != self._bundle.state_dim:
LOG.e_no_callstack(
f"[DQNServing] state_dim 불일치: {state_feat.shape[0]} != {self._bundle.state_dim}")
return None
sc = self._bundle.scores(state_feat, np.stack([f for _, f in candidates]))
i = int(sc.argmax())
n = len(candidates)
return ActionDecision(
action_id=candidates[i][0],
propensity=(1.0 - self._EPS) + self._EPS / n,
q_value=float(sc[i]),
ucb_score=float(sc[i]),
available_actions=[a for a, _ in candidates],
)
class DQNServingStore:
"""번들 lazy 로드 + 캐시. 비활성/부재 시 None (호출부 Q-table 폴백)."""
_bundle: Optional[_Bundle] = None
_load_failed = False
@classmethod
def enabled(cls) -> bool:
return os.getenv("DQN_SERVING", "0").lower() in ("1", "true", "yes")
@classmethod
def policy_for(cls, engine) -> Optional[DQNServingPolicy]:
if not cls.enabled() or cls._load_failed:
return None
if cls._bundle is None:
try:
cls._bundle = _Bundle(np.load(BUNDLE_PATH, allow_pickle=False))
LOG.i(f"[DQNServing] 번들 로드 완료: 카드 {len(cls._bundle.card_feats)}장")
except Exception as ex:
cls._load_failed = True # 요청마다 재시도하지 않음
LOG.e_no_callstack(f"[DQNServing] 번들 로드 실패 → Q-table 폴백: {ex}")
return None
return DQNServingPolicy(cls._bundle, engine)

44
agent/negotiation/qtable/domain/service/feature_builder.py Normal file
View File

@ -0,0 +1,44 @@
"""build_state_features — snapshot(raw 연속값) → 실수 벡터 (DQN/action-as-feature 용).
state_calculator.build_state(이산화)와 대비되는 연속 표현. 이산화(등급/162칸)를 하지 않고
정규화된 raw 값을 그대로 벡터로 내보낸다. 협력사 특징(매출·경쟁사수·유통)이 벡터에 포함되므로
'협력사를 입력으로'(Phase 3)가 자연스럽게 달성된다.
"""
import numpy as np
from negotiation.qtable.domain.model.snapshot import NegotiationSnapshot
DIST_CLASSES = ("A", "B", "C")
STATE_FEATURE_DIM = 9 # build_state_features 벡터 길이. feature 추가 시 갱신.
TENANT_FEATURE_DIM = 5 # build_tenant_features 벡터 길이.
def build_tenant_features(reward_cfg) -> np.ndarray:
"""고객사 '성향'을 ID 가 아니라 보상 설정값(내용)으로 벡터화 (Phase 3 고객사 조건화).
새 고객사도 tenant.yaml 의 reward 설정만 있으면 즉시 조건화된다 (cold-start 없음).
"""
return np.array([
reward_cfg.max_weight, # 가격 중시 정도 (W↑ = 가격보상 비중↑)
reward_cfg.success_reward / 2.0, # 성사를 얼마나 크게 치는가
-reward_cfg.failure_penalty / 2.0, # 결렬을 얼마나 무서워하는가
reward_cfg.penalty_lambda * 20.0, # 속도 성향 (오래 끌수록 벌점)
reward_cfg.beta, # 앵커 초과달성 보너스 성향
], dtype=np.float32)
def build_state_features(s: NegotiationSnapshot) -> np.ndarray:
"""정규화된 연속 상태 벡터. 등급화 없음 — 990원과 850원이 구별된다."""
dist_onehot = [1.0 if s.distribution_code == c else 0.0 for c in DIST_CLASSES]
anchor = max(s.anchor_price, 1.0)
target = max(s.target_price, 1.0)
return np.array([
min(s.revenue_amount, 5e8) / 5e8, # 협력사 매출 (0~1)
*dist_onehot, # 유통 A/B/C
min(s.partner_count, 5) / 5.0, # 대안 협력사 수 (BATNA)
float(np.clip(s.acceptance_ratio, 0.0, 1.0)), # 수용률
float(np.clip((s.input_price - anchor) / anchor, -1.0, 2.0)), # 앵커 대비 격차 (연속!)
float(np.clip((target - s.input_price) / target, -2.0, 1.0)), # 목표 대비 여유
min(s.round_number, 10) / 10.0, # 라운드
], dtype=np.float32)

63
agent/tools/build_card_embeddings.py Normal file
View File

@ -0,0 +1,63 @@
"""카드 스크립트 → 임베딩 캐시 생성 (action-as-feature 준비, 1회 실행).
card.nego_cards(11장)의 name+script 를 문장 임베딩으로 변환해 artifacts/card_embeddings.npz 에 저장.
새 카드가 추가되면 이 스크립트를 다시 돌리면 된다(그 카드만 임베딩돼 캐시에 합류).
실행:
APP_ENV=local python -m tools.build_card_embeddings
출력:
artifacts/card_embeddings.npz (numbers, names, strategy, tone, embeddings[N,384])
"""
import asyncio
import os
import numpy as np
_HERE = os.path.dirname(os.path.abspath(__file__))
ARTIFACTS = os.path.join(_HERE, "..", "artifacts")
OUT_PATH = os.path.join(ARTIFACTS, "card_embeddings.npz")
MODEL_NAME = "paraphrase-multilingual-MiniLM-L12-v2" # 384차원, 한국어 지원, 로컬/무료
async def load_cards():
"""card.nego_cards 에서 (number, name, script, strategy_type, tone) 로드."""
import asyncpg
conn = await asyncpg.connect(
host="127.0.0.1", port=5432, user="postgres", password="password", database="negosium_db")
try:
rows = await conn.fetch(
"SELECT number, name, script, strategy_type, tone FROM card.nego_cards "
"WHERE deleted = FALSE ORDER BY number")
return [(r["number"], r["name"], r["script"], r["strategy_type"], r["tone"]) for r in rows]
finally:
await conn.close()
def main():
cards = asyncio.run(load_cards())
if not cards:
raise SystemExit("card.nego_cards 가 비어있음 — DB 시드 확인 (docker start negosium-pg)")
print(f"카드 {len(cards)}장 로드: {[c[0] for c in cards]}")
from sentence_transformers import SentenceTransformer
model = SentenceTransformer(MODEL_NAME)
texts = [f"{name}. {script}" for _, name, script, _, _ in cards]
emb = model.encode(texts, normalize_embeddings=True) # [N, 384], 단위벡터
print(f"임베딩 shape: {emb.shape}")
os.makedirs(ARTIFACTS, exist_ok=True)
np.savez(
OUT_PATH,
numbers=np.array([c[0] for c in cards]),
names=np.array([c[1] for c in cards]),
strategy=np.array([c[3] for c in cards], dtype=np.int64),
tone=np.array([c[4] for c in cards], dtype=np.int64),
embeddings=emb.astype(np.float32),
)
print(f"저장: {OUT_PATH}")
if __name__ == "__main__":
main()

251
agent/tools/compare_qtable_vs_dqn.py Normal file
View File

@ -0,0 +1,251 @@
"""기존 Q-Table(UCB) vs action-as-feature DQN 공정 비교 — 고객사 성향 조건화 환경 (최종).
같은 환경(FeatureBuyer 2축 + 협력사·고객사성향 랜덤)에서 동일 에피소드로 학습·평가.
- Q-Table: 이산 state 162칸 + 카드=슬롯. 성향(고객사) 입력 자체가 불가능 → 평균 성향에 수렴
- DQN : 연속 상태 + 성향 벡터 + 카드 특징(임베딩+전략/톤 one-hot)
평가 4종:
① 학습 카드 9장 — 평균보상(진짜 목적함수) + top3 적중(MC 정답 기준)
② zero-shot 11장 — 안 본 카드 2장 포함
③ 새 카드 첫 턴 사용률 — 구조적 차이
④ 성향 극단 테스트 — 같은 협력사, 성향만 바꿨을 때 카드를 바꾸는가
실행: APP_ENV=local python -m tools.compare_qtable_vs_dqn
"""
import random
import numpy as np
import torch
from eval_harness.buyer import Scenario
from eval_harness.feature_buyer import FeatureBuyer, SupplierProfile, sample_supplier
from negotiation.policies.feature_dqn_policy import FeatureDQNPolicy
from negotiation.policies.qtable_policy import UCBQTablePolicy
from negotiation.policies.base import EpisodeState, PolicyContext, Transition
from negotiation.qtable.domain.model.q_table import QTable
from negotiation.qtable.domain.model.snapshot import NegotiationOutcome
from negotiation.qtable.domain.service.feature_builder import (
STATE_FEATURE_DIM, TENANT_FEATURE_DIM, build_state_features)
from negotiation.qtable.domain.service.reward_calculator import RewardCalculator
from negotiation.qtable.domain.service.state_calculator import state_index
from tenancy.config_loader import TenantConfigLoader
from tools.train_feature_dqn import (
ANCHOR, HOLDOUT, MAX_TURNS, TARGET, load_cards, make_snapshot, pref_config, sample_tenant_pref)
# ---- 정책 어댑터 ------------------------------------------------------------------
class DQNAdapter:
name = "feature_dqn"
def __init__(self, policy, feat):
self.p, self.feat = policy, feat
def _sf(self, snap, tf):
return np.concatenate([build_state_features(snap), tf])
def choose(self, snap, tf, avail, greedy):
self.p.greedy = greedy
i, _, _ = self.p.select(self._sf(snap, tf), np.stack([self.feat[c] for c in avail]))
return avail[i]
def learn(self, snap, tf, card, reward, next_snap, next_avail, done):
sf = self._sf(snap, tf)
if done or next_snap is None:
self.p.remember(sf, self.feat[card], reward, None, None, True)
else:
self.p.remember(sf, self.feat[card], reward, self._sf(next_snap, tf),
np.stack([self.feat[c] for c in next_avail]), False)
self.p.train_step()
class QTableAdapter:
"""기존 UCBQTablePolicy. 성향(tf)은 구조상 받을 수 없다 — 이산 state 162칸에 그 축이 없음."""
name = "qtable_ucb"
def __init__(self, all_numbers, state_cfg, lr=0.1, gamma=0.95):
self.numbers = list(all_numbers)
self.a_of = {n: i for i, n in enumerate(self.numbers)}
self.state_cfg = state_cfg
self.qt = QTable(162, len(self.numbers), learning_rate=lr, discount_factor=gamma)
self.pol = UCBQTablePolicy(self.qt)
def choose(self, snap, tf, avail, greedy):
idx = state_index(snap, self.state_cfg)
if greedy:
q = self.qt.row(idx)
return max(avail, key=lambda c: q[self.a_of[c]])
mask = np.zeros(len(self.numbers), dtype=bool)
for c in avail:
mask[self.a_of[c]] = True
ctx = PolicyContext(state_index=idx, snapshot=snap, action_space_size=len(self.numbers),
episode=EpisodeState(), available_mask=mask)
return self.numbers[self.pol.select(ctx).action_id]
def learn(self, snap, tf, card, reward, next_snap, next_avail, done):
idx = state_index(snap, self.state_cfg)
nidx = state_index(next_snap, self.state_cfg) if (next_snap is not None and not done) else None
self.pol.update(Transition(state_index=idx, action_id=self.a_of[card], reward=reward,
next_state_index=nidx, done=done))
class RandomAdapter:
name = "random"
def __init__(self, seed=0):
self.rng = np.random.default_rng(seed)
def choose(self, snap, tf, avail, greedy):
return avail[self.rng.integers(len(avail))]
def learn(self, *a, **k):
pass
# ---- 공용 에피소드 -----------------------------------------------------------------
def run_episode(adapter, sup, tf, pool, strat, rc, seed, learn=True, greedy=False, forced_first=None):
buyer = FeatureBuyer(sup, strat, seed=seed, max_turns=MAX_TURNS)
scenario = Scenario(anchor_price=ANCHOR, target_price=TARGET, revenue_amount=sup.revenue_amount,
distribution_code=sup.distribution_code, partner_count=sup.partner_count)
price0 = TARGET * 1.15
price, used, total_r, first_card = price0, set(), 0.0, None
for turn in range(1, MAX_TURNS + 1):
acceptance = max(0.0, (price0 - price) / price0)
snap = make_snapshot(sup, price, turn, acceptance)
avail = [c for c in pool if c not in used] or list(pool)
if turn == 1 and forced_first is not None:
card = forced_first
else:
card = adapter.choose(snap, tf, avail, greedy)
used.add(card)
if first_card is None:
first_card = card
resp = buyer.respond(card, scenario, turn, price)
price = resp.new_price
done = resp.accept or price <= ANCHOR or turn >= MAX_TURNS
success = resp.accept or price <= ANCHOR
outcome = (NegotiationOutcome.SUCCESS if success
else NegotiationOutcome.FAILURE if done else NegotiationOutcome.ONGOING)
# 채점은 최종 결과 시점만 (중간 턴 0 → γ 부트스트랩으로 전파).
# 진행 중 보상을 누적하면 '질질 끄는 전략'이 부당하게 유리해지는 인공물이 생긴다.
r = rc.calculate(make_snapshot(sup, price, turn, acceptance, outcome)).total if done else 0.0
total_r += r
if learn:
if done:
adapter.learn(snap, tf, card, r, None, None, True)
else:
acc2 = max(0.0, (price0 - price) / price0)
nsnap = make_snapshot(sup, price, turn + 1, acc2)
navail = [c for c in pool if c not in used] or list(pool)
adapter.learn(snap, tf, card, r, nsnap, navail, False)
if done:
return total_r, success, price, first_card
return total_r, False, price, first_card
# ---- MC 정답 랭킹: 이 (협력사, 성향)에서 진짜 좋은 첫 카드 top-k ---------------------
_rand = RandomAdapter(seed=1)
def rank_cards_mc(sup, tf, pool, strat, rc, seed, sims=6, k=3):
means = {}
for c in pool:
rs = [run_episode(_rand, sup, tf, pool, strat, rc, seed=seed + 17 * s,
learn=False, greedy=False, forced_first=c)[0] for s in range(sims)]
means[c] = np.mean(rs)
return sorted(means, key=lambda c: -means[c])[:k]
# ---- 학습/평가 ---------------------------------------------------------------------
def train(adapter, pool, strat, base_reward, state_cfg, episodes, seed):
rng = np.random.default_rng(seed)
for ep in range(1, episodes + 1):
sup = sample_supplier(rng)
rcfg, tf = sample_tenant_pref(rng, base_reward)
rc = RewardCalculator(rcfg, state_cfg)
run_episode(adapter, sup, tf, pool, strat, rc, seed=seed * 100 + ep, learn=True)
def evaluate(adapter, pool, strat, base_reward, state_cfg, n=300, seed0=777, label=""):
from negotiation.qtable.domain.service.feature_builder import build_tenant_features
rng = np.random.default_rng(seed0)
rewards, succ, ratios, hits, holdout_first = [], 0, [], 0, 0
for i in range(n):
sup = sample_supplier(rng)
rcfg, tf = sample_tenant_pref(rng, base_reward)
rc = RewardCalculator(rcfg, state_cfg)
good = rank_cards_mc(sup, tf, pool, strat, rc, seed=seed0 * 7 + i)
r, ok, price, first = run_episode(adapter, sup, tf, pool, strat, rc,
seed=seed0 * 1000 + i, learn=False, greedy=True)
rewards.append(r); succ += ok; ratios.append(price / TARGET)
hits += (first in good); holdout_first += (first in HOLDOUT)
m, ci = float(np.mean(rewards)), float(1.96 * np.std(rewards) / np.sqrt(n))
print(f"{label:<14} mean_rwd={m:.4f} ±{ci:.4f} success={succ/n:.3f} "
f"settled/tgt={np.mean(ratios):.3f} top3_hit={hits/n:.3f} 새카드첫턴={holdout_first/n:.3f}")
def pref_behavior_test(adapters, pool, strat, base_reward, state_cfg):
"""④ 같은 협력사, 성향만 바꿨을 때 카드를 바꾸는가 (greedy).
첫 턴은 '일단 깎기'가 공통 정답이라 성향 차이가 잘 안 드러난다.
→ 협상 중반(가격이 이미 target 근처, 3턴째) 상태를 함께 프로브: 여기서
성사중시는 '마무리(수락 잘 되는) 카드', 가격중시는 '더 깎는 카드'가 갈려야 한다.
"""
from negotiation.qtable.domain.service.feature_builder import build_tenant_features
sups = [SupplierProfile(5_000_000, 3, "A"), # 소형·경쟁多
SupplierProfile(200_000_000, 1, "A")] # 대형·단독
probes = [("첫턴", TARGET * 1.15, 1, 0.0),
("중반(3턴,가격↓)", TARGET * 1.02, 3, 0.11)]
for pr_name, price, turn, acc in probes:
print(f"\n ── 프로브: {pr_name} (price={price:.0f}) ──")
print(f" {'협력사':<13} {'성향':<9} " + " ".join(f"{a.name:<15}" for a in adapters))
for sup in sups:
row = {}
for p, pname in [(0.05, "성사중시"), (0.95, "가격중시")]:
rcfg = pref_config(base_reward, p)
tf = build_tenant_features(rcfg)
picks = []
for a in adapters:
snap = make_snapshot(sup, price, turn, acc)
picks.append(a.choose(snap, tf, pool, True))
seg = f"{sup.segment[0]}·{sup.segment[1]}"
print(f" {seg:<13} {pname:<9} " + " ".join(f"{c}(전략{strat[c]})".ljust(15) for c in picks))
def main(episodes=10000, seed=42):
random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)
numbers, feat, strat = load_cards()
train_pool = [c for c in numbers if c not in HOLDOUT]
tcfg = TenantConfigLoader().load("ktcommerce")
card_dim = feat[numbers[0]].shape[0]
print(f"환경: 2축 FeatureBuyer + 성향 랜덤 · 학습 {episodes}ep · 카드특징 {card_dim}차원 "
f"(임베딩384+전략4+톤4) · 학습 {len(train_pool)}장 / 홀드아웃 {HOLDOUT}")
qt = QTableAdapter(numbers, tcfg.state)
dqn = DQNAdapter(FeatureDQNPolicy(state_dim=STATE_FEATURE_DIM + TENANT_FEATURE_DIM,
card_dim=card_dim, eps_decay=4000), feat)
print("\n[학습] qtable_ucb ...")
train(qt, train_pool, strat, tcfg.reward, tcfg.state, episodes, seed)
print("[학습] feature_dqn ...")
train(dqn, train_pool, strat, tcfg.reward, tcfg.state, episodes, seed)
print("\n=== ① 학습 카드 9장 풀 ===")
evaluate(RandomAdapter(seed), train_pool, strat, tcfg.reward, tcfg.state, label="random")
evaluate(qt, train_pool, strat, tcfg.reward, tcfg.state, label="qtable_ucb")
evaluate(dqn, train_pool, strat, tcfg.reward, tcfg.state, label="feature_dqn")
print("\n=== ② zero-shot 11장 풀 (안 본 카드 2장 포함) ===")
evaluate(RandomAdapter(seed), numbers, strat, tcfg.reward, tcfg.state, label="random")
evaluate(qt, numbers, strat, tcfg.reward, tcfg.state, label="qtable_ucb")
evaluate(dqn, numbers, strat, tcfg.reward, tcfg.state, label="feature_dqn")
print("\n=== ④ 성향 극단 테스트 — 같은 협력사, 성향만 바꾸면 카드를 바꾸는가 (11장 풀) ===")
pref_behavior_test([qt, dqn], numbers, strat, tcfg.reward, tcfg.state)
if __name__ == "__main__":
main()

79
agent/tools/export_dqn_serving.py Normal file
View File

@ -0,0 +1,79 @@
"""feature_dqn 체크포인트(.pt) → 서빙 번들(dqn_serving.npz) export.
서빙 컨테이너에 PyTorch 를 넣지 않기 위해 ScoreNet(3층 MLP) 가중치와 카드 특징
(임베딩384 + 전략 one-hot4 + 톤 one-hot4 = 392)을 numpy 번들 하나로 묶는다.
추론은 negotiation.policy.dqn_store 의 numpy forward 가 수행한다.
실행(호스트, torch 필요): APP_ENV=local python -m tools.export_dqn_serving
산출: agent/artifacts/dqn_serving.npz (.dockerignore 미제외 → 이미지에 포함)
"""
import os
import numpy as np
import torch
from tools.train_feature_dqn import load_cards
_HERE = os.path.dirname(os.path.abspath(__file__))
CKPT_PATH = os.path.join(_HERE, "..", "artifacts", "feature_dqn_ktcommerce.pt")
OUT_PATH = os.path.join(_HERE, "..", "artifacts", "dqn_serving.npz")
STATE_DIM = 14 # build_state_features(9) + build_tenant_features(5)
CARD_DIM = 392
def _np_forward(x, W0, b0, W1, b1, W2, b2):
h = np.maximum(x @ W0.T + b0, 0.0)
h = np.maximum(h @ W1.T + b1, 0.0)
return h @ W2.T + b2
def export_bundle(sd, out_path: str) -> str:
"""state_dict → 서빙 번들 npz (원자적 교체: .tmp 작성 후 replace). 반환: 절대경로.
retrain_from_logs 재학습 배포도 이 함수를 쓴다 — 검증(torch/numpy 일치)은 main() 전용.
"""
W0, b0 = sd["net.0.weight"].numpy(), sd["net.0.bias"].numpy()
W1, b1 = sd["net.2.weight"].numpy(), sd["net.2.bias"].numpy()
W2, b2 = sd["net.4.weight"].numpy(), sd["net.4.bias"].numpy()
assert W0.shape[1] == STATE_DIM + CARD_DIM, f"입력 차원 불일치: {W0.shape[1]}"
numbers, feat, _ = load_cards()
card_feats = np.stack([feat[n] for n in numbers]).astype(np.float32)
tmp = out_path + ".tmp"
with open(tmp, "wb") as f:
np.savez(
f,
W0=W0, b0=b0, W1=W1, b1=b1, W2=W2, b2=b2,
card_numbers=np.array(numbers), card_feats=card_feats,
state_dim=STATE_DIM, card_dim=CARD_DIM,
)
if os.path.exists(out_path):
os.replace(out_path, out_path + ".prev") # 직전 번들 백업(롤백용)
os.replace(tmp, out_path)
return os.path.abspath(out_path)
def main():
sd = torch.load(CKPT_PATH, map_location="cpu")
# 정합성 검증: torch forward == numpy forward
from negotiation.policies.feature_dqn_policy import ScoreNet
net = ScoreNet(STATE_DIM, CARD_DIM)
net.load_state_dict(sd)
net.eval()
x = np.random.default_rng(0).normal(size=(8, STATE_DIM + CARD_DIM)).astype(np.float32)
with torch.no_grad():
ref = net(torch.tensor(x)).numpy()
W0, b0 = sd["net.0.weight"].numpy(), sd["net.0.bias"].numpy()
W1, b1 = sd["net.2.weight"].numpy(), sd["net.2.bias"].numpy()
W2, b2 = sd["net.4.weight"].numpy(), sd["net.4.bias"].numpy()
out = _np_forward(x, W0, b0, W1, b1, W2, b2).squeeze(-1)
diff = float(np.abs(ref - out).max())
assert diff < 1e-4, f"numpy/torch forward 불일치: {diff}"
path = export_bundle(sd, OUT_PATH)
print(f"[저장] {path} forward 오차 {diff:.2e}")
if __name__ == "__main__":
main()

61
agent/tools/probe_serving_dqn.py Normal file
View File

@ -0,0 +1,61 @@
"""probe_serving_dqn — 서빙 번들(dqn_serving.npz)의 상황별 카드 선택 프로브.
배포된 모델이 '상황에 맞게' 고르는지 눈으로 확인하는 진단 도구:
협력사 세그먼트 × 고객사 성향 × 협상 국면(가격대)별 선택 카드를 표로 출력한다.
전부 다르길 기대하는 게 아니라, 축을 바꿨을 때 선택이 '움직이는지'를 본다.
실행: APP_ENV=local python -m tools.probe_serving_dqn (numpy 만 필요, DB 불필요)
"""
import numpy as np
from negotiation.qtable.domain.model.snapshot import NegotiationSnapshot
from negotiation.qtable.domain.service.feature_builder import build_state_features, build_tenant_features
from tenancy.config_loader import TenantConfigLoader
from tools.export_dqn_serving import OUT_PATH
from tools.retrain_from_logs import np_scorer_from_bundle
from tools.train_feature_dqn import pref_config
ANCHOR, TARGET = 495_000.0, 500_000.0 # BUGCHECK 견적과 동일 스케일
SUPPLIERS = {
"소형·경쟁多": dict(revenue_amount=5_000_000, partner_count=3, distribution_code="A"),
"소형·단독": dict(revenue_amount=5_000_000, partner_count=1, distribution_code="A"),
"대형·경쟁多": dict(revenue_amount=200_000_000, partner_count=3, distribution_code="A"),
"대형·단독": dict(revenue_amount=200_000_000, partner_count=1, distribution_code="A"),
}
PHASES = { # (라운드, 제시가): 첫턴 높은 가격 / 중반 목표가 근접 / 막판 앵커존 직전
"첫턴(575k)": (1, 575_000.0),
"중반(510k)": (2, 510_000.0),
"막판(501k)": (3, 501_000.0),
}
PREFS = {"성사중시": 0.1, "가격중시": 0.9}
def main():
score = np_scorer_from_bundle(OUT_PATH)
z = np.load(OUT_PATH, allow_pickle=False)
numbers = [str(n) for n in z["card_numbers"]]
feats = z["card_feats"]
base = TenantConfigLoader().load("ktcommerce").reward
for phase, (turn, price) in PHASES.items():
print(f"\n=== {phase} (앵커 {int(ANCHOR):,} / 목표 {int(TARGET):,}) ===")
print(f"{'협력사':<12}" + "".join(f"{p:>16}" for p in PREFS))
for sup_name, sup in SUPPLIERS.items():
row = []
for _, p in PREFS.items():
tf = build_tenant_features(pref_config(base, p))
snap = NegotiationSnapshot(
revenue_amount=sup["revenue_amount"], distribution_code=sup["distribution_code"],
partner_count=sup["partner_count"],
acceptance_ratio=max(0.0, (575_000.0 - price) / 575_000.0),
input_price=price, anchor_price=ANCHOR, target_price=TARGET, round_number=turn,
)
sf = np.concatenate([build_state_features(snap), tf])
row.append(numbers[int(np.argmax(score(sf, feats)))])
print(f"{sup_name:<12}" + "".join(f"{c:>16}" for c in row))
if __name__ == "__main__":
main()

238
agent/tools/retrain_from_logs.py Normal file
View File

@ -0,0 +1,238 @@
"""retrain_from_logs — experience_logs 실데이터로 feature_dqn 오프라인 재학습 + OPE 게이트.
파이프라인:
① learning.experience_logs 로드(전 테넌트 — 범용 에이전트는 테넌트를 특징으로 조건화하므로 통합 학습)
② 세션별 에피소드 재구성: 카드턴(done=False) N개 + 종료행(done=True) 1개.
보상은 학습 규약(최종 결과 시점만 채점)에 맞춰 종료행 reward 만 쓰고 중간턴은 0.
③ 현재 체크포인트에서 fine-tune (낮은 lr — 시뮬 사전학습 망각 방지)
④ OPE(SNIPS, 궤적 IS): 후보 모델 vs 현재 서빙 번들. 후보가 못 넘으면 배포하지 않는다.
⑤ 통과 시 dqn_serving.npz 원자적 교체(직전본 .prev 백업) → `docker compose build agent && up -d agent` 로 배포.
실행(호스트, torch+DB 필요):
APP_ENV=local python -m tools.retrain_from_logs
환경변수:
MIN_EPISODES(기본 200) 재학습 최소 에피소드 수 — 미달 시 skip (과적합 방지)
EPOCHS(기본 20) / LR(기본 1e-4) / FORCE_DEPLOY=1 (OPE 게이트 무시 — 테스트 전용)
주의: 서빙이 greedy(탐색 없음)라 로그가 선택 편향됨 — OPE 의 유효표본(ESS)이 작으면
게이트가 보수적으로 배포를 막는다. 이는 의도된 동작이다(조용한 성능저하 방지).
"""
import asyncio
import json
import os
from collections import defaultdict
import numpy as np
import torch
from common.database.db_session_manager import DB_SESSION_MNG
from common.database.model.models import ExperienceLog
from common.enums import DBType, DBWRType
from negotiation.policies.feature_dqn_policy import FeatureDQNPolicy
from negotiation.qtable.domain.model.snapshot import NegotiationSnapshot
from negotiation.qtable.domain.service.feature_builder import (
STATE_FEATURE_DIM, TENANT_FEATURE_DIM, build_state_features, build_tenant_features)
from sqlalchemy import select
from tenancy.config_loader import TenantConfigLoader
from tools.export_dqn_serving import CKPT_PATH, OUT_PATH, export_bundle
from tools.train_feature_dqn import load_cards
_HERE = os.path.dirname(os.path.abspath(__file__))
RETRAIN_CKPT = os.path.join(_HERE, "..", "artifacts", "feature_dqn_retrained.pt")
REPORT_PATH = os.path.join(_HERE, "..", "artifacts", "retrain_report.json")
MIN_EPISODES = int(os.getenv("MIN_EPISODES", "200"))
EPOCHS = int(os.getenv("EPOCHS", "20"))
LR = float(os.getenv("LR", "1e-4"))
FORCE_DEPLOY = os.getenv("FORCE_DEPLOY") == "1"
PROPENSITY_FALLBACK = 0.9 # 구로그 propensity 누락 시 (UCB/DQN 모두 greedy≈(1-ε)+ε/n)
# ---- ① 로그 로드 -------------------------------------------------------------
async def fetch_logs():
def _q(s):
q = (select(ExperienceLog.company_id, ExperienceLog.session_id, ExperienceLog.card_id,
ExperienceLog.reward, ExperienceLog.done, ExperienceLog.snapshot,
ExperienceLog.propensity, ExperienceLog.turn, ExperienceLog.id)
.where(ExperienceLog.is_invalidated == False) # noqa: E712
.order_by(ExperienceLog.company_id, ExperienceLog.session_id, ExperienceLog.id))
return DB_SESSION_MNG.execute(s, q)
err, rows = await DB_SESSION_MNG.execute_lambda(DBType.MAIN.value, DBWRType.DB_READ.value, _q)
return rows
# ---- ② 에피소드 재구성 --------------------------------------------------------
def build_episodes(rows, known_cards: set):
"""→ [{tenant, steps:[(snapshot, card, propensity)], terminal_reward}], 스킵 사유 카운트."""
by_session = defaultdict(list)
for r in rows:
if r[1] is not None:
by_session[(r[0], str(r[1]))].append(r)
episodes, skipped = [], defaultdict(int)
for (company_id, _sid), items in by_session.items():
selects = [r for r in items if not r[4] and r[5]] # done=False, snapshot 有
terminals = [r for r in items if r[4] and r[3] is not None] # done=True, reward 有
if not selects or not terminals:
skipped["종료행/카드턴 없음(미완결 세션)"] += 1
continue
if any(str(r[2] or "").startswith("AUT|") for r in selects):
skipped["완전 자율 세션(카드 재학습 대상 아님)"] += 1
continue
if any(r[2] not in known_cards for r in selects):
skipped["임베딩 없는 카드(파일매핑 테넌트 등)"] += 1
continue
episodes.append(dict(
tenant=company_id,
steps=[(r[5], r[2], r[6] if r[6] else PROPENSITY_FALLBACK) for r in selects],
terminal_reward=float(terminals[-1][3]),
))
return episodes, skipped
def tenant_feat_for(cache: dict, loader: TenantConfigLoader, company_id: str) -> np.ndarray:
"""테넌트 보상설정 → 성향 특징. 미온보딩/로드 실패는 _base 폴백."""
if company_id not in cache:
try:
cfg = loader.load(company_id)
except Exception:
cfg = loader.load("_base")
cache[company_id] = build_tenant_features(cfg.reward)
return cache[company_id]
def to_transitions(episodes, feat, tenant_feats):
"""학습 규약(train_feature_dqn 과 동일): 중간턴 r=0, 종료턴만 terminal_reward. 다음 후보 = 전체 − 사용분."""
all_cards = list(feat.keys())
out = []
for ep in episodes:
tf = tenant_feats[ep["tenant"]]
used = set()
n = len(ep["steps"])
for i, (snap_d, card, _p) in enumerate(ep["steps"]):
sf = np.concatenate([build_state_features(NegotiationSnapshot.from_dict(snap_d)), tf])
used.add(card)
if i == n - 1:
out.append((sf, feat[card], ep["terminal_reward"], None, None, True))
else:
s2_d = ep["steps"][i + 1][0]
s2 = np.concatenate([build_state_features(NegotiationSnapshot.from_dict(s2_d)), tf])
cands = [c for c in all_cards if c not in used] or all_cards
out.append((sf, feat[card], 0.0, s2, np.stack([feat[c] for c in cands]), False))
return out
# ---- ④ OPE (SNIPS, 궤적 단위 IS) ----------------------------------------------
def _greedy_match(score_fn, ep, feat, tf) -> float:
"""궤적 IS 가중치: Π 1[greedy(sᵢ)=aᵢ]/pᵢ. 한 턴이라도 불일치면 0."""
all_cards = list(feat.keys())
w, used = 1.0, set()
for snap_d, card, p in ep["steps"]:
sf = np.concatenate([build_state_features(NegotiationSnapshot.from_dict(snap_d)), tf])
cands = [c for c in all_cards if c not in used] or all_cards
sc = score_fn(sf, np.stack([feat[c] for c in cands]))
if cands[int(np.argmax(sc))] != card:
return 0.0
w /= max(p, 1e-3)
used.add(card)
return w
def snips(score_fn, episodes, feat, tenant_feats):
"""SNIPS 추정치 + 유효표본크기(ESS). 매치 0건이면 (None, 0)."""
ws, rs = [], []
for ep in episodes:
w = _greedy_match(score_fn, ep, feat, tenant_feats[ep["tenant"]])
ws.append(w)
rs.append(ep["terminal_reward"])
ws, rs = np.array(ws), np.array(rs)
if ws.sum() <= 0:
return None, 0.0
est = float((ws * rs).sum() / ws.sum())
ess = float(ws.sum() ** 2 / (ws ** 2).sum())
return est, ess
def np_scorer_from_bundle(path):
"""현재 서빙 번들(npz) → score_fn (dqn_store 와 동일 forward)."""
z = np.load(path, allow_pickle=False)
W0, b0, W1, b1, W2, b2 = z["W0"], z["b0"], z["W1"], z["b1"], z["W2"], z["b2"]
def score(sf, card_feats):
x = np.concatenate([np.repeat(sf[None, :], card_feats.shape[0], axis=0), card_feats], axis=1)
h = np.maximum(x @ W0.T + b0, 0.0)
h = np.maximum(h @ W1.T + b1, 0.0)
return (h @ W2.T + b2).squeeze(-1)
return score
# ---- 메인 ---------------------------------------------------------------------
async def run():
numbers, feat, _ = load_cards()
rows = await fetch_logs()
episodes, skipped = build_episodes(rows, set(numbers))
print(f"로그 {len(rows)}행 → 에피소드 {len(episodes)}개 (스킵: {dict(skipped) or '없음'})")
report = dict(rows=len(rows), episodes=len(episodes), skipped=dict(skipped),
min_episodes=MIN_EPISODES, deployed=False)
if len(episodes) < MIN_EPISODES and not FORCE_DEPLOY:
print(f"[skip] 에피소드 {len(episodes)} < MIN_EPISODES {MIN_EPISODES} — 과적합 위험으로 재학습 안 함")
report["result"] = "skipped_insufficient_data"
return report
loader = TenantConfigLoader()
tenant_feats = {}
for ep in episodes:
tenant_feat_for(tenant_feats, loader, ep["tenant"])
# ③ fine-tune (시뮬 사전학습 체크포인트에서 이어서, 낮은 lr)
transitions = to_transitions(episodes, feat, tenant_feats)
batch = min(64, max(8, len(transitions) // 4))
policy = FeatureDQNPolicy(state_dim=STATE_FEATURE_DIM + TENANT_FEATURE_DIM,
card_dim=feat[numbers[0]].shape[0], lr=LR, batch_size=batch)
if os.path.exists(CKPT_PATH):
policy.load(CKPT_PATH)
print(f"[fine-tune] 시작점: {os.path.basename(CKPT_PATH)} lr={LR} batch={batch}")
policy.buf.extend(transitions)
steps = EPOCHS * max(1, len(transitions) // batch)
losses = [l for _ in range(steps) if (l := policy.train_step()) is not None]
print(f"[fine-tune] {steps} step loss {losses[0]:.4f} → {losses[-1]:.4f}" if losses else "[fine-tune] 스텝 없음")
# ④ OPE 게이트: 후보 vs 현재 서빙
def cand_score(sf, cf):
return policy.scores(sf, cf)
cand_est, cand_ess = snips(cand_score, episodes, feat, tenant_feats)
cur_est, cur_ess = (snips(np_scorer_from_bundle(OUT_PATH), episodes, feat, tenant_feats)
if os.path.exists(OUT_PATH) else (None, 0.0))
print(f"[OPE/SNIPS] 후보 {cand_est} (ESS {cand_ess:.1f}) vs 현재 {cur_est} (ESS {cur_ess:.1f})")
report.update(ope_candidate=cand_est, ope_candidate_ess=cand_ess,
ope_current=cur_est, ope_current_ess=cur_ess)
min_ess = max(3.0, 0.02 * len(episodes))
passed = (cand_est is not None and cand_ess >= min_ess
and (cur_est is None or cand_est >= cur_est - 0.01))
if not passed and not FORCE_DEPLOY:
print(f"[게이트 불통과] 배포하지 않음 (필요 ESS ≥ {min_ess:.1f}). 현재 번들 유지.")
report["result"] = "gate_failed"
return report
# ⑤ 배포: 후보 저장 + 번들 교체 (.prev 백업)
policy.save(RETRAIN_CKPT)
path = export_bundle(policy.q.state_dict(), OUT_PATH)
print(f"[배포] {path} (직전본 → dqn_serving.npz.prev)")
print(" 적용: docker compose build agent && docker compose up -d agent")
report.update(result="deployed" if passed else "force_deployed", deployed=True,
ckpt=os.path.abspath(RETRAIN_CKPT))
return report
def main():
report = asyncio.run(run())
with open(REPORT_PATH, "w", encoding="utf-8") as f:
json.dump(report, f, ensure_ascii=False, indent=2)
print(f"[리포트] {os.path.abspath(REPORT_PATH)}")
if __name__ == "__main__":
main()

148
agent/tools/train_feature_dqn.py Normal file
View File

@ -0,0 +1,148 @@
"""action-as-feature DQN 학습 (Phase 2·3) — 공용 환경 헬퍼 + 단독 학습 엔트리.
카드 특징 = 스크립트 임베딩(384) + 전략 one-hot(4) + 톤 one-hot(4) = 392차원
상태 특징 = 연속 상태(9) + 고객사 성향(5) = 14차원 ← 협력사·고객사 조건화
학습 환경 = FeatureBuyer(양보력/수락력 2축) + 에피소드마다 협력사·고객사성향 랜덤 샘플링
비교 평가는 tools.compare_qtable_vs_dqn 에서 수행한다.
실행: APP_ENV=local python -m tools.train_feature_dqn
"""
import os
import random
import numpy as np
import torch
from eval_harness.buyer import Scenario
from eval_harness.feature_buyer import FeatureBuyer, sample_supplier
from negotiation.policies.feature_dqn_policy import FeatureDQNPolicy
from negotiation.qtable.domain.model.snapshot import NegotiationOutcome, NegotiationSnapshot
from negotiation.qtable.domain.service.feature_builder import (
STATE_FEATURE_DIM, TENANT_FEATURE_DIM, build_state_features, build_tenant_features)
from negotiation.qtable.domain.service.reward_calculator import RewardCalculator
from tenancy.config_loader import TenantConfigLoader
_HERE = os.path.dirname(os.path.abspath(__file__))
EMB_PATH = os.path.join(_HERE, "..", "artifacts", "card_embeddings.npz")
CKPT_PATH = os.path.join(_HERE, "..", "artifacts", "feature_dqn_ktcommerce.pt")
# zero-shot 실험용 홀드아웃 (전략 1·4 — 남은 풀에도 같은 전략 존재).
# 서빙용 최종 학습은 전체 풀 사용: FULL_POOL=1 python -m tools.train_feature_dqn
HOLDOUT = [] if os.getenv("FULL_POOL") == "1" else ["NGC-002", "NGC-010"]
ANCHOR, TARGET = 8000.0, 10000.0
MAX_TURNS = 5
N_STRATEGY, N_TONE = 4, 4
# ---- 카드 특징: 임베딩 + 전략/톤 one-hot ------------------------------------------
def load_cards():
z = np.load(EMB_PATH, allow_pickle=True)
numbers = [str(n) for n in z["numbers"]]
feat, strat = {}, {}
for i, n in enumerate(numbers):
s, t = int(z["strategy"][i]), int(z["tone"][i])
s_oh = np.eye(N_STRATEGY, dtype=np.float32)[s - 1]
t_oh = np.eye(N_TONE, dtype=np.float32)[t - 1]
feat[n] = np.concatenate([z["embeddings"][i].astype(np.float32), s_oh, t_oh])
strat[n] = s
return numbers, feat, strat
# ---- 고객사 성향: 보상 설정 샘플링 ---------------------------------------------------
def sample_tenant_pref(rng: np.random.Generator, base_cfg):
"""p ∈ [0,1]: 0=성사중시(협력 유리) ↔ 1=가격중시(경쟁 유리). 반환: (RewardConfig, tenant_feat)."""
p = float(rng.uniform(0.0, 1.0))
cfg = base_cfg.model_copy(update=dict(
max_weight=0.25 + 0.60 * p, # 가격보상 비중
min_weight=(0.25 + 0.60 * p) * 0.7,
success_reward=1.6 - 1.2 * p, # 성사중시일수록 성공보상↑
failure_penalty=-(1.4 - 1.1 * p), # 성사중시일수록 결렬이 아픔
beta=0.1 + 0.4 * p,
penalty_lambda=float(rng.uniform(0.005, 0.05)),
))
return cfg, build_tenant_features(cfg)
def pref_config(base_cfg, p: float, lam: float = 0.02):
"""평가용: 성향 p 를 고정해 RewardConfig 생성 (극단 테스트)."""
return base_cfg.model_copy(update=dict(
max_weight=0.25 + 0.60 * p, min_weight=(0.25 + 0.60 * p) * 0.7,
success_reward=1.6 - 1.2 * p, failure_penalty=-(1.4 - 1.1 * p),
beta=0.1 + 0.4 * p, penalty_lambda=lam,
))
def make_snapshot(sup, price: float, turn: int, acceptance: float,
outcome=NegotiationOutcome.ONGOING) -> NegotiationSnapshot:
return NegotiationSnapshot(
revenue_amount=sup.revenue_amount, distribution_code=sup.distribution_code,
partner_count=sup.partner_count, acceptance_ratio=acceptance,
input_price=price, anchor_price=ANCHOR, target_price=TARGET,
round_number=turn, outcome=outcome,
)
# ---- 단독 학습 엔트리 (비교는 compare_qtable_vs_dqn) --------------------------------
def main(episodes=10000, seed=42):
random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)
numbers, feat, strat = load_cards()
train_pool = [c for c in numbers if c not in HOLDOUT]
card_dim = feat[numbers[0]].shape[0]
print(f"카드 {len(numbers)}장 (학습 {len(train_pool)} / 홀드아웃 {HOLDOUT}) card_dim={card_dim}")
tcfg = TenantConfigLoader().load("ktcommerce")
policy = FeatureDQNPolicy(state_dim=STATE_FEATURE_DIM + TENANT_FEATURE_DIM,
card_dim=card_dim, eps_decay=4000, gamma=0.95)
rng = np.random.default_rng(seed)
print(f"=== 학습 {episodes} 에피소드 (협력사·성향 랜덤, CPU) ===")
recent = []
for ep in range(1, episodes + 1):
sup = sample_supplier(rng)
rcfg, tf = sample_tenant_pref(rng, tcfg.reward)
rc = RewardCalculator(rcfg, tcfg.state)
buyer = FeatureBuyer(sup, strat, seed=seed * 100 + ep, max_turns=MAX_TURNS)
scenario = Scenario(anchor_price=ANCHOR, target_price=TARGET, revenue_amount=sup.revenue_amount,
distribution_code=sup.distribution_code, partner_count=sup.partner_count)
price0 = TARGET * 1.15
price, used, total_r = price0, set(), 0.0
for turn in range(1, MAX_TURNS + 1):
acceptance = max(0.0, (price0 - price) / price0)
s = make_snapshot(sup, price, turn, acceptance)
sf = np.concatenate([build_state_features(s), tf])
avail = [c for c in train_pool if c not in used] or list(train_pool)
embs = np.stack([feat[c] for c in avail])
i, _, _ = policy.select(sf, embs)
card = avail[i]; used.add(card)
resp = buyer.respond(card, scenario, turn, price)
price = resp.new_price
done = resp.accept or price <= ANCHOR or turn >= MAX_TURNS
success = resp.accept or price <= ANCHOR
outcome = (NegotiationOutcome.SUCCESS if success
else NegotiationOutcome.FAILURE if done else NegotiationOutcome.ONGOING)
# 최종 결과 시점만 채점 (중간 0 → γ 부트스트랩) — compare 스크립트와 동일 규칙.
r = rc.calculate(make_snapshot(sup, price, turn, acceptance, outcome)).total if done else 0.0
total_r += r
if done:
policy.remember(sf, feat[card], r, None, None, True)
else:
acc2 = max(0.0, (price0 - price) / price0)
s2 = make_snapshot(sup, price, turn + 1, acc2)
navail = [c for c in train_pool if c not in used] or list(train_pool)
policy.remember(sf, feat[card], r, np.concatenate([build_state_features(s2), tf]),
np.stack([feat[c] for c in navail]), False)
policy.train_step()
if done:
break
recent.append(total_r)
if ep % 2000 == 0:
print(f" ep {ep:>6} eps={policy.eps():.3f} 최근2000 평균보상={np.mean(recent[-2000:]):.4f}")
policy.save(CKPT_PATH)
print(f"[저장] {CKPT_PATH}")
return policy
if __name__ == "__main__":
main()