"""action-as-feature DQN 학습 (Phase 2·3) — 공용 환경 헬퍼 + 단독 학습 엔트리. 카드 특징 = 스크립트 임베딩(384) + 전략 one-hot(4) + 톤 one-hot(4) = 392차원 상태 특징 = 연속 상태(9) + 고객사 성향(5) = 14차원 ← 협력사·고객사 조건화 학습 환경 = FeatureBuyer(양보력/수락력 2축) + 에피소드마다 협력사·고객사성향 랜덤 샘플링 비교 평가는 tools.compare_qtable_vs_dqn 에서 수행한다. 실행: APP_ENV=local python -m tools.train_feature_dqn """ import os import random import numpy as np import torch from eval_harness.buyer import Scenario from eval_harness.feature_buyer import FeatureBuyer, sample_supplier from negotiation.policies.feature_dqn_policy import FeatureDQNPolicy from negotiation.qtable.domain.model.snapshot import NegotiationOutcome, NegotiationSnapshot from negotiation.qtable.domain.service.feature_builder import ( STATE_FEATURE_DIM, TENANT_FEATURE_DIM, build_state_features, build_tenant_features) from negotiation.qtable.domain.service.reward_calculator import RewardCalculator from tenancy.config_loader import TenantConfigLoader _HERE = os.path.dirname(os.path.abspath(__file__)) EMB_PATH = os.path.join(_HERE, "..", "artifacts", "card_embeddings.npz") CKPT_PATH = os.path.join(_HERE, "..", "artifacts", "feature_dqn_ktcommerce.pt") # zero-shot 실험용 홀드아웃 (전략 1·4 — 남은 풀에도 같은 전략 존재). # 서빙용 최종 학습은 전체 풀 사용: FULL_POOL=1 python -m tools.train_feature_dqn HOLDOUT = [] if os.getenv("FULL_POOL") == "1" else ["NGC-002", "NGC-010"] ANCHOR, TARGET = 8000.0, 10000.0 MAX_TURNS = 5 N_STRATEGY, N_TONE = 4, 4 # ---- 카드 특징: 임베딩 + 전략/톤 one-hot ------------------------------------------ def load_cards(): z = np.load(EMB_PATH, allow_pickle=True) numbers = [str(n) for n in z["numbers"]] feat, strat = {}, {} for i, n in enumerate(numbers): s, t = int(z["strategy"][i]), int(z["tone"][i]) s_oh = np.eye(N_STRATEGY, dtype=np.float32)[s - 1] t_oh = np.eye(N_TONE, dtype=np.float32)[t - 1] feat[n] = np.concatenate([z["embeddings"][i].astype(np.float32), s_oh, t_oh]) strat[n] = s return numbers, feat, strat # ---- 고객사 성향: 보상 설정 샘플링 --------------------------------------------------- def sample_tenant_pref(rng: np.random.Generator, base_cfg): """p ∈ [0,1]: 0=성사중시(협력 유리) ↔ 1=가격중시(경쟁 유리). 반환: (RewardConfig, tenant_feat).""" p = float(rng.uniform(0.0, 1.0)) cfg = base_cfg.model_copy(update=dict( max_weight=0.25 + 0.60 * p, # 가격보상 비중 min_weight=(0.25 + 0.60 * p) * 0.7, success_reward=1.6 - 1.2 * p, # 성사중시일수록 성공보상↑ failure_penalty=-(1.4 - 1.1 * p), # 성사중시일수록 결렬이 아픔 beta=0.1 + 0.4 * p, penalty_lambda=float(rng.uniform(0.005, 0.05)), )) return cfg, build_tenant_features(cfg) def pref_config(base_cfg, p: float, lam: float = 0.02): """평가용: 성향 p 를 고정해 RewardConfig 생성 (극단 테스트).""" return base_cfg.model_copy(update=dict( max_weight=0.25 + 0.60 * p, min_weight=(0.25 + 0.60 * p) * 0.7, success_reward=1.6 - 1.2 * p, failure_penalty=-(1.4 - 1.1 * p), beta=0.1 + 0.4 * p, penalty_lambda=lam, )) def make_snapshot(sup, price: float, turn: int, acceptance: float, outcome=NegotiationOutcome.ONGOING) -> NegotiationSnapshot: return NegotiationSnapshot( revenue_amount=sup.revenue_amount, distribution_code=sup.distribution_code, partner_count=sup.partner_count, acceptance_ratio=acceptance, input_price=price, anchor_price=ANCHOR, target_price=TARGET, round_number=turn, outcome=outcome, ) # ---- 단독 학습 엔트리 (비교는 compare_qtable_vs_dqn) -------------------------------- def main(episodes=10000, seed=42): random.seed(seed); np.random.seed(seed); torch.manual_seed(seed) numbers, feat, strat = load_cards() train_pool = [c for c in numbers if c not in HOLDOUT] card_dim = feat[numbers[0]].shape[0] print(f"카드 {len(numbers)}장 (학습 {len(train_pool)} / 홀드아웃 {HOLDOUT}) card_dim={card_dim}") tcfg = TenantConfigLoader().load("ktcommerce") policy = FeatureDQNPolicy(state_dim=STATE_FEATURE_DIM + TENANT_FEATURE_DIM, card_dim=card_dim, eps_decay=4000, gamma=0.95) rng = np.random.default_rng(seed) print(f"=== 학습 {episodes} 에피소드 (협력사·성향 랜덤, CPU) ===") recent = [] for ep in range(1, episodes + 1): sup = sample_supplier(rng) rcfg, tf = sample_tenant_pref(rng, tcfg.reward) rc = RewardCalculator(rcfg, tcfg.state) buyer = FeatureBuyer(sup, strat, seed=seed * 100 + ep, max_turns=MAX_TURNS) scenario = Scenario(anchor_price=ANCHOR, target_price=TARGET, revenue_amount=sup.revenue_amount, distribution_code=sup.distribution_code, partner_count=sup.partner_count) price0 = TARGET * 1.15 price, used, total_r = price0, set(), 0.0 for turn in range(1, MAX_TURNS + 1): acceptance = max(0.0, (price0 - price) / price0) s = make_snapshot(sup, price, turn, acceptance) sf = np.concatenate([build_state_features(s), tf]) avail = [c for c in train_pool if c not in used] or list(train_pool) embs = np.stack([feat[c] for c in avail]) i, _, _ = policy.select(sf, embs) card = avail[i]; used.add(card) resp = buyer.respond(card, scenario, turn, price) price = resp.new_price done = resp.accept or price <= ANCHOR or turn >= MAX_TURNS success = resp.accept or price <= ANCHOR outcome = (NegotiationOutcome.SUCCESS if success else NegotiationOutcome.FAILURE if done else NegotiationOutcome.ONGOING) # 최종 결과 시점만 채점 (중간 0 → γ 부트스트랩) — compare 스크립트와 동일 규칙. r = rc.calculate(make_snapshot(sup, price, turn, acceptance, outcome)).total if done else 0.0 total_r += r if done: policy.remember(sf, feat[card], r, None, None, True) else: acc2 = max(0.0, (price0 - price) / price0) s2 = make_snapshot(sup, price, turn + 1, acc2) navail = [c for c in train_pool if c not in used] or list(train_pool) policy.remember(sf, feat[card], r, np.concatenate([build_state_features(s2), tf]), np.stack([feat[c] for c in navail]), False) policy.train_step() if done: break recent.append(total_r) if ep % 2000 == 0: print(f" ep {ep:>6} eps={policy.eps():.3f} 최근2000 평균보상={np.mean(recent[-2000:]):.4f}") policy.save(CKPT_PATH) print(f"[저장] {CKPT_PATH}") return policy if __name__ == "__main__": main()