o2o-negosium-original/agent/tools/train_feature_dqn.py

"""action-as-feature DQN 학습 (Phase 2·3) — 공용 환경 헬퍼 + 단독 학습 엔트리.

카드 특징 = 스크립트 임베딩(384) + 전략 one-hot(4) + 톤 one-hot(4) = 392차원
상태 특징 = 연속 상태(9) + 고객사 성향(5) = 14차원   ← 협력사·고객사 조건화
학습 환경 = FeatureBuyer(양보력/수락력 2축) + 에피소드마다 협력사·고객사성향 랜덤 샘플링

비교 평가는 tools.compare_qtable_vs_dqn 에서 수행한다.
실행:  APP_ENV=local python -m tools.train_feature_dqn
"""

import os
import random

import numpy as np
import torch

from eval_harness.buyer import Scenario
from eval_harness.feature_buyer import FeatureBuyer, sample_supplier
from negotiation.policies.feature_dqn_policy import FeatureDQNPolicy
from negotiation.qtable.domain.model.snapshot import NegotiationOutcome, NegotiationSnapshot
from negotiation.qtable.domain.service.feature_builder import (
    STATE_FEATURE_DIM, TENANT_FEATURE_DIM, build_state_features, build_tenant_features)
from negotiation.qtable.domain.service.reward_calculator import RewardCalculator
from tenancy.config_loader import TenantConfigLoader

_HERE = os.path.dirname(os.path.abspath(__file__))
EMB_PATH = os.path.join(_HERE, "..", "artifacts", "card_embeddings.npz")
CKPT_PATH = os.path.join(_HERE, "..", "artifacts", "feature_dqn_ktcommerce.pt")

# zero-shot 실험용 홀드아웃 (전략 1·4 — 남은 풀에도 같은 전략 존재).
# 서빙용 최종 학습은 전체 풀 사용: FULL_POOL=1 python -m tools.train_feature_dqn
HOLDOUT = [] if os.getenv("FULL_POOL") == "1" else ["NGC-002", "NGC-010"]
ANCHOR, TARGET = 8000.0, 10000.0
MAX_TURNS = 5
N_STRATEGY, N_TONE = 4, 4


# ---- 카드 특징: 임베딩 + 전략/톤 one-hot ------------------------------------------
def load_cards():
    z = np.load(EMB_PATH, allow_pickle=True)
    numbers = [str(n) for n in z["numbers"]]
    feat, strat = {}, {}
    for i, n in enumerate(numbers):
        s, t = int(z["strategy"][i]), int(z["tone"][i])
        s_oh = np.eye(N_STRATEGY, dtype=np.float32)[s - 1]
        t_oh = np.eye(N_TONE, dtype=np.float32)[t - 1]
        feat[n] = np.concatenate([z["embeddings"][i].astype(np.float32), s_oh, t_oh])
        strat[n] = s
    return numbers, feat, strat


# ---- 고객사 성향: 보상 설정 샘플링 ---------------------------------------------------
def sample_tenant_pref(rng: np.random.Generator, base_cfg):
    """p ∈ [0,1]: 0=성사중시(협력 유리) ↔ 1=가격중시(경쟁 유리). 반환: (RewardConfig, tenant_feat)."""
    p = float(rng.uniform(0.0, 1.0))
    cfg = base_cfg.model_copy(update=dict(
        max_weight=0.25 + 0.60 * p,               # 가격보상 비중
        min_weight=(0.25 + 0.60 * p) * 0.7,
        success_reward=1.6 - 1.2 * p,             # 성사중시일수록 성공보상↑
        failure_penalty=-(1.4 - 1.1 * p),         # 성사중시일수록 결렬이 아픔
        beta=0.1 + 0.4 * p,
        penalty_lambda=float(rng.uniform(0.005, 0.05)),
    ))
    return cfg, build_tenant_features(cfg)


def pref_config(base_cfg, p: float, lam: float = 0.02):
    """평가용: 성향 p 를 고정해 RewardConfig 생성 (극단 테스트)."""
    return base_cfg.model_copy(update=dict(
        max_weight=0.25 + 0.60 * p, min_weight=(0.25 + 0.60 * p) * 0.7,
        success_reward=1.6 - 1.2 * p, failure_penalty=-(1.4 - 1.1 * p),
        beta=0.1 + 0.4 * p, penalty_lambda=lam,
    ))


def make_snapshot(sup, price: float, turn: int, acceptance: float,
                  outcome=NegotiationOutcome.ONGOING) -> NegotiationSnapshot:
    return NegotiationSnapshot(
        revenue_amount=sup.revenue_amount, distribution_code=sup.distribution_code,
        partner_count=sup.partner_count, acceptance_ratio=acceptance,
        input_price=price, anchor_price=ANCHOR, target_price=TARGET,
        round_number=turn, outcome=outcome,
    )


# ---- 단독 학습 엔트리 (비교는 compare_qtable_vs_dqn) --------------------------------
def main(episodes=10000, seed=42):
    random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)
    numbers, feat, strat = load_cards()
    train_pool = [c for c in numbers if c not in HOLDOUT]
    card_dim = feat[numbers[0]].shape[0]
    print(f"카드 {len(numbers)}장 (학습 {len(train_pool)} / 홀드아웃 {HOLDOUT})  card_dim={card_dim}")

    tcfg = TenantConfigLoader().load("ktcommerce")
    policy = FeatureDQNPolicy(state_dim=STATE_FEATURE_DIM + TENANT_FEATURE_DIM,
                              card_dim=card_dim, eps_decay=4000, gamma=0.95)
    rng = np.random.default_rng(seed)

    print(f"=== 학습 {episodes} 에피소드 (협력사·성향 랜덤, CPU) ===")
    recent = []
    for ep in range(1, episodes + 1):
        sup = sample_supplier(rng)
        rcfg, tf = sample_tenant_pref(rng, tcfg.reward)
        rc = RewardCalculator(rcfg, tcfg.state)
        buyer = FeatureBuyer(sup, strat, seed=seed * 100 + ep, max_turns=MAX_TURNS)
        scenario = Scenario(anchor_price=ANCHOR, target_price=TARGET, revenue_amount=sup.revenue_amount,
                            distribution_code=sup.distribution_code, partner_count=sup.partner_count)
        price0 = TARGET * 1.15
        price, used, total_r = price0, set(), 0.0
        for turn in range(1, MAX_TURNS + 1):
            acceptance = max(0.0, (price0 - price) / price0)
            s = make_snapshot(sup, price, turn, acceptance)
            sf = np.concatenate([build_state_features(s), tf])
            avail = [c for c in train_pool if c not in used] or list(train_pool)
            embs = np.stack([feat[c] for c in avail])
            i, _, _ = policy.select(sf, embs)
            card = avail[i]; used.add(card)
            resp = buyer.respond(card, scenario, turn, price)
            price = resp.new_price
            done = resp.accept or price <= ANCHOR or turn >= MAX_TURNS
            success = resp.accept or price <= ANCHOR
            outcome = (NegotiationOutcome.SUCCESS if success
                       else NegotiationOutcome.FAILURE if done else NegotiationOutcome.ONGOING)
            # 최종 결과 시점만 채점 (중간 0 → γ 부트스트랩) — compare 스크립트와 동일 규칙.
            r = rc.calculate(make_snapshot(sup, price, turn, acceptance, outcome)).total if done else 0.0
            total_r += r
            if done:
                policy.remember(sf, feat[card], r, None, None, True)
            else:
                acc2 = max(0.0, (price0 - price) / price0)
                s2 = make_snapshot(sup, price, turn + 1, acc2)
                navail = [c for c in train_pool if c not in used] or list(train_pool)
                policy.remember(sf, feat[card], r, np.concatenate([build_state_features(s2), tf]),
                                np.stack([feat[c] for c in navail]), False)
            policy.train_step()
            if done:
                break
        recent.append(total_r)
        if ep % 2000 == 0:
            print(f"  ep {ep:>6}  eps={policy.eps():.3f}  최근2000 평균보상={np.mean(recent[-2000:]):.4f}")

    policy.save(CKPT_PATH)
    print(f"[저장] {CKPT_PATH}")
    return policy


if __name__ == "__main__":
    main()