"""QTablePolicyStore — learning 스키마에서 테넌트 UCBQTablePolicy 를 로드/영속화 (H1).

요청마다 활성 버전의 q_values/visit_counts 를 numpy QTable 로 적재해 정책을 조립한다(상태는 DB 가
단일 소스). 갱신은 touched 셀만 write-through → 재시작/요청 간 학습이 보존된다.

PoC 단순화: 온라인 single-step 갱신(다음상태 부트스트랩은 outcome 종료 시 생략). 시퀀스 보상링크는
P5/H 트랙에서 transition_id 기반으로 정교화한다.
"""

import math
from typing import Tuple

from negotiation.policies.qtable_policy import UCBQTablePolicy
from negotiation.qtable.domain.model.q_table import QTable
from negotiation.qtable.infra.repository.learning_repository import LearningRepository
from tenancy.registry import TenantEngine


class QTablePolicyStore:
    @staticmethod
    async def load(engine: TenantEngine) -> Tuple[UCBQTablePolicy, object, LearningRepository]:
        repo = LearningRepository(engine.company_id)
        S = engine.state_space_size
        A = engine.action_space_size
        pol_cfg = engine.config.policy
        lr = pol_cfg.learning_rate
        gamma = pol_cfg.gamma

        # cold-start 3단 (계획서 D):
        #  ① 활성 버전 있으면 그대로  ② 없고 inherits_base 면 base warm-start 복제(차원 호환 시)
        #  ③ 차원 불일치/base 없음 → 휴리스틱 빈 버전
        err, active = await repo.read(lambda s: repo.get_active_version(s))
        if active is not None:
            version_id = active.version_id
        else:
            version_id = None
            if engine.config.inherits_base:
                version_id = await repo.warm_start_from_base(
                    state_space_size=S, action_space_size=A, learning_rate=lr, discount_factor=gamma)
            if version_id is None:
                version_id = await repo.get_or_create_active_version(
                    state_space_size=S, action_space_size=A, learning_rate=lr, discount_factor=gamma)

        qtable = QTable(S, A, learning_rate=lr, discount_factor=gamma)
        qrows, vrows = await repo.load_cells(version_id)
        for st, a, q in qrows:
            if 0 <= st < S and 0 <= a < A:
                qtable.q[st, a] = q
        for st, a, c in vrows:
            if 0 <= st < S and 0 <= a < A:
                qtable.visits[st, a] = c

        params = pol_cfg.params or {}
        policy = UCBQTablePolicy(
            qtable,
            exploration_constant=params.get("exploration_constant", math.sqrt(2.0)),
            epsilon=params.get("propensity_epsilon", 0.1),
        )
        return policy, version_id, repo

    @staticmethod
    async def persist_cell(repo: LearningRepository, version_id, policy: UCBQTablePolicy,
                           state_index: int, action_id: int):
        """(state, action) 셀 write-through (select 의 visit 증가 + update 의 Q 변화 반영)."""
        q_value = float(policy.qtable.q[state_index, action_id])
        count = int(policy.qtable.visits[state_index, action_id])
        await repo.upsert_cell(version_id, state_index, action_id, q_value, count)