Deep CFR playability encoding 추가

src/coolrl_lost_cities/games/classic/deep_cfr/encoding.pyx

@@ -4,7 +4,12 @@
 from coolrl_lost_cities.games.classic.game cimport GameState
 
 
-cdef int input_dim_c(GameState state) noexcept:
+cdef int DERIVED_PLAYABILITY_PER_COLOR = 19
+cdef int DERIVED_PLAYABILITY_COMMON = 3
+cdef int SLOT_AWARE_PLAYABILITY_PER_SLOT = 12
+
+
+cdef int _base_input_dim_c(GameState state) noexcept:
     cdef int action_size = 2 * state.hand_size + 1 + state.n_colors
     cdef int card_type_size = state.n_colors * (state.n_ranks + 1)
     return (
@@ -20,7 +25,274 @@ cdef int input_dim_c(GameState state) noexcept:
     )
 
 
+cdef int input_dim_c(GameState state) noexcept:
+    return _input_dim_with_flags_c(state, False, False)
+
+
+cdef int _input_dim_with_flags_c(GameState state, bint derived_playability, bint slot_aware_playability) noexcept:
+    cdef int total = _base_input_dim_c(state)
+    if derived_playability:
+        total += state.n_colors * DERIVED_PLAYABILITY_PER_COLOR + DERIVED_PLAYABILITY_COMMON
+    if slot_aware_playability:
+        total += state.hand_size * SLOT_AWARE_PLAYABILITY_PER_SLOT
+    return total
+
+
+cdef int _numeric_value_c(GameState state, int rank) noexcept:
+    if rank == 0:
+        return 0
+    return state.min_rank + rank - 1
+
+
+cdef float _max_numeric_sum_c(GameState state) noexcept:
+    return <float>(state.n_ranks * (2 * state.min_rank + state.n_ranks - 1)) / 2.0
+
+
+cdef float _max_score_estimate_c(GameState state) noexcept:
+    cdef float max_numeric_sum = _max_numeric_sum_c(state)
+    cdef float break_even = <float>abs(state.expedition_penalty)
+    cdef float estimate = (max_numeric_sum - break_even) * <float>(state.n_handshakes + 1)
+    if estimate < 1.0:
+        return 1.0
+    return estimate
+
+
+cdef void _color_playability_summary_c(
+    GameState state,
+    int player,
+    int color,
+    int* is_unopened,
+    int* has_only_wagers_opened,
+    int* current_numeric_sum,
+    int* current_wager_count,
+    int* current_expedition_len,
+    int* last_numeric_rank,
+    int* hand_count,
+    int* hand_wager_count,
+    int* playable_hand_wager_count,
+    int* playable_hand_numeric_sum,
+    int* playable_hand_numeric_count,
+    int* dead_hand_numeric_count,
+    int* dead_hand_numeric_sum,
+    int* recoverable_margin_no_bonus,
+    int* recoverable_score_no_bonus,
+    int* min_needed_to_break_even,
+    int* discard_top_playable_flag,
+    int* discard_top_playable_value,
+    int* unknown_remaining_count,
+    int* has_bonus_path,
+    int* cards_needed_for_bonus,
+) noexcept:
+    cdef int cache_index = state._expedition_len_index(player, color)
+    cdef int opponent_index = state._expedition_len_index(1 - player, color)
+    cdef int slot
+    cdef int card
+    cdef int rank
+    cdef int length = state.expedition_lens[cache_index]
+    cdef int projected_numeric_sum
+    cdef int projected_wager_count
+    cdef int projected_len
+    cdef int break_even = abs(state.expedition_penalty)
+    cdef int known_color_count
+    cdef int top_card
+
+    current_numeric_sum[0] = state.numeric_sums[cache_index]
+    current_wager_count[0] = state.handshake_counts[cache_index]
+    current_expedition_len[0] = length
+    last_numeric_rank[0] = state.last_numeric_ranks[cache_index]
+    is_unopened[0] = 1 if length == 0 else 0
+    has_only_wagers_opened[0] = 1 if length > 0 and last_numeric_rank[0] == 0 else 0
+
+    hand_count[0] = 0
+    hand_wager_count[0] = 0
+    playable_hand_wager_count[0] = 0
+    playable_hand_numeric_sum[0] = 0
+    playable_hand_numeric_count[0] = 0
+    dead_hand_numeric_count[0] = 0
+    dead_hand_numeric_sum[0] = 0
+
+    for slot in range(state.hand_lens[player]):
+        card = state.hand_cards[state._hand_index(player, slot)]
+        if state._card_color(card) != color:
+            continue
+        rank = state._card_rank(card)
+        hand_count[0] += 1
+        if rank == 0:
+            hand_wager_count[0] += 1
+            if last_numeric_rank[0] == 0:
+                playable_hand_wager_count[0] += 1
+        elif rank > last_numeric_rank[0]:
+            playable_hand_numeric_count[0] += 1
+            playable_hand_numeric_sum[0] += _numeric_value_c(state, rank)
+        else:
+            dead_hand_numeric_count[0] += 1
+            dead_hand_numeric_sum[0] += _numeric_value_c(state, rank)
+
+    projected_numeric_sum = current_numeric_sum[0] + playable_hand_numeric_sum[0]
+    projected_wager_count = current_wager_count[0] + playable_hand_wager_count[0]
+    recoverable_margin_no_bonus[0] = projected_numeric_sum - break_even
+    recoverable_score_no_bonus[0] = recoverable_margin_no_bonus[0] * (projected_wager_count + 1)
+    min_needed_to_break_even[0] = max(0, break_even - projected_numeric_sum)
+    projected_len = length + playable_hand_numeric_count[0] + playable_hand_wager_count[0]
+    has_bonus_path[0] = 1 if projected_len >= state.bonus_threshold else 0
+    cards_needed_for_bonus[0] = max(0, state.bonus_threshold - projected_len)
+
+    discard_top_playable_flag[0] = 0
+    discard_top_playable_value[0] = 0
+    if state.discard_lens[color] > 0:
+        top_card = state.discard_cards[state._discard_index(color, state.discard_lens[color] - 1)]
+        rank = state._card_rank(top_card)
+        if state._card_color(top_card) == color and rank > 0 and rank > last_numeric_rank[0]:
+            discard_top_playable_flag[0] = 1
+            discard_top_playable_value[0] = _numeric_value_c(state, rank)
+
+    known_color_count = hand_count[0]
+    known_color_count += state.expedition_lens[cache_index]
+    known_color_count += state.expedition_lens[opponent_index]
+    known_color_count += state.discard_lens[color]
+    unknown_remaining_count[0] = max(0, state.cards_per_color - known_color_count)
+
+
+cdef int _append_derived_playability_features_c(GameState state, int player, float* out, int idx) noexcept:
+    cdef float max_numeric_sum = _max_numeric_sum_c(state)
+    cdef float max_cards_per_color = <float>max(1, state.cards_per_color)
+    cdef float max_wagers = <float>max(1, state.n_handshakes)
+    cdef float max_score_estimate = _max_score_estimate_c(state)
+    cdef int color
+    cdef int is_unopened, has_only_wagers_opened, current_numeric_sum, current_wager_count
+    cdef int current_expedition_len, last_numeric_rank, hand_count, hand_wager_count
+    cdef int playable_hand_wager_count, playable_hand_numeric_sum, playable_hand_numeric_count
+    cdef int dead_hand_numeric_count, dead_hand_numeric_sum, recoverable_margin_no_bonus
+    cdef int recoverable_score_no_bonus, min_needed_to_break_even, discard_top_playable_flag
+    cdef int discard_top_playable_value, unknown_remaining_count, has_bonus_path
+    cdef int cards_needed_for_bonus
+
+    for color in range(state.n_colors):
+        _color_playability_summary_c(
+            state, player, color,
+            &is_unopened, &has_only_wagers_opened, &current_numeric_sum,
+            &current_wager_count, &current_expedition_len, &last_numeric_rank,
+            &hand_count, &hand_wager_count, &playable_hand_wager_count,
+            &playable_hand_numeric_sum, &playable_hand_numeric_count,
+            &dead_hand_numeric_count, &dead_hand_numeric_sum, &recoverable_margin_no_bonus,
+            &recoverable_score_no_bonus, &min_needed_to_break_even,
+            &discard_top_playable_flag, &discard_top_playable_value,
+            &unknown_remaining_count, &has_bonus_path, &cards_needed_for_bonus,
+        )
+        out[idx] = <float>is_unopened
+        out[idx + 1] = <float>has_only_wagers_opened
+        out[idx + 2] = <float>current_numeric_sum / max_numeric_sum
+        out[idx + 3] = <float>current_wager_count / max_wagers
+        out[idx + 4] = <float>current_expedition_len / max_cards_per_color
+        out[idx + 5] = <float>last_numeric_rank / max_numeric_sum
+        out[idx + 6] = <float>hand_count / max_cards_per_color
+        out[idx + 7] = <float>hand_wager_count / max_wagers
+        out[idx + 8] = <float>playable_hand_numeric_sum / max_numeric_sum
+        out[idx + 9] = <float>playable_hand_numeric_count / max_cards_per_color
+        out[idx + 10] = <float>dead_hand_numeric_count / max_cards_per_color
+        out[idx + 11] = <float>dead_hand_numeric_sum / max_numeric_sum
+        out[idx + 12] = <float>recoverable_margin_no_bonus / max_numeric_sum
+        out[idx + 13] = <float>recoverable_score_no_bonus / max_score_estimate
+        out[idx + 14] = <float>min_needed_to_break_even / max_numeric_sum
+        out[idx + 15] = <float>discard_top_playable_flag
+        out[idx + 16] = <float>discard_top_playable_value / max_numeric_sum
+        out[idx + 17] = <float>unknown_remaining_count / max_cards_per_color
+        out[idx + 18] = <float>cards_needed_for_bonus / max_cards_per_color
+        idx += DERIVED_PLAYABILITY_PER_COLOR
+
+    out[idx] = <float>state.deck_len / <float>max(1, state.total_cards)
+    out[idx + 1] = <float>state.turn_count / <float>max(1, 2 * state.total_cards)
+    out[idx + 2] = <float>state.deck_len / <float>max(1, 2 * state.total_cards)
+    return idx + DERIVED_PLAYABILITY_COMMON
+
+
+cdef int _append_slot_aware_playability_features_c(GameState state, int player, float* out, int idx) noexcept:
+    cdef float max_numeric_sum = _max_numeric_sum_c(state)
+    cdef float max_score_estimate = _max_score_estimate_c(state)
+    cdef int slot
+    cdef int card
+    cdef int color
+    cdef int rank
+    cdef int is_unopened, has_only_wagers_opened, current_numeric_sum, current_wager_count
+    cdef int current_expedition_len, last_numeric_rank, hand_count, hand_wager_count
+    cdef int playable_hand_wager_count, playable_hand_numeric_sum, playable_hand_numeric_count
+    cdef int dead_hand_numeric_count, dead_hand_numeric_sum, recoverable_margin_no_bonus
+    cdef int recoverable_score_no_bonus, min_needed_to_break_even, discard_top_playable_flag
+    cdef int discard_top_playable_value, unknown_remaining_count, has_bonus_path
+    cdef int cards_needed_for_bonus
+    cdef bint legal_play
+    cdef bint has_numeric_started
+    cdef bint is_numeric
+    cdef bint is_wager
+    cdef bint would_start_color_commitment
+    cdef bint is_playable_to_existing
+    cdef bint is_dead_numeric
+    cdef bint is_wager_before_numeric
+    cdef bint is_numeric_open
+    cdef bint is_wager_first_open
+    cdef bint is_bad_open_candidate
+    cdef bint is_safe_continuation
+    cdef float open_risk_score
+
+    for slot in range(state.hand_size):
+        if slot >= state.hand_lens[player]:
+            idx += SLOT_AWARE_PLAYABILITY_PER_SLOT
+            continue
+        card = state.hand_cards[state._hand_index(player, slot)]
+        color = state._card_color(card)
+        rank = state._card_rank(card)
+        _color_playability_summary_c(
+            state, player, color,
+            &is_unopened, &has_only_wagers_opened, &current_numeric_sum,
+            &current_wager_count, &current_expedition_len, &last_numeric_rank,
+            &hand_count, &hand_wager_count, &playable_hand_wager_count,
+            &playable_hand_numeric_sum, &playable_hand_numeric_count,
+            &dead_hand_numeric_count, &dead_hand_numeric_sum, &recoverable_margin_no_bonus,
+            &recoverable_score_no_bonus, &min_needed_to_break_even,
+            &discard_top_playable_flag, &discard_top_playable_value,
+            &unknown_remaining_count, &has_bonus_path, &cards_needed_for_bonus,
+        )
+        legal_play = state._can_play_encoded_card_c(player, card)
+        has_numeric_started = last_numeric_rank > 0
+        is_numeric = rank > 0
+        is_wager = rank == 0
+        would_start_color_commitment = legal_play and not has_numeric_started
+        is_numeric_open = would_start_color_commitment and is_numeric
+        is_wager_first_open = would_start_color_commitment and is_wager and is_unopened
+        is_playable_to_existing = legal_play and has_numeric_started
+        is_dead_numeric = is_numeric and not legal_play and rank <= last_numeric_rank
+        is_wager_before_numeric = is_wager and legal_play and not has_numeric_started
+        is_bad_open_candidate = would_start_color_commitment and recoverable_score_no_bonus < 0
+        open_risk_score = min(0.0, <float>recoverable_score_no_bonus) if would_start_color_commitment else 0.0
+        is_safe_continuation = (not would_start_color_commitment) and is_playable_to_existing
+
+        out[idx] = <float>recoverable_score_no_bonus / max_score_estimate
+        out[idx + 1] = <float>recoverable_margin_no_bonus / max_numeric_sum
+        out[idx + 2] = <float>would_start_color_commitment
+        out[idx + 3] = <float>is_numeric_open
+        out[idx + 4] = <float>is_wager_first_open
+        out[idx + 5] = <float>is_playable_to_existing
+        out[idx + 6] = <float>is_dead_numeric
+        out[idx + 7] = <float>is_wager_before_numeric
+        out[idx + 8] = <float>has_bonus_path
+        out[idx + 9] = <float>is_bad_open_candidate
+        out[idx + 10] = open_risk_score / max_score_estimate
+        out[idx + 11] = <float>is_safe_continuation
+        idx += SLOT_AWARE_PLAYABILITY_PER_SLOT
+    return idx
+
+
 cdef int encode_info_state_c(GameState state, int player, float* out) except -1:
+    return _encode_info_state_with_flags_c(state, player, out, False, False)
+
+
+cdef int _encode_info_state_with_flags_c(
+    GameState state,
+    int player,
+    float* out,
+    bint derived_playability,
+    bint slot_aware_playability,
+) except -1:
     cdef int idx = 0
     cdef int slot
     cdef int card
@@ -132,18 +404,32 @@ cdef int encode_info_state_c(GameState state, int player, float* out) except -1:
     for i in range(action_count):
         out[idx + actions[i]] = 1.0
     idx += action_size
+    if derived_playability:
+        idx = _append_derived_playability_features_c(state, player, out, idx)
+    if slot_aware_playability:
+        idx = _append_slot_aware_playability_features_c(state, player, out, idx)
     return idx
 
 
-def input_dim(GameState state) -> int:
-    return input_dim_c(state)
+def input_dim(GameState state, encoding=None) -> int:
+    cdef bint derived_playability = False
+    cdef bint slot_aware_playability = False
+    if encoding is not None:
+        derived_playability = bool(encoding.derived_playability)
+        slot_aware_playability = bool(encoding.slot_aware_playability)
+    return _input_dim_with_flags_c(state, derived_playability, slot_aware_playability)
 
 
-def encode_info_state(GameState state, int player):
+def encode_info_state(GameState state, int player, encoding=None):
     cdef float[::1] out_view
+    cdef bint derived_playability = False
+    cdef bint slot_aware_playability = False
     import numpy as np
 
-    out = np.empty(input_dim_c(state), dtype=np.float32)
+    if encoding is not None:
+        derived_playability = bool(encoding.derived_playability)
+        slot_aware_playability = bool(encoding.slot_aware_playability)
+    out = np.empty(_input_dim_with_flags_c(state, derived_playability, slot_aware_playability), dtype=np.float32)
     out_view = out
-    encode_info_state_c(state, player, &out_view[0])
+    _encode_info_state_with_flags_c(state, player, &out_view[0], derived_playability, slot_aware_playability)
     return out

src/coolrl_lost_cities/games/classic/deep_cfr/evaluate.py

@@ -6,7 +6,7 @@ import numpy as np
 import torch
 
 from coolrl_lost_cities.games.classic.bots import build_bot
-from coolrl_lost_cities.games.classic.deep_cfr.config import config_from_dict
+from coolrl_lost_cities.games.classic.deep_cfr.config import EncodingConfig, config_from_dict
 from coolrl_lost_cities.games.classic.deep_cfr.encoding import encode_info_state
 from coolrl_lost_cities.games.classic.deep_cfr.networks import DeepCFRMLP
 from coolrl_lost_cities.games.classic.evaluation import evaluate_policy
@@ -22,11 +22,13 @@ class StrategyNetPolicy(LostCitiesPolicy):
         device: torch.device | str = "cpu",
         sample: bool = False,
         seed: int | None = None,
+        encoding: EncodingConfig | None = None,
     ) -> None:
         self.strategy_network = strategy_network
         self.device = torch.device(device)
         self.sample = sample
         self.rng = np.random.default_rng(seed)
+        self.encoding = encoding
 
     def act(self, obs_or_state: PolicyInput) -> int:
         if not isinstance(obs_or_state, GameState):
@@ -40,7 +42,7 @@ class StrategyNetPolicy(LostCitiesPolicy):
         legal_actions = np.flatnonzero(legal)
         if len(legal_actions) == 0:
             raise RuntimeError("no legal action available")
-        info = encode_info_state(state, state.current_player)
+        info = encode_info_state(state, state.current_player, self.encoding)
         with torch.inference_mode():
             x = torch.as_tensor(info, dtype=torch.float32, device=self.device).unsqueeze(0)
             logits = self.strategy_network(x).squeeze(0).detach().cpu().numpy()
@@ -64,11 +66,14 @@ def evaluate_strategy_network(
     opponent: str = "random",
     device: torch.device | str = "cpu",
     max_steps: int = 10_000,
+    encoding: EncodingConfig | None = None,
 ) -> dict[str, float | int]:
     strategy_network.eval()
 
     def make_strategy(seed_value: int | None = None) -> StrategyNetPolicy:
-        return StrategyNetPolicy(strategy_network, device=device, seed=seed_value)
+        return StrategyNetPolicy(
+            strategy_network, device=device, seed=seed_value, encoding=encoding
+        )
 
     def opponent_factory(seed_value=None):
         return build_bot(opponent, seed=seed_value)
@@ -101,4 +106,7 @@ def load_strategy_policy_from_checkpoint(
     ).to(device)
     network.load_state_dict(payload["strategy_network"])
     network.eval()
-    return StrategyNetPolicy(network, device=device, sample=sample, seed=seed), game_config
+    return (
+        StrategyNetPolicy(network, device=device, sample=sample, seed=seed, encoding=cfg.encoding),
+        game_config,
+    )

src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py

@@ -81,7 +81,7 @@ class DeepCFRTrainer:
         self.device = _resolve_torch_device(device)
 
         probe = GameState.new_game(self.game_config, seed=self.config.run.seed)
-        self.input_dim = input_dim(probe)
+        self.input_dim = input_dim(probe, self.config.encoding)
         self.action_size = 2 * probe.config.hand_size + 1 + probe.config.n_colors
 
         torch.manual_seed(self.config.run.seed)
@@ -206,6 +206,7 @@ class DeepCFRTrainer:
             self_play_older_weight=self.config.self_play.older_weight,
             self_play_anchor_weight=self.config.self_play.anchor_weight,
             self_play_recent_window=self.config.self_play.recent_window,
+            encoding=self.config.encoding,
             rng=self.rng,
         )
         for network in self.advantage_networks:
@@ -394,6 +395,7 @@ class DeepCFRTrainer:
                 opponent=opponent,
                 device=self.device,
                 max_steps=self.config.evaluation.max_steps,
+                encoding=self.config.encoding,
             )
             for key, value in result.items():
                 results[f"eval_{opponent}_{key}"] = value

src/coolrl_lost_cities/games/classic/deep_cfr/traverser.py

@@ -103,6 +103,7 @@ class DeepCFRTraverser:
         self_play_older_weight: float = 0.2,
         self_play_anchor_weight: float = 0.0,
         self_play_recent_window: int = 5,
+        encoding=None,
         rng: np.random.Generator | None = None,
     ) -> None:
         self.advantage_networks = advantage_networks
@@ -145,6 +146,7 @@ class DeepCFRTraverser:
         self.self_play_older_weight = max(0.0, float(self_play_older_weight))
         self.self_play_anchor_weight = max(0.0, float(self_play_anchor_weight))
         self.self_play_recent_window = max(0, int(self_play_recent_window))
+        self.encoding = encoding
         self.rng = rng or np.random.default_rng()
         self._safe_heuristic_rollout_bot = (
             SafeHeuristicBot() if self.cutoff_rollout_policy == "safe_heuristic" else None
@@ -275,7 +277,7 @@ class DeepCFRTraverser:
         state: GameState,
         player: int,
     ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
-        info_state = encode_info_state(state, player)
+        info_state = encode_info_state(state, player, self.encoding)
         legal = np.asarray(state.unified_legal_mask(), dtype=bool)
         with torch.inference_mode():
             x = torch.as_tensor(info_state, dtype=torch.float32, device=self.device).unsqueeze(0)

src/coolrl_lost_cities/games/classic/deep_cfr/workers.py

@@ -85,6 +85,7 @@ def run_traversal_worker_batch(batch: TraversalWorkerBatch) -> TraversalWorkerRe
         self_play_older_weight=cfg.self_play.older_weight,
         self_play_anchor_weight=cfg.self_play.anchor_weight,
         self_play_recent_window=cfg.self_play.recent_window,
+        encoding=cfg.encoding,
         rng=np.random.default_rng(batch.worker_seed),
     )
     game_config = LostCitiesConfig(**batch.game_config)

tests/games/classic/test_deep_cfr_trainer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import numpy as np
+from coolrl_lost_cities.games.classic.deep_cfr.encoding import encode_info_state, input_dim
 from coolrl_lost_cities.games.classic.game import GameState, LostCitiesConfig
 
 from coolrl_lost_cities.games.classic.deep_cfr.benchmark import (
@@ -53,6 +54,47 @@ def test_deep_cfr_loads_mapped_legacy_reproduction_config() -> None:
     assert config.checkpoint.save_iteration_interval == 10
 
 
+def test_deep_cfr_playability_encoding_extends_input_shape() -> None:
+    state = GameState.new_game(LostCitiesConfig(seed=61), seed=61)
+    base_dim = input_dim(state)
+    derived_config = _deep_cfr_config({"encoding": {"derived_playability": True}})
+    slot_config = _deep_cfr_config(
+        {"encoding": {"derived_playability": True, "slot_aware_playability": True}}
+    )
+
+    derived_dim = input_dim(state, derived_config.encoding)
+    slot_dim = input_dim(state, slot_config.encoding)
+
+    assert derived_dim == base_dim + state.config.n_colors * 19 + 3
+    assert slot_dim == derived_dim + state.config.hand_size * 12
+    assert encode_info_state(state, 0, slot_config.encoding).shape == (slot_dim,)
+
+
+def test_deep_cfr_trainer_uses_playability_encoding() -> None:
+    config = _deep_cfr_config(
+        {
+            "run": {"iterations": 1, "seed": 62},
+            "encoding": {"derived_playability": True, "slot_aware_playability": True},
+            "network": {"hidden_size": 16},
+            "traversal": {"traversals_per_iteration": 1, "max_depth": 1, "max_nodes": 16},
+            "optimization": {
+                "advantage_train_steps": 1,
+                "strategy_train_steps": 1,
+                "batch_size": 2,
+            },
+            "checkpoint": {"save_every_iteration": False},
+        }
+    )
+    game_config = LostCitiesConfig(seed=62)
+    trainer = DeepCFRTrainer(config, game_config)
+
+    metrics = trainer.train()
+
+    probe = GameState.new_game(game_config, seed=62)
+    assert trainer.input_dim == input_dim(probe, config.encoding)
+    assert metrics[0].advantage_samples > 0
+
+
 def test_deep_cfr_trainer_smoke_run() -> None:
     trainer = DeepCFRTrainer(
         _deep_cfr_config(