From f63c4b8059279f17ac38ea9a63b31f954b160240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=8B=9C=EC=9B=90?= Date: Sun, 10 May 2026 14:26:16 +0900 Subject: [PATCH] Strip heuristic input features and add selectivity diagnostics Audit of the Deep CFR information-state encoding identified two tiers of non-pure features and removed both: - Tier 3 (judgment): is_bad_open_candidate, open_risk_score, is_safe_continuation. Same heuristic family used to label bad_open in evaluation, embedded as model input. - Tier 2 (projection): recoverable_score_no_bonus, recoverable_margin_no_bonus, min_needed_to_break_even, cards_needed_for_bonus, has_bonus_path. Mechanical but assumption-laden ("commit and play all currently-playable cards"). The no_bonus form is asymmetric: it amplifies the immediate -20 penalty while truncating the +20 bonus upside, biasing the model toward the same "don't open" basin the diagnostics already flagged. Input dim 365 -> 297. DERIVED_PLAYABILITY_PER_COLOR 19 -> 15; SLOT_AWARE_PLAYABILITY_PER_SLOT 12 -> 6. Test shape assertions updated. Also adds selectivity diagnostic infrastructure used to reach this point: - traversal.outcome_unsampled_first_open_prior_alpha config field with signed-prior overlay on unsampled first-open advantage targets (A1). - analyze_first_open_counterfactual.py --post-policy to swap the policy_player rollout policy and isolate selection bias (D1). - analyze_first_open_followup.py to inspect post-forced-open behavior (E2): same-color play vs discard counts, other-open rate, terminal hand composition. Findings recorded in docs/plans/deep-cfr-selectivity.md sections 3-6. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/plans/deep-cfr-selectivity.md | 298 ++++++++++++++++- scripts/analyze_first_open_counterfactual.py | 30 +- scripts/analyze_first_open_followup.py | 312 ++++++++++++++++++ scripts/analyze_first_open_targets.py | 3 + .../games/classic/deep_cfr/config.py | 1 + .../games/classic/deep_cfr/encoding.pyx | 41 +-- .../classic/deep_cfr/interleaved_traversal.py | 72 +++- .../games/classic/deep_cfr/trainer.py | 3 + .../games/classic/deep_cfr/workers.py | 3 + tests/games/classic/test_deep_cfr_trainer.py | 60 +++- 10 files changed, 786 insertions(+), 37 deletions(-) create mode 100644 scripts/analyze_first_open_followup.py diff --git a/docs/plans/deep-cfr-selectivity.md b/docs/plans/deep-cfr-selectivity.md index 1e7f3ff..077f1e5 100644 --- a/docs/plans/deep-cfr-selectivity.md +++ b/docs/plans/deep-cfr-selectivity.md @@ -1,6 +1,6 @@ # Deep CFR Selectivity Investigation -Last updated: 2026-05-09 +Last updated: 2026-05-10 ## Current conclusion @@ -272,7 +272,301 @@ around 120-200 iterations, then regressed by 300-500 iterations. This suggests the replay emphasis is not enough if the underlying target does not separate good first opens from bad first opens. -### 3. Short open-selectivity ablation +### 3. First-open target prior (A1) — running 2026-05-10 + +Hypothesis: target audit shows the regenerated traversal target does not +separate good first opens from bad first opens. With +`outcome_unsampled_regret=zero`, every unsampled first-open candidate is +labeled with target `0`, regardless of whether it is heuristically good +or bad. A small signed prior on unsampled first-open play actions should +break the symmetry without overriding the high-variance sampled-action +target. + +Implementation: + +- Config field: `traversal.outcome_unsampled_first_open_prior_alpha` (float, + default `0.0`). +- Plumbed through `InterleavedTraversalConfig` and + `run_interleaved_traversal_batch`. Cython traversal path unchanged + (default scheduler is interleaved; pyx path is a follow-up if needed). +- Effect site: `_after_child` in `interleaved_traversal.py`. When + `is_first_open` and `alpha != 0`, unsampled legal play actions whose color + has an empty expedition are overwritten in `target` with `+alpha` if the + visible `recoverable_score` for that color is `>= 0`, else `-alpha`. The + sampled action's cell stays as `sampled_action_value - node_value`. +- `recoverable_score` mirrors `evaluate._visible_recoverable_summary` for + the empty-expedition case (no future-card lookahead, hand-only signal). +- Tests: `test_first_open_prior_overrides_unsampled_play_targets_with_signed_alpha`, + `test_first_open_prior_zero_alpha_is_noop`. + +Note on "pure self-play": this prior introduces a hand-written heuristic +into the advantage target. The opponent and rollout policy remain +self-play (no external bot). This is treated as a diagnostic experiment; +if effective, the permanent solution is a counterfactual-based prior +(A2) that recovers full pure self-play. + +Initial planned run: + +- `run.experiment_name=first-open-prior-alpha5-512x3-det-200` +- `traversal.outcome_unsampled_first_open_prior_alpha=5.0` +- `traversal.outcome_unsampled_regret=zero` +- `traversal.outcome_sampling_epsilon=0.05` +- `run.deterministic=true` +- `run.max_iterations=200` +- W&B group: `first-open-prior-v1` + +Primary comparisons: + +- `confirm-eps-005-zero-512x3-det-500` (baseline; iter 200 reference) +- `first-open-reweight-50-512x3-det-500-indexed` (replay-side intervention) + +Success criterion: `bad_open_rate` and `score_per_opened_color` improve +*together* at iter 200 without large regression in `avg_score_diff0`. If +positive, follow up with a 500-iter run to test stability. If null/regress, +try alpha sweep (e.g. 2.0, 10.0) before abandoning the direction. + +Result (2026-05-10): + +- Run: `runs/2026-05-10_072718_first-open-prior-alpha5-512x3-det-200` +- W&B: synced online to group `first-open-prior-v1` (run `teuh915r`) +- Commit: see HEAD at run start + +`safe_heuristic_strict` at iter 200: + +| Run | Score diff | Win rate | Bad open | Score/opened | +| --- | ---: | ---: | ---: | ---: | +| baseline `confirm-eps-005-zero` (iter 200) | -40.01 | 0.12 | 0.893 | -6.25 | +| **A1 prior α=5.0 (iter 200)** | **-67.54** | **0.02** | **0.796** | **-8.85** | + +Other opponents at iter 200: random +32.51 / 0.86, noisy_safe -71.52 / 0.07, +safe_heuristic -81.81 / 0.02, safe_heuristic_loose -81.24 / 0.05. + +Conclusion: **mixed result, net regression.** The prior did shift behavior +in the intended direction on one axis — `bad_open_rate` dropped from 0.893 +to 0.796 (~10% absolute reduction). This is the only metric where the +hypothesis "the prior breaks ranking symmetry" looks supported. + +But the overall game-quality metrics regressed: score diff worsened +(-40 → -67), win rate collapsed (0.12 → 0.02), and `score_per_opened_color` +got worse (-6.25 → -8.85). The model fails the success criterion (which +required `bad_open_rate` AND `score_per_opened_color` to improve together). + +Two interpretations: + +1. **α = 5.0 too strong.** The prior is overpowering the sampled-action + target rather than acting as a weak symmetry-breaker. The good-open prior + pushes the model to open *more often*, but those forced opens are bad + *given the actual game state*, just labeled good by the visible-only + heuristic. The counterfactual audit already flagged this: heuristic + `open_good` candidates often lose to best non-open in real continuation. +2. **Heuristic itself misaligned.** Even the right α won't help if the + sign assigned to candidates is wrong relative to true game value. + +Next steps (in order): + +- (A1.b) α sweep at 200 iter: α ∈ {1.0, 2.0} to test "weaker prior" hypothesis. + If α=2.0 still regresses score diff while reducing bad_open, the prior shape + itself is wrong, not just strength. +- (A2) Counterfactual prior: replace heuristic sign with sign of + `value(force open) - value(best non-open)` from a small in-traversal + rollout. Cleaner signal, recovers pure self-play, but more expensive. +- If both fail to improve score diff, the issue is upstream of unsampled + regret labeling (likely traversal sample distribution or sampled-action + target variance), and the next experiment family should target those. + +### 4. D1 diagnostic: counterfactual with strong post-policy (2026-05-10) + +Question: are forced-open continuation values low because the openings +themselves are bad, or because the self-play rollout policy poisons the +post-action play (selection bias)? + +Method: re-ran `analyze_first_open_counterfactual.py` on the +`confirm-eps-005-zero-512x3-det-500` baseline checkpoints (iter 200, iter +500), but with `--post-policy safe_heuristic_strict`. The opponent and +state-collection policy stayed the same; only the policy_player's actions +*after* the forced first action used the strong fixed bot. + +Output: `runs/tmp/first_open_counterfactual_d1_strong_post_policy_200_vs_500.jsonl` + +`delta_open = value(force open) - value(best non-open)` comparison: + +| iter | bucket | n | self-play post | strong post | shift | +| ---: | --- | ---: | ---: | ---: | ---: | +| 200 | open_good | 40 | -26.57 | **-3.35** | +23.22 | +| 200 | open_bad | 460 | -23.15 | **-5.52** | +17.63 | +| 500 | open_good | 30 | -23.43 | **-10.90** | +12.53 | +| 500 | open_bad | 470 | -11.18 | **-8.99** | +2.19 | + +`delta_positive_rate`: + +| iter | bucket | self-play post | strong post | +| ---: | --- | ---: | ---: | +| 200 | open_good | 0.050 | 0.225 | +| 200 | open_bad | 0.130 | 0.317 | +| 500 | open_good | 0.167 | 0.300 | +| 500 | open_bad | 0.226 | 0.230 | + +Verdict — two findings, both important: + +**1. Selection bias is real and significant.** Strong post-policy +improves forced-open continuation values by 12–23 points on average. The +self-play policy is meaningfully poisoning rollouts: forced opens look +much less bad once a competent player handles the followup. + +**2. Heuristic labels do not separate cleanly even under strong post-policy.** +At iter 200, `open_good` delta_mean is only ~2 points better than +`open_bad` (-3.35 vs -5.52). At iter 500, ranking is essentially flat +(`open_good` -10.9 vs `open_bad` -8.99 — slightly *worse*). Median deltas +agree. Sample size for `open_good` is small (30–40) so noise contributes, +but there is no clean signal that the heuristic `recoverable_score` +classifier matches actual continuation value. + +Implications: + +- A1 prior was destined to fail — the heuristic sign is at best weakly + aligned with continuation value, even with optimistic post-policy. +- A2 with self-play rollouts would inherit selection bias and likely + reproduce the same misranking. A2 with a strong post-policy would + give clean signs but breaks pure self-play. +- The deeper bottleneck is post-open play quality. Until the trained + policy plays competently *after* opening, training signals about + *whether to open* will be biased toward "don't open." + +Candidate next directions (decision pending): + +- (E1) Train with `cutoff_rollout_policy=safe_heuristic` instead of `random`. + Already a config option; gives leaf nodes stronger value estimates + during traversal. Trades some pure-self-play purity for a stronger + bootstrap signal. Cheap to test. +- (E2) Investigate post-open behavior directly: forced-open + observe + next-2-3 turns. Diagnoses *why* the model can't follow up (e.g. always + discards followup cards, switches color, etc.). +- (E3) Curriculum / staged training that exposes the network to good + post-open trajectories before forcing it to make first-open decisions. +- A2 deferred unless we adopt a strong post-policy in rollouts. + +### 5. E2 diagnostic: post-forced-open behavior (2026-05-10) + +Question: D1 showed selection bias is real — *what is the model actually +doing after a forced first-open* that poisons rollouts? + +Method: forced each first-open candidate at sampled states, then observed +the policy_player's next 3 decisions (window). Used baseline checkpoints +(`confirm-eps-005-zero` iter 200, iter 500). Categorized each followup +decision relative to the forced-open color. + +Output: `runs/tmp/first_open_followup_baseline_200_vs_500.jsonl` +Script: `scripts/analyze_first_open_followup.py` + +Per-window mean counts (3 policy_player decisions = ~1.5 game turns; +each game turn includes one play/discard plus one draw): + +| iter | bucket | n | held@force | plays(same) | discards(same) | open_other | other_discard | held@end | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 200 | good | 33 | 4.82 | 0.15 | 0.15 | **0.49** | 0.21 | 2.70 | +| 200 | bad | 367 | 1.97 | 0.03 | 0.12 | 0.10 | 0.75 | 1.08 | +| 500 | good | 22 | 4.45 | 0.14 | **0.46** | 0.14 | 0.27 | 1.23 | +| 500 | bad | 378 | 2.56 | 0.01 | 0.23 | 0.05 | 0.71 | 1.29 | + +Findings: + +**1. Model under-plays followup cards even when it holds many.** At iter +200, good-open candidates start with ~4.8 same-color cards in hand. In the +next 3 decisions, the model plays only 0.15 of them on average (3% of +its window). 2.7 cards remain in hand at terminal — nearly 3 cards of +the just-opened color never reach the expedition. + +**2. Model opens additional new colors after a forced open.** At iter 200 +good bucket, `open_other` = 0.49 in a 3-decision window — the model is +~3× more likely to open *another* new color than to follow up the one it +was forced into. This compounds the recoverable-score drag. + +**3. By iter 500 the model actively dumps the forced color.** Good bucket +at iter 500: `same-color discards` 0.46 vs `same-color plays` 0.14. The +model discards followup cards more than 3× as often as it plays them, +despite holding 4.45 of them at force time. This is the strongest single +piece of evidence so far that the post-open value head is poisoned. + +**4. Bad-open bucket shows even sharper avoidance.** Hand has ~2 +same-color cards, plays effectively zero. Almost all play-phase decisions +are "discard other" (0.71–0.75). The model treats forced opens as bad +news to liquidate rather than commit to. + +Interpretation: the trained advantage network *learned* that opens of +these colors are net-negative, so once forced into one, it hedges by +opening other colors and dumping followup cards. That hedging is rational +under the policy's own value estimates but is the exact mechanism that +keeps forced-open continuation values low and keeps the training target +labeling opens as bad. Closed loop. + +This combined with D1 means: + +- A1/A2 priors operate on first-open *labels*. They cannot fix a model + that, even after committing to an open, refuses to follow up. +- Any fix needs to either (a) produce stronger leaf values during training + so the value head learns post-open play matters, or (b) explicitly + curriculum or prior-shape the *post-open* decision (not the open + decision). + +Updated next-step priorities: + +- **(E1) `cutoff_rollout_policy=safe_heuristic` training ablation.** + Strongest single lever: gives traversal leaves stronger value estimates + during training, which should ripple back to "open + follow up" signal. + Pure-self-play purity dented, but only at cutoff leaves. +- (E1.b) Sanity check: at iter 200 baseline, the same-color discard rate + is already 0.12 in good bucket — early. So this is not a late-training + collapse; it's baked in from early iterations. Curriculum-style fix + would have to start very early. +- A2 effectively dead unless paired with strong post-policy in rollouts. + +### 6. Input feature audit and cleanup (2026-05-10) + +After D1 + E2 confirmed the bottleneck is *training signal* and not *input +poverty*, audited every feature the model receives. Goal: align the input +representation with a strict pure-self-play definition by removing any +feature that embeds a hand-coded judgment or strategy assumption. + +Three tiers found: + +- **Tier 1**: pure game state and single-step game rules (hand cards, + expedition state, discard piles, scores, public histogram, legal-action + mask, `playable_*`/`dead_*` rule checks, `unknown_remaining_count`, + etc.). No judgment. Kept. +- **Tier 2**: projection-based features that compute "if I commit and + play all currently-playable cards, what is the score?" — mechanical + but assumption-laden. Includes `recoverable_score_no_bonus`, + `recoverable_margin_no_bonus`, `min_needed_to_break_even`, + `cards_needed_for_bonus`, `has_bonus_path`. Removed. +- **Tier 3**: explicit hand-coded judgments — `is_bad_open_candidate`, + `open_risk_score`, `is_safe_continuation`. Same heuristic family used + to label `bad_open` in evaluation. Removed. + +Notable additional finding: the `no_bonus` projection family is +asymmetric. It includes wager multipliers but excludes the +20 bonus, +which systematically *under*-estimates the upside of commitable +expeditions by roughly 20 × P(complete bonus). This bias points in the +same direction as the phase 1 trap ("opens look like loss"), so removing +the projection family is consistent with diagnosing-not-baking-in the +trap. + +Implementation: + +- `encoding.pyx`: `DERIVED_PLAYABILITY_PER_COLOR` 19 → 15; + `SLOT_AWARE_PLAYABILITY_PER_SLOT` 12 → 6 (across two cleanup passes). +- Test shape assertions updated. +- Cython module rebuilt. + +Input dimension change: 365 → 341 (Tier 3 removal) → **297** (Tier 2 +removal). Net 18.6% reduction. + +Result is not yet measured. The hypothesis is that with the heuristic +crutches removed, training behavior is a cleaner measurement of what +Deep CFR can do in this game from raw input. The model may stay in the +phase 1 trap longer or fail more visibly, both of which are useful +information. + +### 7. Short open-selectivity ablation Run a 200-300 iteration ablation only after the target audit identifies a specific change. Candidate changes include: diff --git a/scripts/analyze_first_open_counterfactual.py b/scripts/analyze_first_open_counterfactual.py index 1bd3286..7c2bf2b 100644 --- a/scripts/analyze_first_open_counterfactual.py +++ b/scripts/analyze_first_open_counterfactual.py @@ -218,15 +218,27 @@ def _rollout_value( opponent: str, seed: int, max_steps: int, + post_policy: str = "model", ) -> float: + """Roll out from `state` to terminal and return policy_player's score diff. + + When `post_policy == "model"`, policy_player uses the trained advantage + network policy. Otherwise `post_policy` is treated as a bot name and a + fresh bot is built for policy_player too — used to diagnose whether the + self-play rollout itself is poisoning forced-open continuation values. + """ rollout = state.clone() opponent_policy = build_bot(opponent, seed=seed) + post_policy_bot = build_bot(post_policy, seed=seed * 7 + 1) if post_policy != "model" else None steps = 0 while not rollout.terminal and steps < max_steps: current = int(rollout.current_player) if current == policy_player: - unified = policy.select_unified(rollout) - action = rollout.from_unified_action(unified) + if post_policy_bot is not None: + action = post_policy_bot.act(rollout) + else: + unified = policy.select_unified(rollout) + action = rollout.from_unified_action(unified) else: action = opponent_policy.act(rollout) rollout.apply_action(action) @@ -260,6 +272,7 @@ def analyze_checkpoint( device: torch.device, max_steps: int, max_candidates: int, + post_policy: str = "model", ) -> dict[str, Any]: _cfg, game_config, policy, iteration = _load_checkpoint(checkpoint, device) buckets: dict[str, Bucket] = defaultdict(Bucket) @@ -306,6 +319,7 @@ def analyze_checkpoint( opponent=opponent, seed=game_seed * 10_000 + candidate_states * 101 + 1, max_steps=max_steps, + post_policy=post_policy, ) for open_action in open_actions: if evaluated_open_candidates >= max_candidates: @@ -319,6 +333,7 @@ def analyze_checkpoint( opponent=opponent, seed=game_seed * 10_000 + candidate_states * 101 + 2, max_steps=max_steps, + post_policy=post_policy, ) label = labels[open_action] buckets[label].add( @@ -344,6 +359,7 @@ def analyze_checkpoint( "policy_turns": policy_turns, "candidate_states": candidate_states, "first_open_candidates": first_open_candidates, + "post_policy": post_policy, "buckets": {key: bucket.to_dict() for key, bucket in sorted(buckets.items())}, } @@ -357,6 +373,15 @@ def main() -> None: parser.add_argument("--device", default="cuda") parser.add_argument("--max-steps", type=int, default=10_000) parser.add_argument("--max-candidates", type=int, default=500) + parser.add_argument( + "--post-policy", + default="model", + help=( + "Policy used for the policy_player during forced-action rollouts. " + "'model' uses the trained advantage network; any other value is " + "treated as a bot name (e.g. 'safe_heuristic_strict')." + ), + ) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() @@ -372,6 +397,7 @@ def main() -> None: device=device, max_steps=args.max_steps, max_candidates=args.max_candidates, + post_policy=args.post_policy, ) rows.append(row) print(json.dumps(row, sort_keys=True)) diff --git a/scripts/analyze_first_open_followup.py b/scripts/analyze_first_open_followup.py new file mode 100644 index 0000000..c7e01fb --- /dev/null +++ b/scripts/analyze_first_open_followup.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python +"""Inspect model's post-forced-open behavior over the next K policy_player turns. + +Diagnoses *why* forced-open continuation values are poor under self-play +rollouts: does the model play followup cards of the opened color, discard +them, or open another color instead? +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from coolrl_lost_cities.games.classic.game import GameState, LostCitiesConfig + +from coolrl_lost_cities.games.classic.bots import build_bot +from coolrl_lost_cities.games.classic.deep_cfr.config import config_from_dict +from coolrl_lost_cities.games.classic.deep_cfr.networks import DeepCFRMLP + +_SCRIPT_DIR = Path(__file__).resolve().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +from analyze_first_open_counterfactual import ( # noqa: E402 + AdvantagePolicy, + _classify_action, +) + + +@dataclass +class FollowupBucket: + candidates: int = 0 + same_color_plays: list[int] = field(default_factory=list) + same_color_discards: list[int] = field(default_factory=list) + other_open_new: list[int] = field(default_factory=list) + other_play_existing: list[int] = field(default_factory=list) + other_discard: list[int] = field(default_factory=list) + draw_deck: list[int] = field(default_factory=list) + draw_pile: list[int] = field(default_factory=list) + same_color_held_at_force: list[int] = field(default_factory=list) + same_color_played_in_window: list[int] = field(default_factory=list) + same_color_discarded_in_window: list[int] = field(default_factory=list) + final_score_diff: list[float] = field(default_factory=list) + same_color_held_terminal: list[int] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + def stats(values: list[float] | list[int]) -> dict[str, float]: + if not values: + return {"mean": 0.0, "median": 0.0} + return { + "mean": float(np.mean(values)), + "median": float(np.median(values)), + } + + return { + "candidates": self.candidates, + "same_color_plays": stats(self.same_color_plays), + "same_color_discards": stats(self.same_color_discards), + "other_open_new": stats(self.other_open_new), + "other_play_existing": stats(self.other_play_existing), + "other_discard": stats(self.other_discard), + "draw_deck": stats(self.draw_deck), + "draw_pile": stats(self.draw_pile), + "same_color_held_at_force": stats(self.same_color_held_at_force), + "same_color_played_in_window": stats(self.same_color_played_in_window), + "same_color_discarded_in_window": stats(self.same_color_discarded_in_window), + "same_color_held_terminal": stats(self.same_color_held_terminal), + "final_score_diff": stats(self.final_score_diff), + } + + +def _load_checkpoint( + checkpoint: Path, device: torch.device +) -> tuple[Any, LostCitiesConfig, AdvantagePolicy, int]: + payload = torch.load(checkpoint, map_location="cpu") + cfg = config_from_dict(payload["config"]) + game_config = LostCitiesConfig(**payload["game_config"]) + action_size = int(payload["action_size"]) + networks = [ + DeepCFRMLP.from_config(int(payload["input_dim"]), action_size, cfg.network).to(device) + for _ in range(2) + ] + for network, state_dict in zip(networks, payload["advantage_networks"], strict=True): + network.load_state_dict(state_dict) + network.eval() + policy = AdvantagePolicy( + networks, + device=device, + encoding=cfg.encoding, + epsilon=cfg.traversal.regret_matching_epsilon, + fallback=cfg.regret_matching.all_negative_fallback, + ) + return cfg, game_config, policy, int(payload.get("iteration", -1)) + + +def _count_color_cards_in_hand(state: GameState, player: int, color: int) -> int: + return sum( + 1 for card in state.hand_slots(player) if card is not None and int(card.color) == color + ) + + +def _label_followup_action( + state: GameState, unified_action: int, player: int, forced_color: int +) -> str: + """Categorise a followup action relative to the previously forced-open color.""" + card_action_size = state.config.hand_size * 2 + if unified_action == card_action_size: + return "draw_deck" + if unified_action > card_action_size: + return "draw_pile" + if unified_action % 2 == 1: + slot = unified_action // 2 + card = state.hand_slots(player)[slot] + if card is None: + return "discard_invalid" + return "discard_same" if int(card.color) == forced_color else "discard_other" + slot = unified_action // 2 + card = state.hand_slots(player)[slot] + if card is None: + return "play_invalid" + color = int(card.color) + is_open_action = not state.expeditions[player][color] + if color == forced_color: + return "play_same" + return "open_other" if is_open_action else "play_other_existing" + + +def _force_and_observe( + base_state: GameState, + *, + forced_action: int, + forced_color: int, + bucket: FollowupBucket, + policy: AdvantagePolicy, + opponent_policy: Any, + policy_player: int, + window: int, + max_steps: int, +) -> None: + rollout = base_state.clone() + same_at_force = _count_color_cards_in_hand(rollout, policy_player, forced_color) + rollout.apply_action(rollout.from_unified_action(forced_action)) + counts: Counter[str] = Counter() + policy_turns_seen = 0 + same_color_plays = 0 + same_color_discards = 0 + steps = 0 + while not rollout.terminal and steps < max_steps and policy_turns_seen < window: + current = int(rollout.current_player) + if current != policy_player: + rollout.apply_action(opponent_policy.act(rollout)) + steps += 1 + continue + unified = policy.select_unified(rollout) + label = _label_followup_action(rollout, unified, policy_player, forced_color) + counts[label] += 1 + if label == "play_same": + same_color_plays += 1 + elif label == "discard_same": + same_color_discards += 1 + rollout.apply_action(rollout.from_unified_action(unified)) + steps += 1 + policy_turns_seen += 1 + while not rollout.terminal and steps < max_steps: + current = int(rollout.current_player) + if current == policy_player: + unified = policy.select_unified(rollout) + rollout.apply_action(rollout.from_unified_action(unified)) + else: + rollout.apply_action(opponent_policy.act(rollout)) + steps += 1 + + bucket.candidates += 1 + bucket.same_color_plays.append(counts.get("play_same", 0)) + bucket.same_color_discards.append(counts.get("discard_same", 0)) + bucket.other_open_new.append(counts.get("open_other", 0)) + bucket.other_play_existing.append(counts.get("play_other_existing", 0)) + bucket.other_discard.append(counts.get("discard_other", 0)) + bucket.draw_deck.append(counts.get("draw_deck", 0)) + bucket.draw_pile.append(counts.get("draw_pile", 0)) + bucket.same_color_held_at_force.append(same_at_force) + bucket.same_color_played_in_window.append(same_color_plays) + bucket.same_color_discarded_in_window.append(same_color_discards) + bucket.same_color_held_terminal.append( + _count_color_cards_in_hand(rollout, policy_player, forced_color) + ) + bucket.final_score_diff.append(float(rollout.score_diff(policy_player))) + + +def analyze_checkpoint( + checkpoint: Path, + *, + games: int, + seed: int, + opponent: str, + device: torch.device, + max_steps: int, + max_candidates: int, + window: int, +) -> dict[str, Any]: + _cfg, game_config, policy, iteration = _load_checkpoint(checkpoint, device) + buckets: dict[str, FollowupBucket] = defaultdict(FollowupBucket) + candidates_evaluated = 0 + started = time.perf_counter() + + for game_index in range(games): + if candidates_evaluated >= max_candidates: + break + game_seed = seed + game_index + swap = game_index % 2 == 1 + policy_player = 1 if swap else 0 + opponent_policy = build_bot(opponent, seed=game_seed * 2 + (1 - policy_player)) + state = GameState.new_game(game_config, seed=game_seed) + for _step in range(max_steps): + if state.terminal or candidates_evaluated >= max_candidates: + break + current = int(state.current_player) + if current != policy_player: + state.apply_action(opponent_policy.act(state)) + continue + legal_actions, _policy_probs, _advantages = policy.distribution(state) + labels = { + int(action): _classify_action(state, int(action), current) + for action in legal_actions + } + open_actions = [ + int(action) for action, label in labels.items() if label.startswith("open_") + ] + if open_actions: + for open_action in open_actions: + if candidates_evaluated >= max_candidates: + break + color = int(state.hand_slots(current)[open_action // 2].color) + bucket = buckets[labels[open_action]] + _force_and_observe( + state, + forced_action=open_action, + forced_color=color, + bucket=bucket, + policy=policy, + opponent_policy=opponent_policy, + policy_player=policy_player, + window=window, + max_steps=max_steps, + ) + candidates_evaluated += 1 + unified = policy.select_unified(state) + state.apply_action(state.from_unified_action(unified)) + + return { + "checkpoint": str(checkpoint), + "iteration": iteration, + "opponent": opponent, + "games": games, + "seed": seed, + "device": str(device), + "max_candidates": max_candidates, + "window": window, + "elapsed_seconds": time.perf_counter() - started, + "candidates_evaluated": candidates_evaluated, + "buckets": {key: bucket.to_dict() for key, bucket in sorted(buckets.items())}, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("checkpoints", nargs="+", type=Path) + parser.add_argument("--opponent", default="safe_heuristic_strict") + parser.add_argument("--games", type=int, default=100) + parser.add_argument("--seed", type=int, default=232_000) + parser.add_argument("--device", default="cpu") + parser.add_argument("--max-steps", type=int, default=10_000) + parser.add_argument("--max-candidates", type=int, default=400) + parser.add_argument( + "--window", + type=int, + default=3, + help="Number of policy_player turns to observe after the forced open.", + ) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + device = torch.device(args.device) + args.output.parent.mkdir(parents=True, exist_ok=True) + rows = [] + for checkpoint in args.checkpoints: + row = analyze_checkpoint( + checkpoint, + games=args.games, + seed=args.seed, + opponent=args.opponent, + device=device, + max_steps=args.max_steps, + max_candidates=args.max_candidates, + window=args.window, + ) + rows.append(row) + print(json.dumps(row, sort_keys=True)) + args.output.write_text("\n".join(json.dumps(row, sort_keys=True) for row in rows) + "\n") + print(f"wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/scripts/analyze_first_open_targets.py b/scripts/analyze_first_open_targets.py index ce07732..48dba72 100644 --- a/scripts/analyze_first_open_targets.py +++ b/scripts/analyze_first_open_targets.py @@ -345,6 +345,9 @@ def analyze_checkpoint( outcome_sampling_epsilon=cfg.traversal.outcome_sampling_epsilon, outcome_sampling_value_clip=cfg.traversal.outcome_sampling_value_clip, outcome_unsampled_regret=cfg.traversal.outcome_unsampled_regret, + outcome_unsampled_first_open_prior_alpha=getattr( + cfg.traversal, "outcome_unsampled_first_open_prior_alpha", 0.0 + ), max_depth=cfg.traversal.max_depth, max_nodes=cfg.traversal.max_nodes_per_traversal, strategy_sample_interval=cfg.traversal.strategy_sample_interval, diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/config.py b/src/coolrl_lost_cities/games/classic/deep_cfr/config.py index 3b5021f..bab9be5 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/config.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/config.py @@ -99,6 +99,7 @@ class TraversalConfig(StrictModel): outcome_sampling_epsilon: float = 0.0 outcome_sampling_value_clip: float | None = None outcome_unsampled_regret: str = "negative_node_value" + outcome_unsampled_first_open_prior_alpha: float = 0.0 cutoff_value_mode: str = "score_diff" cutoff_rollouts: int = 0 cutoff_rollout_policy: str = "random" diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/encoding.pyx b/src/coolrl_lost_cities/games/classic/deep_cfr/encoding.pyx index 062a567..f565109 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/encoding.pyx +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/encoding.pyx @@ -4,9 +4,9 @@ from coolrl_lost_cities.games.classic.game cimport GameState -cdef int DERIVED_PLAYABILITY_PER_COLOR = 19 +cdef int DERIVED_PLAYABILITY_PER_COLOR = 15 cdef int DERIVED_PLAYABILITY_COMMON = 3 -cdef int SLOT_AWARE_PLAYABILITY_PER_SLOT = 12 +cdef int SLOT_AWARE_PLAYABILITY_PER_SLOT = 6 cdef int _base_input_dim_c(GameState state) noexcept: @@ -157,7 +157,6 @@ cdef int _append_derived_playability_features_c(GameState state, int player, flo cdef float max_numeric_sum = _max_numeric_sum_c(state) cdef float max_cards_per_color = max(1, state.cards_per_color) cdef float max_wagers = max(1, state.n_handshakes) - cdef float max_score_estimate = _max_score_estimate_c(state) cdef int color cdef int is_unopened, has_only_wagers_opened, current_numeric_sum, current_wager_count cdef int current_expedition_len, last_numeric_rank, hand_count, hand_wager_count @@ -191,13 +190,9 @@ cdef int _append_derived_playability_features_c(GameState state, int player, flo out[idx + 9] = playable_hand_numeric_count / max_cards_per_color out[idx + 10] = dead_hand_numeric_count / max_cards_per_color out[idx + 11] = dead_hand_numeric_sum / max_numeric_sum - out[idx + 12] = recoverable_margin_no_bonus / max_numeric_sum - out[idx + 13] = recoverable_score_no_bonus / max_score_estimate - out[idx + 14] = min_needed_to_break_even / max_numeric_sum - out[idx + 15] = discard_top_playable_flag - out[idx + 16] = discard_top_playable_value / max_numeric_sum - out[idx + 17] = unknown_remaining_count / max_cards_per_color - out[idx + 18] = cards_needed_for_bonus / max_cards_per_color + out[idx + 12] = discard_top_playable_flag + out[idx + 13] = discard_top_playable_value / max_numeric_sum + out[idx + 14] = unknown_remaining_count / max_cards_per_color idx += DERIVED_PLAYABILITY_PER_COLOR out[idx] = state.deck_len / max(1, state.total_cards) @@ -207,8 +202,6 @@ cdef int _append_derived_playability_features_c(GameState state, int player, flo cdef int _append_slot_aware_playability_features_c(GameState state, int player, float* out, int idx) noexcept: - cdef float max_numeric_sum = _max_numeric_sum_c(state) - cdef float max_score_estimate = _max_score_estimate_c(state) cdef int slot cdef int card cdef int color @@ -230,9 +223,6 @@ cdef int _append_slot_aware_playability_features_c(GameState state, int player, cdef bint is_wager_before_numeric cdef bint is_numeric_open cdef bint is_wager_first_open - cdef bint is_bad_open_candidate - cdef bint is_safe_continuation - cdef float open_risk_score for slot in range(state.hand_size): if slot >= state.hand_lens[player]: @@ -264,22 +254,13 @@ cdef int _append_slot_aware_playability_features_c(GameState state, int player, is_playable_to_existing = legal_play and has_numeric_started is_dead_numeric = is_numeric and not legal_play and rank <= last_numeric_rank is_wager_before_numeric = is_wager and legal_play and not has_numeric_started - is_bad_open_candidate = would_start_color_commitment and recoverable_score_no_bonus < 0 - open_risk_score = min(0.0, recoverable_score_no_bonus) if would_start_color_commitment else 0.0 - is_safe_continuation = (not would_start_color_commitment) and is_playable_to_existing - out[idx] = recoverable_score_no_bonus / max_score_estimate - out[idx + 1] = recoverable_margin_no_bonus / max_numeric_sum - out[idx + 2] = would_start_color_commitment - out[idx + 3] = is_numeric_open - out[idx + 4] = is_wager_first_open - out[idx + 5] = is_playable_to_existing - out[idx + 6] = is_dead_numeric - out[idx + 7] = is_wager_before_numeric - out[idx + 8] = has_bonus_path - out[idx + 9] = is_bad_open_candidate - out[idx + 10] = open_risk_score / max_score_estimate - out[idx + 11] = is_safe_continuation + out[idx] = would_start_color_commitment + out[idx + 1] = is_numeric_open + out[idx + 2] = is_wager_first_open + out[idx + 3] = is_playable_to_existing + out[idx + 4] = is_dead_numeric + out[idx + 5] = is_wager_before_numeric idx += SLOT_AWARE_PLAYABILITY_PER_SLOT return idx diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/interleaved_traversal.py b/src/coolrl_lost_cities/games/classic/deep_cfr/interleaved_traversal.py index ea50ca1..f9354a5 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/interleaved_traversal.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/interleaved_traversal.py @@ -106,6 +106,63 @@ def _has_legal_first_open(state: GameState, player: int, legal_mask: np.ndarray) return False +def _first_open_recoverable_score(state: GameState, player: int, color: int) -> float: + """Visible-only recoverable score for an unopened expedition. + + Mirrors evaluate._visible_recoverable_summary for the first-open case + (empty expedition, last_numeric == 0). Used as a weak prior signal for + unsampled first-open actions in outcome sampling. + """ + config = state.config + proj_sum = 0 + proj_wagers = 0 + for card in state.hand_slots(player): + if card is None or int(card.color) != color: + continue + if card.rank == 0: + proj_wagers += 1 + else: + proj_sum += config.min_rank + card.rank - 1 + margin = proj_sum + config.expedition_penalty + return float(margin * (proj_wagers + 1)) + + +def _apply_first_open_prior( + target: np.ndarray, + state: GameState, + player: int, + legal_mask: np.ndarray, + sampled_action: int, + alpha: float, +) -> None: + """Overwrite first-open play targets (other than the sampled action) with ±alpha. + + Sign is taken from the visible recoverable_score for the candidate's color. + Only legal play actions whose color has an empty expedition are touched. + """ + if alpha == 0.0: + return + card_action_size = state.config.hand_size * 2 + hand = state.hand_slots(player) + expeditions = state.expeditions[player] + score_by_color: dict[int, float] = {} + for unified_action in np.flatnonzero(legal_mask): + action = int(unified_action) + if action == sampled_action: + continue + if action >= card_action_size or action % 2 == 1: + continue + card = hand[action // 2] + if card is None: + continue + color = int(card.color) + if expeditions[color]: + continue + if color not in score_by_color: + score_by_color[color] = _first_open_recoverable_score(state, player, color) + target[action] = alpha if score_by_color[color] >= 0.0 else -alpha + + def _record_endpoint(stats: TraversalStats, depth: int, width: int, max_depth: int) -> None: stats.endpoint_depth_sum += depth start = (depth // width) * width @@ -121,6 +178,7 @@ class InterleavedTraversalConfig: outcome_sampling_epsilon: float outcome_sampling_value_clip: float | None outcome_unsampled_regret: str + outcome_unsampled_first_open_prior_alpha: float max_depth: int | None max_nodes: int | None strategy_sample_interval: int @@ -402,6 +460,16 @@ class InterleavedContext: if self.cfg.outcome_unsampled_regret == "negative_node_value": target[frame.legal_mask] = -node_value target[frame.action] = sampled_action_value - node_value + is_first_open = _has_legal_first_open(self.state, frame.player, frame.legal_mask) + if is_first_open and self.cfg.outcome_unsampled_first_open_prior_alpha != 0.0: + _apply_first_open_prior( + target, + self.state, + frame.player, + frame.legal_mask, + frame.action, + self.cfg.outcome_unsampled_first_open_prior_alpha, + ) self.samples.advantage.append( TrainingSample( info_state=frame.info_state, @@ -409,7 +477,7 @@ class InterleavedContext: legal_mask=frame.legal_mask.copy(), iteration=self.iteration, player=frame.player, - is_first_open=_has_legal_first_open(self.state, frame.player, frame.legal_mask), + is_first_open=is_first_open, ) ) self.stats.advantage_samples += 1 @@ -635,6 +703,7 @@ def run_interleaved_traversal_batch( outcome_sampling_value_clip: float | None, outcome_unsampled_regret: str, opponent_policy: str, + outcome_unsampled_first_open_prior_alpha: float = 0.0, endpoint_depth_bucket_width: int, endpoint_depth_bucket_max: int, seed: int, @@ -650,6 +719,7 @@ def run_interleaved_traversal_batch( outcome_sampling_epsilon=outcome_sampling_epsilon, outcome_sampling_value_clip=outcome_sampling_value_clip, outcome_unsampled_regret=outcome_unsampled_regret, + outcome_unsampled_first_open_prior_alpha=outcome_unsampled_first_open_prior_alpha, max_depth=max_depth, max_nodes=max_nodes, strategy_sample_interval=strategy_sample_interval, diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py index 6b7b123..3aa0d5c 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py @@ -445,6 +445,9 @@ class DeepCFRTrainer: self.config.traversal.outcome_sampling_value_clip ), outcome_unsampled_regret=(self.config.traversal.outcome_unsampled_regret), + outcome_unsampled_first_open_prior_alpha=( + self.config.traversal.outcome_unsampled_first_open_prior_alpha + ), opponent_policy=self.config.traversal.opponent_policy, endpoint_depth_bucket_width=( self.config.traversal.endpoint_depth_bucket_width diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py b/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py index cbda874..7c562a9 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py @@ -187,6 +187,9 @@ def run_traversal_worker_batch(batch: TraversalWorkerBatch) -> TraversalWorkerRe outcome_sampling_epsilon=cfg.traversal.outcome_sampling_epsilon, outcome_sampling_value_clip=cfg.traversal.outcome_sampling_value_clip, outcome_unsampled_regret=cfg.traversal.outcome_unsampled_regret, + outcome_unsampled_first_open_prior_alpha=( + cfg.traversal.outcome_unsampled_first_open_prior_alpha + ), opponent_policy=cfg.traversal.opponent_policy, endpoint_depth_bucket_width=cfg.traversal.endpoint_depth_bucket_width, endpoint_depth_bucket_max=cfg.traversal.endpoint_depth_bucket_max, diff --git a/tests/games/classic/test_deep_cfr_trainer.py b/tests/games/classic/test_deep_cfr_trainer.py index 754ca80..492027c 100644 --- a/tests/games/classic/test_deep_cfr_trainer.py +++ b/tests/games/classic/test_deep_cfr_trainer.py @@ -27,6 +27,8 @@ from coolrl_lost_cities.games.classic.deep_cfr.cli import ( from coolrl_lost_cities.games.classic.deep_cfr.config import DeepCFRConfig, load_config from coolrl_lost_cities.games.classic.deep_cfr.evaluate import evaluate_strategy_network from coolrl_lost_cities.games.classic.deep_cfr.interleaved_traversal import ( + _apply_first_open_prior, + _first_open_recoverable_score, run_interleaved_traversal_batch, ) from coolrl_lost_cities.games.classic.deep_cfr.memory import ReservoirMemory, TrainingSample @@ -283,8 +285,8 @@ def test_deep_cfr_playability_encoding_extends_input_shape() -> None: derived_dim = input_dim(state, derived_config.encoding) slot_dim = input_dim(state, slot_config.encoding) - assert derived_dim == base_dim + state.config.n_colors * 19 + 3 - assert slot_dim == derived_dim + state.config.hand_size * 12 + assert derived_dim == base_dim + state.config.n_colors * 15 + 3 + assert slot_dim == derived_dim + state.config.hand_size * 6 assert encode_info_state(state, 0, slot_config.encoding).shape == (slot_dim,) @@ -1399,3 +1401,57 @@ def test_deep_cfr_trainer_does_not_log_eval_warning_when_eval_disabled(tmp_path) trainer.train() log_text = (tmp_path / "train.log").read_text() assert "WARNING evaluation will not run" not in log_text + + +def test_first_open_prior_overrides_unsampled_play_targets_with_signed_alpha() -> None: + state = GameState.new_game(LostCitiesConfig(seed=123), seed=123) + player = int(state.current_player) + card_action_size = state.config.hand_size * 2 + legal_mask = np.zeros(card_action_size + 5, dtype=bool) + play_actions: list[int] = [] + for slot, card in enumerate(state.hand_slots(player)): + if card is None: + continue + play_actions.append(2 * slot) + legal_mask[2 * slot] = True + legal_mask[2 * slot + 1] = True + assert play_actions, "fresh game state should have legal play actions" + + sampled_action = play_actions[0] + target = np.zeros(legal_mask.shape[0], dtype=np.float32) + target[sampled_action] = 7.0 + + alpha = 5.0 + _apply_first_open_prior(target, state, player, legal_mask, sampled_action, alpha) + + assert target[sampled_action] == pytest.approx(7.0) + + expeditions = state.expeditions[player] + hand = state.hand_slots(player) + for action in play_actions: + if action == sampled_action: + continue + card = hand[action // 2] + color = int(card.color) + if expeditions[color]: + assert target[action] == 0.0 + continue + score = _first_open_recoverable_score(state, player, color) + expected = alpha if score >= 0.0 else -alpha + assert target[action] == pytest.approx(expected) + assert target[action + 1] == 0.0 + + +def test_first_open_prior_zero_alpha_is_noop() -> None: + state = GameState.new_game(LostCitiesConfig(seed=7), seed=7) + player = int(state.current_player) + card_action_size = state.config.hand_size * 2 + legal_mask = np.zeros(card_action_size + 5, dtype=bool) + for slot, card in enumerate(state.hand_slots(player)): + if card is None: + continue + legal_mask[2 * slot] = True + legal_mask[2 * slot + 1] = True + target = np.zeros(legal_mask.shape[0], dtype=np.float32) + _apply_first_open_prior(target, state, player, legal_mask, sampled_action=0, alpha=0.0) + assert np.all(target == 0.0)