diff --git a/configs/deep_cfr/default.yaml b/configs/deep_cfr/default.yaml index 0665e0f..ef7d719 100644 --- a/configs/deep_cfr/default.yaml +++ b/configs/deep_cfr/default.yaml @@ -31,7 +31,7 @@ traversal: max_depth: null max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 - outcome_sampling_epsilon: 0.2 + outcome_sampling_epsilon: 0.05 outcome_sampling_value_clip: 500.0 outcome_unsampled_regret: zero cutoff_value_mode: score_diff diff --git a/docs/plans/deep-cfr-selectivity.md b/docs/plans/deep-cfr-selectivity.md index eb1f61f..056d590 100644 --- a/docs/plans/deep-cfr-selectivity.md +++ b/docs/plans/deep-cfr-selectivity.md @@ -608,7 +608,42 @@ architecture has been run yet. Fair test deferred — the diagnosis from sections 4–5 (selection bias, post-open behaviour) suggests that even a correct color-aware encoder would not break the closed loop on its own. -### 8. Short open-selectivity ablation +### 8. Interleaved scheduler honours all_negative_fallback (2026-05-10) + +The interleaved traversal scheduler's `_regret_matching` was hard-coded to +spread fallback policy uniformly across legal actions, regardless of the +configured `regret_matching.all_negative_fallback`. The default config has +shipped with `all_negative_fallback: argmax_tiebreak` since +`618d5f8 Promote avg-strategy 1000iter to default.yaml` based on prior +20-iteration audit + 1000-iteration empirical evidence (see +`docs/archive/deep-cfr-regret-fallback-audit-2026-05-07.md`), but the +default scheduler was switched to interleaved in `09bbe7c Make interleaved +traversal the default`, after which the configured fallback mode silently +no-op'd in interleaved code paths. + +Fix: + +- `_regret_matching(advantages, legal_mask, epsilon, fallback_mode="uniform")` + in `interleaved_traversal.py` now honours `argmax_tiebreak` by + concentrating policy mass on the lowest-index tied action (deterministic + tiebreak; the Cython recursive traverser randomises ties using its + per-traverser RNG, which the batched policy does not have). +- `BatchedPolicy` accepts `fallback_mode` and threads it through. +- `InterleavedTraversalConfig` carries `all_negative_fallback`. +- `run_interleaved_traversal_batch`, `trainer.py`, `workers.py`, and the + `analyze_first_open_targets.py` callers pass the field through. + +Also bumped `traversal.outcome_sampling_epsilon` in `default.yaml` from +0.2 to 0.05. The 200-iteration sweep (section 1) showed 0.05 produced the +best short-run safe_heuristic_strict score diff (-40.01 vs -57.87 for +0.20). All recent experimental runs already used 0.05; the default now +matches actual experimental practice. + +These changes do not target the diagnosed selection-bias bottleneck. They +align config intent with actual scheduler behaviour and make the default +config reproduce known-best knob settings out of the box. + +### 9. Short open-selectivity ablation Run a 200-300 iteration ablation only after the target audit identifies a specific change. Candidate changes include: diff --git a/scripts/analyze_first_open_targets.py b/scripts/analyze_first_open_targets.py index 48dba72..3206627 100644 --- a/scripts/analyze_first_open_targets.py +++ b/scripts/analyze_first_open_targets.py @@ -348,6 +348,7 @@ def analyze_checkpoint( outcome_unsampled_first_open_prior_alpha=getattr( cfg.traversal, "outcome_unsampled_first_open_prior_alpha", 0.0 ), + all_negative_fallback=cfg.regret_matching.all_negative_fallback, max_depth=cfg.traversal.max_depth, max_nodes=cfg.traversal.max_nodes_per_traversal, strategy_sample_interval=cfg.traversal.strategy_sample_interval, diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/interleaved_traversal.py b/src/coolrl_lost_cities/games/classic/deep_cfr/interleaved_traversal.py index f9354a5..3f49bb0 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/interleaved_traversal.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/interleaved_traversal.py @@ -41,7 +41,20 @@ def _regret_matching( advantages: np.ndarray, legal_mask: np.ndarray, epsilon: float, + fallback_mode: str = "uniform", ) -> tuple[np.ndarray, bool, int, bool]: + """Compute regret-matching policy with fallback when no positive regrets. + + ``fallback_mode``: + - ``"uniform"``: spread mass uniformly across all legal actions. + - ``"argmax_tiebreak"``: concentrate mass on a single best-advantage + action (lowest index among ties), matching the spirit of the + Cython recursive traverser's argmax_tiebreak fallback. Note: the + Cython traverser randomises the tiebreak using its per-traverser + RNG; the interleaved scheduler uses deterministic lowest-index + selection so behaviour stays reproducible without per-trajectory + RNG plumbing into the batched policy. + """ policy = np.zeros_like(advantages, dtype=np.float32) legal = np.flatnonzero(legal_mask) positives = np.maximum(advantages[legal], 0.0) @@ -59,7 +72,10 @@ def _regret_matching( tied = legal[np.flatnonzero(advantages[legal] == best)] tie_size = int(len(tied)) full_tie = tie_size > 1 and tie_size == len(legal) - policy[legal] = 1.0 / float(len(legal)) + if fallback_mode == "argmax_tiebreak": + policy[int(tied[0])] = 1.0 + else: + policy[legal] = 1.0 / float(len(legal)) return policy, fallback, tie_size, full_tie @@ -179,6 +195,7 @@ class InterleavedTraversalConfig: outcome_sampling_value_clip: float | None outcome_unsampled_regret: str outcome_unsampled_first_open_prior_alpha: float + all_negative_fallback: str max_depth: int | None max_nodes: int | None strategy_sample_interval: int @@ -257,12 +274,14 @@ class BatchedPolicy: epsilon: float, strategy_network: torch.nn.Module | None = None, deterministic: bool = False, + fallback_mode: str = "uniform", ) -> None: self.networks = networks self.strategy_network = strategy_network self.device = device self.epsilon = epsilon self.deterministic = deterministic + self.fallback_mode = fallback_mode self.batch_sizes: list[int] = [] self.forward_seconds = 0.0 @@ -312,7 +331,10 @@ class BatchedPolicy: full_tie = False else: policy, fallback, tie_size, full_tie = _regret_matching( - values[local_idx], request.legal_mask, self.epsilon + values[local_idx], + request.legal_mask, + self.epsilon, + self.fallback_mode, ) out[request_idx] = PolicyResult( info_state=request.info_state, @@ -704,6 +726,7 @@ def run_interleaved_traversal_batch( outcome_unsampled_regret: str, opponent_policy: str, outcome_unsampled_first_open_prior_alpha: float = 0.0, + all_negative_fallback: str = "uniform", endpoint_depth_bucket_width: int, endpoint_depth_bucket_max: int, seed: int, @@ -720,6 +743,7 @@ def run_interleaved_traversal_batch( outcome_sampling_value_clip=outcome_sampling_value_clip, outcome_unsampled_regret=outcome_unsampled_regret, outcome_unsampled_first_open_prior_alpha=outcome_unsampled_first_open_prior_alpha, + all_negative_fallback=all_negative_fallback, max_depth=max_depth, max_nodes=max_nodes, strategy_sample_interval=strategy_sample_interval, @@ -740,6 +764,7 @@ def run_interleaved_traversal_batch( epsilon=cfg.epsilon, strategy_network=strategy_network, deterministic=cfg.deterministic, + fallback_mode=cfg.all_negative_fallback, ) scheduler = InterleavedTraversalScheduler(cfg, policy) _values, _rng_out, stats_rows, sample_rows, batch_sizes = scheduler.run( diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py index 3aa0d5c..25dd1e8 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py @@ -448,6 +448,7 @@ class DeepCFRTrainer: outcome_unsampled_first_open_prior_alpha=( self.config.traversal.outcome_unsampled_first_open_prior_alpha ), + all_negative_fallback=(self.config.regret_matching.all_negative_fallback), opponent_policy=self.config.traversal.opponent_policy, endpoint_depth_bucket_width=( self.config.traversal.endpoint_depth_bucket_width diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py b/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py index 7c562a9..2d67ce5 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py @@ -190,6 +190,7 @@ def run_traversal_worker_batch(batch: TraversalWorkerBatch) -> TraversalWorkerRe outcome_unsampled_first_open_prior_alpha=( cfg.traversal.outcome_unsampled_first_open_prior_alpha ), + all_negative_fallback=cfg.regret_matching.all_negative_fallback, opponent_policy=cfg.traversal.opponent_policy, endpoint_depth_bucket_width=cfg.traversal.endpoint_depth_bucket_width, endpoint_depth_bucket_max=cfg.traversal.endpoint_depth_bucket_max, diff --git a/tests/games/classic/test_deep_cfr_trainer.py b/tests/games/classic/test_deep_cfr_trainer.py index 492027c..274bf73 100644 --- a/tests/games/classic/test_deep_cfr_trainer.py +++ b/tests/games/classic/test_deep_cfr_trainer.py @@ -1455,3 +1455,42 @@ def test_first_open_prior_zero_alpha_is_noop() -> None: target = np.zeros(legal_mask.shape[0], dtype=np.float32) _apply_first_open_prior(target, state, player, legal_mask, sampled_action=0, alpha=0.0) assert np.all(target == 0.0) + + +def test_interleaved_regret_matching_argmax_tiebreak_concentrates_on_best() -> None: + from coolrl_lost_cities.games.classic.deep_cfr.interleaved_traversal import _regret_matching + + advantages = np.array([-1.0, -0.5, -0.5, -2.0, -0.5], dtype=np.float32) + legal_mask = np.array([True, True, True, True, True]) + + uniform_policy, fallback_u, _, _ = _regret_matching( + advantages, legal_mask, epsilon=1.0e-8, fallback_mode="uniform" + ) + argmax_policy, fallback_a, tie_size, _ = _regret_matching( + advantages, legal_mask, epsilon=1.0e-8, fallback_mode="argmax_tiebreak" + ) + + assert fallback_u is True + assert fallback_a is True + assert tie_size == 3 + assert np.allclose(uniform_policy, np.full(5, 0.2, dtype=np.float32)) + expected_argmax = np.zeros(5, dtype=np.float32) + expected_argmax[1] = 1.0 + assert np.allclose(argmax_policy, expected_argmax) + + +def test_interleaved_regret_matching_no_fallback_unchanged_by_mode() -> None: + from coolrl_lost_cities.games.classic.deep_cfr.interleaved_traversal import _regret_matching + + advantages = np.array([1.0, 3.0, 0.0, 2.0], dtype=np.float32) + legal_mask = np.array([True, True, True, True]) + policy_uniform, fallback_u, _, _ = _regret_matching( + advantages, legal_mask, epsilon=1.0e-8, fallback_mode="uniform" + ) + policy_argmax, fallback_a, _, _ = _regret_matching( + advantages, legal_mask, epsilon=1.0e-8, fallback_mode="argmax_tiebreak" + ) + assert fallback_u is False + assert fallback_a is False + assert np.allclose(policy_uniform, policy_argmax) + assert np.allclose(policy_uniform.sum(), 1.0)