coolguy/coorl-lost-cities
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add discard_only opponent_policy + analyze.py merges

Browse Source 
- Plumb discard_only through config validator and interleaved_traversal.
  Bypasses PolicyRequest for opponent nodes, uses DiscardOnlyBot via
  Snapshot. Recursive scheduler explicitly rejected (Cython unchanged).
- analyze.py: merge per-opponent eval plots into multi-line plots, add
  twin y-axis support (PlotSpec.secondary_metrics), per-axes translucent
  legends instead of one global legend, add avg_game_length to GameFlow.
- Tests: discard_only smoke run, validator accept/reject, all 57 pass.
This commit is contained in:
coolguy
2026-05-10 17:39:00 +09:00
parent 4ac74e2501
commit e3f2423f46
4 changed files with 201 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/coolrl_lost_cities/games/classic/deep_cfr/analyze.py
+99 -88
View File
@@ -19,6 +19,9 @@ class PlotSpec:
kind: str = "eval"
fixed_ylim: tuple[float, float] | None = None
opponents: tuple[str, ...] | None = None
secondary_metrics: tuple[str, ...] = ()
secondary_ylabel: str | None = None
secondary_scale: float = 1.0
@dataclass(frozen=True)
@@ -33,64 +36,45 @@ SECTIONS: tuple[SectionSpec, ...] = (
"Core",
"analysis_00_core.png",
(
PlotSpec("Advantage Loss", ("loss/advantage",), "loss", kind="train"),
PlotSpec("Strategy Loss", ("loss/strategy",), "loss", kind="train"),
PlotSpec(
"Avg Score Diff (heuristic_cautious)",
("avg_score_diff0",),
"score diff",
opponents=("heuristic_cautious",),
"Losses (advantage / strategy)",
("loss/advantage",),
"advantage MSE",
kind="train",
secondary_metrics=("loss/strategy",),
secondary_ylabel="strategy CE",
),
PlotSpec("Avg Score Diff (all opponents)", ("avg_score_diff0",), "score diff"),
PlotSpec(
"Win Rate (heuristic_cautious)",
"Win Rate (all opponents)",
("win_rate0",),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
opponents=("heuristic_cautious",),
),
PlotSpec(
"Win Rate (random)",
("win_rate0",),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
opponents=("random",),
),
PlotSpec(
"Avg Opened Colors (heuristic_cautious)",
"Avg Opened Colors (all opponents)",
("avg_opened_colors",),
"colors",
fixed_ylim=(0, 5),
opponents=("heuristic_cautious",),
),
PlotSpec(
"Positive Expedition Rate (heuristic_cautious)",
("positive_expedition_rate",),
"Positive / Bonus Expedition Rate (heuristic_cautious)",
("positive_expedition_rate", "bonus_expedition_rate"),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
opponents=("heuristic_cautious",),
),
PlotSpec(
"Bonus Expedition Rate (heuristic_cautious)",
("bonus_expedition_rate",),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
opponents=("heuristic_cautious",),
),
PlotSpec(
"Score per Opened Color (heuristic_cautious)",
"Score per Opened Color (all opponents)",
("score_per_opened_color",),
"score / color",
opponents=("heuristic_cautious",),
),
PlotSpec(
"Policy Entropy (heuristic_cautious)",
"Policy Entropy (all opponents)",
("policy_entropy",),
"entropy",
opponents=("heuristic_cautious",),
),
),
),
@@ -98,8 +82,14 @@ SECTIONS: tuple[SectionSpec, ...] = (
"Loss",
"analysis_01_loss.png",
(
PlotSpec("Advantage Loss", ("loss/advantage",), "loss", kind="train"),
PlotSpec("Strategy Loss", ("loss/strategy",), "loss", kind="train"),
PlotSpec(
"Losses (advantage / strategy)",
("loss/advantage",),
"advantage MSE",
kind="train",
secondary_metrics=("loss/strategy",),
secondary_ylabel="strategy CE",
),
PlotSpec(
"Samples",
("samples/advantage", "samples/strategy"),
@@ -129,32 +119,12 @@ SECTIONS: tuple[SectionSpec, ...] = (
"analysis_03_action.png",
(
PlotSpec(
"Play Action Rate",
("play_action_rate",),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
),
PlotSpec(
"Discard Action Rate",
("discard_action_rate",),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
),
PlotSpec(
"Draw Deck Rate",
("draw_deck_rate",),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
),
PlotSpec(
"Draw Pile Rate",
("draw_pile_rate",),
"Action Rates (heuristic_cautious)",
("play_action_rate", "discard_action_rate", "draw_deck_rate", "draw_pile_rate"),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
opponents=("heuristic_cautious",),
),
),
),
@@ -171,6 +141,7 @@ SECTIONS: tuple[SectionSpec, ...] = (
fixed_ylim=(0, 100),
),
PlotSpec("Expedition Cards", ("avg_expedition_cards",), "cards"),
PlotSpec("Avg Game Length", ("avg_game_length",), "steps"),
),
),
SectionSpec(
@@ -178,28 +149,19 @@ SECTIONS: tuple[SectionSpec, ...] = (
"analysis_06_expedition_outcomes.png",
(
PlotSpec(
"Positive Expedition Rate",
("positive_expedition_rate",),
"Expedition Outcome Rates (heuristic_cautious)",
(
"positive_expedition_rate",
"negative_expedition_rate",
"bonus_expedition_rate",
),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
opponents=("heuristic_cautious",),
),
PlotSpec(
"Negative Expedition Rate",
("negative_expedition_rate",),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
),
PlotSpec(
"Bonus Expedition Rate",
("bonus_expedition_rate",),
"rate (%)",
scale=100.0,
fixed_ylim=(0, 100),
),
PlotSpec(
"Per-Game Expedition Counts",
"Per-Game Expedition Counts (heuristic_cautious)",
(
"per_game_positive_expeditions",
"per_game_negative_expeditions",
@@ -207,9 +169,18 @@ SECTIONS: tuple[SectionSpec, ...] = (
"per_game_below_minus_20_expeditions",
),
"expeditions / game",
opponents=("heuristic_cautious",),
),
PlotSpec(
"Final Expedition Score (all opponents)",
("avg_final_score_per_opened_expedition",),
"score",
),
PlotSpec(
"Score per Opened Color (all opponents)",
("score_per_opened_color",),
"score / color",
),
PlotSpec("Final Expedition Score", ("avg_final_score_per_opened_expedition",), "score"),
PlotSpec("Score per Opened Color", ("score_per_opened_color",), "score / color"),
),
),
SectionSpec(
@@ -441,19 +412,13 @@ def plot_section(
for ax in axes_flat[next_axis:]:
ax.axis("off")
if section.name != "Traversal":
handles, labels = _legend_items(axes_flat)
if handles:
fig.legend(
handles, labels, loc="upper center", ncols=min(len(labels), 6), fontsize="small"
)
suffix = f" ({smoothing_window}-iter moving average)" if smoothing_window > 1 else ""
fig.suptitle(
f"Lost Cities Deep CFR {section.name} metrics{suffix}",
fontsize=14,
fontweight="bold",
)
fig.tight_layout(rect=(0, 0, 1, 0.95))
fig.tight_layout(rect=(0, 0, 1, 0.97))
if not plotted_any:
plt.close(fig)
return False
@@ -598,6 +563,35 @@ def _plot_train_spec(
)
or plotted
)
if spec.secondary_metrics:
ax2 = ax.twinx()
secondary_palette = ("tab:red", "tab:purple", "tab:brown", "tab:olive")
for idx, metric in enumerate(spec.secondary_metrics):
pairs = []
for row in rows:
if "iteration" not in row:
continue
value = _train_value(row, metric)
if value is None:
continue
pairs.append((int(row["iteration"]), value * spec.secondary_scale))
color = (
_train_metric_color(metric, section_title=spec.title)
or secondary_palette[idx % len(secondary_palette)]
)
plotted = (
_plot_pairs(
ax2,
pairs,
label=_train_metric_label(metric),
color=color,
smoothing_window=smoothing_window,
)
or plotted
)
if spec.secondary_ylabel:
ax2.set_ylabel(spec.secondary_ylabel)
ax2.grid(False)
return plotted
@@ -632,8 +626,10 @@ def _plot_eval_spec(
) -> bool:
plotted = False
multi_metric = len(spec.metrics) > 1
color_by_metric = multi_metric and len(opponents) == 1
metric_palette = ("tab:blue", "tab:orange", "tab:green", "tab:red", "tab:purple", "tab:brown")
for opponent in opponents:
for metric in spec.metrics:
for idx, metric in enumerate(spec.metrics):
pairs: list[tuple[int, float]] = []
for row in rows:
if "iteration" not in row:
@@ -645,16 +641,23 @@ def _plot_eval_spec(
value = float("nan")
pairs.append((int(row["iteration"]), value * spec.scale))
if color_by_metric:
label = _short_metric_label(metric)
color = metric_palette[idx % len(metric_palette)]
linestyle = "-"
else:
label = opponent
if multi_metric:
label = f"{opponent}: {_short_metric_label(metric)}"
color = _opponent_color(opponent)
linestyle = _metric_linestyle(metric) if multi_metric else "-"
plotted = (
_plot_pairs(
ax,
pairs,
label=label,
color=_opponent_color(opponent),
linestyle=_metric_linestyle(metric) if multi_metric else "-",
color=color,
linestyle=linestyle,
smoothing_window=smoothing_window,
)
or plotted
@@ -823,9 +826,17 @@ def _finish_axis(
ax.set_ylim(*fixed_ylim)
ax.grid(True, alpha=0.3)
if plotted:
handles, _ = ax.get_legend_handles_labels()
handles, labels = ax.get_legend_handles_labels()
for sibling in ax.figure.axes:
if sibling is ax:
continue
if sibling.bbox.bounds != ax.bbox.bounds:
continue
twin_handles, twin_labels = sibling.get_legend_handles_labels()
handles.extend(twin_handles)
labels.extend(twin_labels)
if handles:
ax.legend(loc="best", fontsize="x-small")
ax.legend(handles, labels, loc="best", fontsize="x-small", framealpha=0.7, frameon=True)
else:
ax.text(0.5, 0.5, "No data", ha="center", va="center", transform=ax.transAxes)
src/coolrl_lost_cities/games/classic/deep_cfr/config.py
+17 -4
View File
@@ -150,9 +150,17 @@ class TraversalConfig(StrictModel):
@field_validator("opponent_policy")
@classmethod
def _validate_opponent_policy(cls, value: str) -> str:
if value not in {"network", "heuristic_balanced", "self_play_league", "average_strategy"}:
allowed = {
"network",
"heuristic_balanced",
"self_play_league",
"average_strategy",
"discard_only",
}
if value not in allowed:
raise ValueError(
"must be 'network', 'heuristic_balanced', 'self_play_league', or 'average_strategy'"
"must be 'network', 'heuristic_balanced', 'self_play_league', "
"'average_strategy', or 'discard_only'"
)
return value
@@ -189,10 +197,10 @@ class TraversalConfig(StrictModel):
if self.scheduler == "interleaved":
if self.sampling_mode != "outcome":
raise ValueError("scheduler='interleaved' currently supports only outcome sampling")
if self.opponent_policy not in {"network", "average_strategy"}:
if self.opponent_policy not in {"network", "average_strategy", "discard_only"}:
raise ValueError(
"scheduler='interleaved' currently supports only "
"opponent_policy='network' or 'average_strategy'"
"opponent_policy='network', 'average_strategy', or 'discard_only'"
)
if self.cutoff_rollouts != 0 or self.cutoff_value_mode != "score_diff":
raise ValueError(
@@ -207,6 +215,11 @@ class TraversalConfig(StrictModel):
raise ValueError("interleave_width must be positive")
if self.interleave_max_batch <= 0:
raise ValueError("interleave_max_batch must be positive")
if self.scheduler == "recursive" and self.opponent_policy == "discard_only":
raise ValueError(
"opponent_policy='discard_only' is currently only supported with "
"scheduler='interleaved'."
)
return self
def resolved_num_workers(self, batches: int | None = None) -> int:
src/coolrl_lost_cities/games/classic/deep_cfr/interleaved_traversal.py
+29 -1
View File
@@ -8,10 +8,12 @@ from typing import Any
import numpy as np
import torch
from coolrl_lost_cities.games.classic.bots.discard_only import DiscardOnlyBot
from coolrl_lost_cities.games.classic.deep_cfr.encoding import encode_info_state
from coolrl_lost_cities.games.classic.deep_cfr.memory import TrainingSample
from coolrl_lost_cities.games.classic.deep_cfr.traversal_stats import TraversalStats
from coolrl_lost_cities.games.classic.game import GameState
from coolrl_lost_cities.games.classic.snapshots import snapshot_from_state
def _next_u32(state: int) -> tuple[int, int]:
@@ -376,6 +378,9 @@ class InterleavedContext:
self.last_value = 0.0
self.done = False
self.value = 0.0
self._discard_only_bot: DiscardOnlyBot | None = (
DiscardOnlyBot() if cfg.opponent_policy == "discard_only" else None
)
def advance_until_policy(self, context_index: int) -> None:
while not self.done and self.pending is None and self.stack:
@@ -454,9 +459,32 @@ class InterleavedContext:
self._return_value(cutoff)
return
player = int(self.state.current_player)
legal_actions = self.state.unified_legal_actions()
if not legal_actions:
self.stats.terminals += 1
_record_endpoint(
self.stats,
depth,
self.cfg.endpoint_depth_bucket_width,
self.cfg.endpoint_depth_bucket_max,
)
self._return_value(float(self.state.score_diff(self.traverser)))
return
if player != self.traverser and self._discard_only_bot is not None:
snapshot = snapshot_from_state(self.state)
action = int(
self._discard_only_bot._act_unified(
snapshot.phase, snapshot.legal_mask, snapshot.card_action_size
)
)
swapped_deck_index = self._sample_deck_draw_chance(action)
self.state.push_unified_action(action)
self.stack.append(FixedActionFrame(swapped_deck_index=swapped_deck_index))
self.stack.append(EnterFrame(depth + 1))
return
info_state = encode_info_state(self.state, player, self.cfg.encoding)
legal_mask = np.zeros(self.cfg.action_size, dtype=bool)
legal_mask[self.state.unified_legal_actions()] = True
legal_mask[legal_actions] = True
network_kind = (
"strategy"
if player != self.traverser and self.cfg.opponent_policy == "average_strategy"
tests/games/classic/test_deep_cfr_trainer.py
+53 -1
View File
@@ -138,7 +138,9 @@ def test_deep_cfr_config_accepts_interleaved_scheduler() -> None:
def test_deep_cfr_config_rejects_unsupported_interleaved_options() -> None:
with pytest.raises(ValueError, match="opponent_policy='network' or 'average_strategy'"):
with pytest.raises(
ValueError, match="opponent_policy='network', 'average_strategy', or 'discard_only'"
):
_deep_cfr_config(
{"traversal": {"scheduler": "interleaved", "opponent_policy": "self_play_league"}}
)
@@ -154,6 +156,20 @@ def test_deep_cfr_config_rejects_unsupported_interleaved_options() -> None:
)
def test_deep_cfr_config_accepts_discard_only_with_interleaved() -> None:
config = _deep_cfr_config(
{"traversal": {"scheduler": "interleaved", "opponent_policy": "discard_only"}}
)
assert config.traversal.opponent_policy == "discard_only"
def test_deep_cfr_config_rejects_discard_only_with_recursive() -> None:
with pytest.raises(ValueError, match="discard_only.*scheduler='interleaved'"):
_deep_cfr_config(
{"traversal": {"scheduler": "recursive", "opponent_policy": "discard_only"}}
)
def test_deep_cfr_train_cli_checkpoint_save_overrides() -> None:
args = type(
"Args",
@@ -452,6 +468,42 @@ def test_deep_cfr_trainer_interleaved_scheduler_smoke_run(tmp_path) -> None:
assert runtime["interleaved/avg_batch_size"] >= 1.0
def test_deep_cfr_trainer_discard_only_opponent_smoke_run(tmp_path) -> None:
trainer = DeepCFRTrainer(
_deep_cfr_config(
{
"run": {"max_iterations": 1, "seed": 25},
"network": {"hidden_size": 16},
"traversal": {
"scheduler": "interleaved",
"opponent_policy": "discard_only",
"traversals_per_player": 2,
"max_depth": 3,
"max_nodes_per_traversal": 64,
"interleave_width": 4,
"interleave_max_batch": 8,
},
"optimization": {
"advantage_updates_per_iteration": 1,
"strategy_updates_per_iteration": 1,
"advantage_batch_size": 2,
"strategy_batch_size": 2,
},
"checkpoint": {"save_every": 0, "save_latest": False},
"evaluation": {"eval_every": 0},
}
),
LostCitiesConfig(seed=25),
run_dir=tmp_path / "discard_only",
)
metrics = trainer.train()
assert len(metrics) == 1
assert metrics[0].advantage_samples > 0
assert metrics[0].traversal_nodes > 0
def test_deep_cfr_interleaved_scheduler_matches_recursive_single_traversal() -> None:
config = _deep_cfr_config(
{