From 9bcc88c1be6d77b8ddafb28f8f7a64af5e3438ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=8B=9C=EC=9B=90?= Date: Thu, 7 May 2026 00:46:09 +0900 Subject: [PATCH] =?UTF-8?q?Deep=20CFR=20legacy=20=EC=9E=AC=ED=98=84=20conf?= =?UTF-8?q?ig=20=EA=B8=B0=EB=B0=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...poc_full_depth_slot_aware_playability.yaml | 89 +++++++++++++ .../games/classic/deep_cfr/benchmark.py | 3 +- .../games/classic/deep_cfr/cli.py | 2 +- .../games/classic/deep_cfr/config.py | 119 ++++++++++++++++++ .../games/classic/deep_cfr/evaluate.py | 4 +- .../games/classic/deep_cfr/networks.py | 44 +++++-- .../games/classic/deep_cfr/trainer.py | 107 +++++++++++----- .../games/classic/deep_cfr/workers.py | 6 +- tests/games/classic/test_deep_cfr_trainer.py | 27 ++++ 9 files changed, 356 insertions(+), 45 deletions(-) create mode 100644 configs/deep_cfr/pure_self_play_zero_pit_poc_full_depth_slot_aware_playability.yaml diff --git a/configs/deep_cfr/pure_self_play_zero_pit_poc_full_depth_slot_aware_playability.yaml b/configs/deep_cfr/pure_self_play_zero_pit_poc_full_depth_slot_aware_playability.yaml new file mode 100644 index 0000000..14b78a2 --- /dev/null +++ b/configs/deep_cfr/pure_self_play_zero_pit_poc_full_depth_slot_aware_playability.yaml @@ -0,0 +1,89 @@ +run: + experiment_name: lost_cities_deep_cfr_pure_self_play_zero_pit_poc_full_depth_slot_aware_playability + seed: 79 + max_iterations: null + max_hours: 4 + device: cuda + use_amp: false + +rules: + n_colors: 5 + n_ranks: 9 + min_rank: 2 + n_handshakes: 3 + hand_size: 8 + expedition_penalty: -20 + bonus_threshold: 8 + bonus_amount: 20 + +encoding: + derived_playability: true + slot_aware_playability: true + +network: + hidden_size: 256 + num_layers: 3 + activation: relu + +traversal: + traversals_per_player: 70 + strategy_sample_interval: 1 + store_strategy_on_opponent_nodes: false + store_strategy_on_traverser_nodes: true + max_depth: null + max_nodes_per_traversal: 1000 + opponent_policy: self_play_league + cutoff_value_mode: score_diff + cutoff_rollouts: 0 + cutoff_rollout_policy: random + cutoff_rollout_max_steps: 300 + num_workers: 8 + traversal_worker_chunk_size: 8 + regret_matching_epsilon: 0.0001 + outcome_sampling_epsilon: 0.2 + outcome_sampling_value_clip: 500 + outcome_unsampled_regret: zero + endpoint_depth_bucket_width: 100 + endpoint_depth_bucket_max: 1000 + +self_play: + current_weight: 0.5 + recent_weight: 0.3 + older_weight: 0.2 + anchor_weight: 0.0 + recent_window: 5 + max_snapshots: 20 + snapshot_every: 1 + +optimization: + advantage_batch_size: 1024 + strategy_batch_size: 1024 + advantage_updates_per_iteration: 256 + strategy_updates_per_iteration: 256 + learning_rate: 0.00003 + weight_decay: 0.0001 + grad_clip: 1.0 + +memory: + advantage_capacity: 2000000 + strategy_capacity: 2000000 + +evaluation: + eval_every: 5 + games: 100 + max_steps: 1000 + on_max_steps: score_diff + opponents: + - random + - passive_discard + - safe_heuristic + - safe_heuristic_loose + - safe_heuristic_strict + - noisy_safe + +checkpoint: + directory: checkpoints/lost_cities_deep_cfr_pure_self_play_zero_pit_poc_full_depth_slot_aware_playability + save_every_iteration: false + save_iteration_interval: 10 + save_latest_only: false + progress_interval_seconds: 20.0 diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/benchmark.py b/src/coolrl_lost_cities/games/classic/deep_cfr/benchmark.py index d989391..f26101f 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/benchmark.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/benchmark.py @@ -4,7 +4,6 @@ import time from coolrl_lost_cities.games.classic.deep_cfr.config import DeepCFRConfig from coolrl_lost_cities.games.classic.deep_cfr.trainer import DeepCFRTrainer -from coolrl_lost_cities.games.classic.game import classic_config def benchmark_traversal( @@ -24,7 +23,7 @@ def benchmark_traversal( data["checkpoint"]["save_every_iteration"] = False data["evaluation"]["eval_every"] = 0 cfg = DeepCFRConfig.model_validate(data) - trainer = DeepCFRTrainer(cfg, classic_config(seed=cfg.run.seed)) + trainer = DeepCFRTrainer(cfg, cfg.rules.to_lost_cities_config(seed=cfg.run.seed)) started = time.perf_counter() metrics = trainer.run_iteration(1) elapsed = time.perf_counter() - started diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py b/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py index 39484cf..008e0b4 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py @@ -65,7 +65,7 @@ def train_command(args: argparse.Namespace) -> None: config = _with_overrides(config, overrides) trainer = DeepCFRTrainer( config, - classic_config(seed=config.run.seed), + config.rules.to_lost_cities_config(seed=config.run.seed), device=args.device or config.run.device, ) if args.resume: diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/config.py b/src/coolrl_lost_cities/games/classic/deep_cfr/config.py index 6767827..29687e3 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/config.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/config.py @@ -9,25 +9,86 @@ from typing import Any import yaml from pydantic import BaseModel, ConfigDict, Field, field_validator +from coolrl_lost_cities.games.classic.game import LostCitiesConfig + class StrictModel(BaseModel): model_config = ConfigDict(extra="forbid") class RunConfig(StrictModel): + experiment_name: str = "deep_cfr" iterations: int = 1 + max_iterations: int | None = None + max_hours: float | None = None seed: int = 1 device: str = "cpu" + use_amp: bool = False + + @field_validator("device") + @classmethod + def _normalize_device(cls, value: str) -> str: + token = value.strip().lower() + if token == "cuda": + return "cuda" + if token == "cpu": + return "cpu" + if token == "auto": + return "auto" + return token + + +class RulesConfig(StrictModel): + n_colors: int = 5 + n_ranks: int = 9 + min_rank: int = 2 + n_handshakes: int = 3 + hand_size: int = 8 + expedition_penalty: int = -20 + bonus_threshold: int = 8 + bonus_amount: int = 20 + + def to_lost_cities_config(self, seed: int | None = None) -> LostCitiesConfig: + config = LostCitiesConfig( + n_colors=self.n_colors, + n_ranks=self.n_ranks, + min_rank=self.min_rank, + n_handshakes=self.n_handshakes, + hand_size=self.hand_size, + expedition_penalty=self.expedition_penalty, + bonus_threshold=self.bonus_threshold, + bonus_amount=self.bonus_amount, + seed=seed, + ) + config.validate() + return config + + +class EncodingConfig(StrictModel): + derived_playability: bool = False + slot_aware_playability: bool = False class NetworkConfig(StrictModel): hidden_size: int = 64 + num_layers: int = 2 + activation: str = "relu" + + @field_validator("activation") + @classmethod + def _validate_activation(cls, value: str) -> str: + token = value.strip().lower() + if token not in {"relu", "gelu"}: + raise ValueError("must be 'relu' or 'gelu'") + return token class TraversalConfig(StrictModel): traversals_per_iteration: int = 2 + traversals_per_player: int | None = None max_depth: int | None = 8 max_nodes: int | None = 10_000 + max_nodes_per_traversal: int | None = None regret_matching_epsilon: float = 1.0e-8 outcome_sampling_epsilon: float = 0.0 outcome_sampling_value_clip: float | None = None @@ -42,6 +103,9 @@ class TraversalConfig(StrictModel): store_strategy_on_opponent_nodes: bool = True num_workers: int | str = 0 worker_chunk_size: int = 4 + traversal_worker_chunk_size: int | None = None + endpoint_depth_bucket_width: int = 100 + endpoint_depth_bucket_max: int = 1000 @field_validator("outcome_unsampled_regret") @classmethod @@ -80,6 +144,21 @@ class TraversalConfig(StrictModel): return max(0, int(token)) return max(0, int(self.num_workers)) + def resolved_traversals_per_player(self) -> int: + if self.traversals_per_player is not None: + return max(0, int(self.traversals_per_player)) + return max(0, int(self.traversals_per_iteration)) + + def resolved_max_nodes(self) -> int | None: + if self.max_nodes_per_traversal is not None: + return self.max_nodes_per_traversal + return self.max_nodes + + def resolved_worker_chunk_size(self) -> int: + if self.traversal_worker_chunk_size is not None: + return max(1, int(self.traversal_worker_chunk_size)) + return max(1, int(self.worker_chunk_size)) + class SelfPlayLeagueConfig(StrictModel): snapshot_every: int = 1 @@ -96,7 +175,33 @@ class OptimizationConfig(StrictModel): advantage_train_steps: int = 1 strategy_train_steps: int = 1 batch_size: int = 32 + advantage_batch_size: int | None = None + strategy_batch_size: int | None = None + advantage_updates_per_iteration: int | None = None + strategy_updates_per_iteration: int | None = None learning_rate: float = 1.0e-3 + weight_decay: float = 0.0 + grad_clip: float = 0.0 + + def resolved_advantage_batch_size(self) -> int: + if self.advantage_batch_size is not None: + return max(1, int(self.advantage_batch_size)) + return max(1, int(self.batch_size)) + + def resolved_strategy_batch_size(self) -> int: + if self.strategy_batch_size is not None: + return max(1, int(self.strategy_batch_size)) + return max(1, int(self.batch_size)) + + def resolved_advantage_train_steps(self) -> int: + if self.advantage_updates_per_iteration is not None: + return max(0, int(self.advantage_updates_per_iteration)) + return max(0, int(self.advantage_train_steps)) + + def resolved_strategy_train_steps(self) -> int: + if self.strategy_updates_per_iteration is not None: + return max(0, int(self.strategy_updates_per_iteration)) + return max(0, int(self.strategy_train_steps)) class MemoryConfig(StrictModel): @@ -107,6 +212,9 @@ class MemoryConfig(StrictModel): class CheckpointConfig(StrictModel): directory: str = "runs/deep_cfr/default" save_every_iteration: bool = True + save_iteration_interval: int = 0 + save_latest_only: bool = False + progress_interval_seconds: float = 20.0 @property def path(self) -> Path: @@ -118,10 +226,21 @@ class EvaluationConfig(StrictModel): games: int = 10 opponents: tuple[str, ...] = ("random",) max_steps: int = 10_000 + on_max_steps: str = "score_diff" + + @field_validator("on_max_steps") + @classmethod + def _validate_on_max_steps(cls, value: str) -> str: + token = value.strip().lower() + if token not in {"score_diff", "loss", "draw"}: + raise ValueError("must be 'score_diff', 'loss', or 'draw'") + return token class DeepCFRConfig(StrictModel): run: RunConfig = Field(default_factory=RunConfig) + rules: RulesConfig = Field(default_factory=RulesConfig) + encoding: EncodingConfig = Field(default_factory=EncodingConfig) network: NetworkConfig = Field(default_factory=NetworkConfig) traversal: TraversalConfig = Field(default_factory=TraversalConfig) self_play: SelfPlayLeagueConfig = Field(default_factory=SelfPlayLeagueConfig) diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/evaluate.py b/src/coolrl_lost_cities/games/classic/deep_cfr/evaluate.py index 27b1e28..30b69c0 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/evaluate.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/evaluate.py @@ -94,10 +94,10 @@ def load_strategy_policy_from_checkpoint( payload = torch.load(checkpoint_path, map_location="cpu") cfg = config_from_dict(payload["config"]) game_config = LostCitiesConfig(**payload["game_config"]) - network = DeepCFRMLP( + network = DeepCFRMLP.from_config( int(payload["input_dim"]), int(payload["action_size"]), - cfg.network.hidden_size, + cfg.network, ).to(device) network.load_state_dict(payload["strategy_network"]) network.eval() diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/networks.py b/src/coolrl_lost_cities/games/classic/deep_cfr/networks.py index 2251a3f..d7d8e47 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/networks.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/networks.py @@ -3,16 +3,46 @@ from __future__ import annotations import torch from torch import nn +from coolrl_lost_cities.games.classic.deep_cfr.config import NetworkConfig + + +def _activation(name: str) -> nn.Module: + token = name.lower() + if token == "relu": + return nn.ReLU() + if token == "gelu": + return nn.GELU() + raise ValueError(f"unsupported activation: {name!r}") + class DeepCFRMLP(nn.Module): - def __init__(self, input_dim: int, output_dim: int, hidden_size: int = 64) -> None: + def __init__( + self, + input_dim: int, + output_dim: int, + hidden_size: int = 64, + *, + num_layers: int = 2, + activation: str = "relu", + ) -> None: super().__init__() - self.net = nn.Sequential( - nn.Linear(input_dim, hidden_size), - nn.ReLU(), - nn.Linear(hidden_size, hidden_size), - nn.ReLU(), - nn.Linear(hidden_size, output_dim), + layers: list[nn.Module] = [] + last_dim = input_dim + for _ in range(max(0, int(num_layers))): + layers.append(nn.Linear(last_dim, hidden_size)) + layers.append(_activation(activation)) + last_dim = hidden_size + layers.append(nn.Linear(last_dim, output_dim)) + self.net = nn.Sequential(*layers) + + @classmethod + def from_config(cls, input_dim: int, output_dim: int, config: NetworkConfig) -> DeepCFRMLP: + return cls( + input_dim, + output_dim, + config.hidden_size, + num_layers=config.num_layers, + activation=config.activation, ) def forward(self, x: torch.Tensor) -> torch.Tensor: diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py index b17d98d..8eb9dce 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py @@ -28,6 +28,13 @@ from coolrl_lost_cities.games.classic.deep_cfr.workers import ( from coolrl_lost_cities.games.classic.game import GameState, LostCitiesConfig +def _resolve_torch_device(device: str) -> torch.device: + token = device.strip().lower() + if token == "auto": + token = "cuda" if torch.cuda.is_available() else "cpu" + return torch.device(token) + + @dataclass(frozen=True) class IterationMetrics: iteration: int @@ -68,8 +75,10 @@ class DeepCFRTrainer: device: str = "cpu", ) -> None: self.config = config or DeepCFRConfig() - self.game_config = game_config or LostCitiesConfig(seed=self.config.run.seed) - self.device = torch.device(device) + self.game_config = game_config or self.config.rules.to_lost_cities_config( + seed=self.config.run.seed + ) + self.device = _resolve_torch_device(device) probe = GameState.new_game(self.game_config, seed=self.config.run.seed) self.input_dim = input_dim(probe) @@ -77,20 +86,26 @@ class DeepCFRTrainer: torch.manual_seed(self.config.run.seed) self.advantage_networks = [ - DeepCFRMLP(self.input_dim, self.action_size, self.config.network.hidden_size).to( + DeepCFRMLP.from_config(self.input_dim, self.action_size, self.config.network).to( self.device ) for _ in range(2) ] - self.strategy_network = DeepCFRMLP( - self.input_dim, self.action_size, self.config.network.hidden_size + self.strategy_network = DeepCFRMLP.from_config( + self.input_dim, self.action_size, self.config.network ).to(self.device) self.advantage_optimizers = [ - torch.optim.Adam(network.parameters(), lr=self.config.optimization.learning_rate) + torch.optim.Adam( + network.parameters(), + lr=self.config.optimization.learning_rate, + weight_decay=self.config.optimization.weight_decay, + ) for network in self.advantage_networks ] self.strategy_optimizer = torch.optim.Adam( - self.strategy_network.parameters(), lr=self.config.optimization.learning_rate + self.strategy_network.parameters(), + lr=self.config.optimization.learning_rate, + weight_decay=self.config.optimization.weight_decay, ) self.advantage_memory = ReservoirMemory(self.config.memory.advantage_capacity) self.strategy_memory = ReservoirMemory(self.config.memory.strategy_capacity) @@ -175,7 +190,7 @@ class DeepCFRTrainer: store_strategy_on_traverser_nodes=self.config.traversal.store_strategy_on_traverser_nodes, store_strategy_on_opponent_nodes=self.config.traversal.store_strategy_on_opponent_nodes, max_depth=self.config.traversal.max_depth, - max_nodes=self.config.traversal.max_nodes, + max_nodes=self.config.traversal.resolved_max_nodes(), outcome_sampling_epsilon=self.config.traversal.outcome_sampling_epsilon, outcome_sampling_value_clip=self.config.traversal.outcome_sampling_value_clip, outcome_unsampled_regret=self.config.traversal.outcome_unsampled_regret, @@ -195,7 +210,7 @@ class DeepCFRTrainer: ) for network in self.advantage_networks: network.eval() - for traversal_index in range(self.config.traversal.traversals_per_iteration): + for traversal_index in range(self.config.traversal.resolved_traversals_per_player()): for player in range(2): seed = self.config.run.seed + iteration * 10_000 + traversal_index * 10 + player state = GameState.new_game(self.game_config, seed=seed) @@ -227,12 +242,12 @@ class DeepCFRTrainer: {name: value.detach().cpu() for name, value in network.state_dict().items()} for network in self.advantage_networks ] - chunk_size = max(1, self.config.traversal.worker_chunk_size) + chunk_size = self.config.traversal.resolved_worker_chunk_size() batch_index = 0 for player in range(2): seeds = [ self.config.run.seed + iteration * 10_000 + index * 10 + player - for index in range(self.config.traversal.traversals_per_iteration) + for index in range(self.config.traversal.resolved_traversals_per_player()) ] for start in range(0, len(seeds), chunk_size): chunk = seeds[start : start + chunk_size] @@ -275,7 +290,7 @@ class DeepCFRTrainer: league: list[list[nn.Module]] = [] for snapshot in self.self_play_league_snapshots: networks = [ - DeepCFRMLP(self.input_dim, self.action_size, self.config.network.hidden_size).to( + DeepCFRMLP.from_config(self.input_dim, self.action_size, self.config.network).to( self.device ) for _ in range(2) @@ -293,20 +308,48 @@ class DeepCFRTrainer: self._start_run_logging() metrics: list[IterationMetrics] = [] start = self.iteration + 1 - stop = self.iteration + self.config.run.iterations - for iteration in range(start, stop + 1): + stop = self._stop_iteration() + run_started = time.perf_counter() + iteration = start + while iteration <= stop: started = time.perf_counter() item = self.run_iteration(iteration) elapsed = time.perf_counter() - started metrics.append(item) self._append_metrics(item, elapsed) self._maybe_record_self_play_snapshot(iteration) - if self.config.checkpoint.save_every_iteration: - checkpoint_dir = self.run_dir - self.save_checkpoint(checkpoint_dir / f"iteration_{iteration:05d}.pt", item) - self.save_checkpoint(checkpoint_dir / "latest.pt", item) + if self._should_save_iteration(iteration): + self._save_iteration_checkpoints(iteration, item) + if self._time_limit_reached(run_started): + break + iteration += 1 return metrics + def _stop_iteration(self) -> int: + if self.config.run.max_iterations is not None: + return max(self.iteration, int(self.config.run.max_iterations)) + if self.config.run.max_hours is not None: + return 2**31 - 1 + return self.iteration + self.config.run.iterations + + def _time_limit_reached(self, run_started: float) -> bool: + if self.config.run.max_hours is None: + return False + elapsed_hours = (time.perf_counter() - run_started) / 3600.0 + return elapsed_hours >= self.config.run.max_hours + + def _should_save_iteration(self, iteration: int) -> bool: + if self.config.checkpoint.save_every_iteration: + return True + interval = int(self.config.checkpoint.save_iteration_interval) + return interval > 0 and iteration % interval == 0 + + def _save_iteration_checkpoints(self, iteration: int, item: IterationMetrics) -> None: + checkpoint_dir = self.run_dir + if not self.config.checkpoint.save_latest_only: + self.save_checkpoint(checkpoint_dir / f"iteration_{iteration:05d}.pt", item) + self.save_checkpoint(checkpoint_dir / "latest.pt", item) + def _start_run_logging(self) -> None: self.run_dir.mkdir(parents=True, exist_ok=True) config_path = self.run_dir / "config.json" @@ -371,14 +414,6 @@ class DeepCFRTrainer: return 0.0 return self._train_strategy(self.strategy_network, self.strategy_optimizer, samples) - def _batch(self, samples: list[TrainingSample], step: int) -> list[TrainingSample]: - batch_size = min(self.config.optimization.batch_size, len(samples)) - offset = (step * batch_size) % len(samples) - batch = samples[offset : offset + batch_size] - if len(batch) < batch_size: - batch = batch + samples[0 : batch_size - len(batch)] - return batch - def _batch_tensors( self, batch: list[TrainingSample], @@ -408,10 +443,12 @@ class DeepCFRTrainer: ) -> float: last_loss = 0.0 network.train() - for _step in range(max(self.config.optimization.advantage_train_steps, 0)): + for _step in range(self.config.optimization.resolved_advantage_train_steps()): x, y, legal = self._batch_tensors( self.advantage_memory.sample( - self.config.optimization.batch_size, self.rng, player=player + self.config.optimization.resolved_advantage_batch_size(), + self.rng, + player=player, ) ) pred = network(x) @@ -419,6 +456,10 @@ class DeepCFRTrainer: loss = diff.square().sum() / legal.sum().clamp_min(1) optimizer.zero_grad(set_to_none=True) loss.backward() + if self.config.optimization.grad_clip > 0.0: + torch.nn.utils.clip_grad_norm_( + network.parameters(), self.config.optimization.grad_clip + ) optimizer.step() last_loss = float(loss.detach().cpu()) return last_loss @@ -431,15 +472,21 @@ class DeepCFRTrainer: ) -> float: last_loss = 0.0 network.train() - for _step in range(max(self.config.optimization.strategy_train_steps, 0)): + for _step in range(self.config.optimization.resolved_strategy_train_steps()): x, y, legal = self._batch_tensors( - self.strategy_memory.sample(self.config.optimization.batch_size, self.rng) + self.strategy_memory.sample( + self.config.optimization.resolved_strategy_batch_size(), self.rng + ) ) logits = network(x).masked_fill(~legal, torch.finfo(torch.float32).min) log_probs = nn.functional.log_softmax(logits, dim=-1).masked_fill(~legal, 0.0) loss = -(y * log_probs).sum(dim=-1).mean() optimizer.zero_grad(set_to_none=True) loss.backward() + if self.config.optimization.grad_clip > 0.0: + torch.nn.utils.clip_grad_norm_( + network.parameters(), self.config.optimization.grad_clip + ) optimizer.step() last_loss = float(loss.detach().cpu()) return last_loss diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py b/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py index 9138fb5..b8cc414 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py @@ -40,7 +40,7 @@ def run_traversal_worker_batch(batch: TraversalWorkerBatch) -> TraversalWorkerRe cfg = config_from_dict(batch.config) device = torch.device("cpu") networks = [ - DeepCFRMLP(batch.input_dim, batch.action_size, cfg.network.hidden_size).to(device) + DeepCFRMLP.from_config(batch.input_dim, batch.action_size, cfg.network).to(device) for _ in range(2) ] for network, state_dict in zip(networks, batch.advantage_networks, strict=True): @@ -49,7 +49,7 @@ def run_traversal_worker_batch(batch: TraversalWorkerBatch) -> TraversalWorkerRe league_networks: list[list[torch.nn.Module]] = [] for snapshot in batch.league_advantage_networks: snapshot_networks = [ - DeepCFRMLP(batch.input_dim, batch.action_size, cfg.network.hidden_size).to(device) + DeepCFRMLP.from_config(batch.input_dim, batch.action_size, cfg.network).to(device) for _ in range(2) ] for network, state_dict in zip(snapshot_networks, snapshot, strict=True): @@ -69,7 +69,7 @@ def run_traversal_worker_batch(batch: TraversalWorkerBatch) -> TraversalWorkerRe store_strategy_on_traverser_nodes=cfg.traversal.store_strategy_on_traverser_nodes, store_strategy_on_opponent_nodes=cfg.traversal.store_strategy_on_opponent_nodes, max_depth=cfg.traversal.max_depth, - max_nodes=cfg.traversal.max_nodes, + max_nodes=cfg.traversal.resolved_max_nodes(), outcome_sampling_epsilon=cfg.traversal.outcome_sampling_epsilon, outcome_sampling_value_clip=cfg.traversal.outcome_sampling_value_clip, outcome_unsampled_regret=cfg.traversal.outcome_unsampled_regret, diff --git a/tests/games/classic/test_deep_cfr_trainer.py b/tests/games/classic/test_deep_cfr_trainer.py index 90a7540..f7fee83 100644 --- a/tests/games/classic/test_deep_cfr_trainer.py +++ b/tests/games/classic/test_deep_cfr_trainer.py @@ -26,6 +26,33 @@ def test_deep_cfr_loads_smoke_yaml_config() -> None: assert config.checkpoint.directory == "runs/deep_cfr/smoke" +def test_deep_cfr_loads_mapped_legacy_reproduction_config() -> None: + config = load_config( + "configs/deep_cfr/pure_self_play_zero_pit_poc_full_depth_slot_aware_playability.yaml" + ) + + assert config.run.experiment_name.endswith("slot_aware_playability") + assert config.run.seed == 79 + assert config.run.max_iterations is None + assert config.run.max_hours == 4 + assert config.encoding.derived_playability is True + assert config.encoding.slot_aware_playability is True + assert config.network.hidden_size == 256 + assert config.network.num_layers == 3 + assert config.traversal.resolved_traversals_per_player() == 70 + assert config.traversal.max_depth is None + assert config.traversal.resolved_max_nodes() == 1000 + assert config.traversal.resolved_worker_chunk_size() == 8 + assert config.optimization.resolved_advantage_batch_size() == 1024 + assert config.optimization.resolved_strategy_batch_size() == 1024 + assert config.optimization.resolved_advantage_train_steps() == 256 + assert config.optimization.resolved_strategy_train_steps() == 256 + assert config.optimization.weight_decay == 0.0001 + assert config.optimization.grad_clip == 1.0 + assert config.evaluation.on_max_steps == "score_diff" + assert config.checkpoint.save_iteration_interval == 10 + + def test_deep_cfr_trainer_smoke_run() -> None: trainer = DeepCFRTrainer( _deep_cfr_config(