From 4ec784217bc3c86424a0d65da812041503c66877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=8B=9C=EC=9B=90?= Date: Thu, 7 May 2026 04:20:11 +0900 Subject: [PATCH] =?UTF-8?q?Deep=20CFR=20legacy=20=ED=95=99=EC=8A=B5=20dyna?= =?UTF-8?q?mics=20=EC=A0=95=EB=A0=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../games/classic/deep_cfr/trainer.py | 10 ++++---- .../games/classic/deep_cfr/traversal.pyx | 24 +++++++++++++++---- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py index a07710c..3fe81cd 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py @@ -181,14 +181,14 @@ class DeepCFRTrainer: self.input_dim, self.action_size, self.config.network ).to(self.device) self.advantage_optimizers = [ - torch.optim.Adam( + torch.optim.AdamW( network.parameters(), lr=self.config.optimization.learning_rate, weight_decay=self.config.optimization.weight_decay, ) for network in self.advantage_networks ] - self.strategy_optimizer = torch.optim.Adam( + self.strategy_optimizer = torch.optim.AdamW( self.strategy_network.parameters(), lr=self.config.optimization.learning_rate, weight_decay=self.config.optimization.weight_decay, @@ -717,7 +717,7 @@ class DeepCFRTrainer: network: nn.Module, optimizer: torch.optim.Optimizer, ) -> float: - last_loss = 0.0 + losses: list[float] = [] network.train() for _step in range(self.config.optimization.resolved_advantage_train_steps()): sample_started = time.perf_counter() @@ -741,8 +741,8 @@ class DeepCFRTrainer: network.parameters(), self.config.optimization.grad_clip ) optimizer.step() - last_loss = float(loss.detach().cpu()) - return last_loss + losses.append(float(loss.detach().cpu())) + return float(np.mean(losses)) if losses else 0.0 def _train_strategy( self, diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/traversal.pyx b/src/coolrl_lost_cities/games/classic/deep_cfr/traversal.pyx index 2937103..8d7b95c 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/traversal.pyx +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/traversal.pyx @@ -94,6 +94,8 @@ cdef class CythonDeepCFRTraverser: cdef float self_play_older_weight cdef float self_play_anchor_weight cdef int self_play_recent_window + cdef int active_self_play_bucket + cdef object active_self_play_networks cdef int endpoint_depth_bucket_width cdef int endpoint_depth_bucket_max cdef bint derived_playability @@ -187,6 +189,8 @@ cdef class CythonDeepCFRTraverser: self.self_play_older_weight = max(0.0, self_play_older_weight) self.self_play_anchor_weight = max(0.0, self_play_anchor_weight) self.self_play_recent_window = max(0, self_play_recent_window) + self.active_self_play_bucket = 0 + self.active_self_play_networks = None self.safe_heuristic_opponent_bot = ( SafeHeuristicBot() if self.opponent_policy_id == 1 or self.self_play_anchor_probability > 0.0 @@ -203,8 +207,20 @@ cdef class CythonDeepCFRTraverser: self.input_dim = _input_dim_with_flags_c( state, self.derived_playability, self.slot_aware_playability ) - value = self._traverse(state, traverser, iteration, 0, stats) - return value, stats + if self.opponent_policy_id == 2: + self.active_self_play_bucket = self._self_play_bucket() + if self.active_self_play_bucket in (1, 2): + self.active_self_play_networks = self._self_play_snapshot_networks( + self.active_self_play_bucket + ) + else: + self.active_self_play_networks = None + try: + value = self._traverse(state, traverser, iteration, 0, stats) + return value, stats + finally: + self.active_self_play_bucket = 0 + self.active_self_play_networks = None cdef float _traverse( self, @@ -401,14 +417,14 @@ cdef class CythonDeepCFRTraverser: if self.safe_heuristic_opponent_bot is None: self.safe_heuristic_opponent_bot = SafeHeuristicBot() return int(self.safe_heuristic_opponent_bot.act(state)) - bucket = self._self_play_bucket() + bucket = self.active_self_play_bucket if bucket == 0: return -1 if bucket == 3: if self.safe_heuristic_opponent_bot is None: self.safe_heuristic_opponent_bot = SafeHeuristicBot() return int(self.safe_heuristic_opponent_bot.act(state)) - networks = self._self_play_snapshot_networks(bucket) + networks = self.active_self_play_networks if networks is None: return -1 self._policy_from_networks(networks, state, player, legal, policy)