diff --git a/docs/deep-cfr-profile-advantage-memory-split-2026-05-07.md b/docs/deep-cfr-profile-advantage-memory-split-2026-05-07.md new file mode 100644 index 0000000..6fe7d23 --- /dev/null +++ b/docs/deep-cfr-profile-advantage-memory-split-2026-05-07.md @@ -0,0 +1,61 @@ +# Deep CFR Advantage Memory Split Profile 2026-05-07 + +Run directory: + +`/mnt/2tbhdd/coolrl-lost-cities-runs/2026-05-07_025639_deep_cfr_profile_adv_memory_split_10iter` + +Command: + +```bash +uv run python -m coolrl_lost_cities.games.classic.deep_cfr.cli train \ + --config configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml \ + --checkpoint-dir /mnt/2tbhdd/coolrl-lost-cities-runs/2026-05-07_025639_deep_cfr_profile_adv_memory_split_10iter \ + --max-iterations 10 \ + --save-latest-only +``` + +## Summary + +The run completed 10 iterations. Loss values stayed finite. + +Non-evaluation iterations were iterations 1-4 and 6-9: + +| Metric | Before | After | +| --- | ---: | ---: | +| `iteration_seconds` | 9.132692 | 5.832958 | +| `traversal_seconds` | 3.143286 | 3.160975 | +| `memory_add_seconds` | 0.171213 | 0.157649 | +| `advantage_train_seconds` | 5.061859 | 1.742895 | +| `strategy_train_seconds` | 0.911080 | 0.912324 | +| `evaluation_seconds` | 0.000004 | 0.000004 | +| `checkpoint_seconds` | 0.015469 | 0.015756 | +| `batch_tensor_seconds` | 1.613572 | 1.553183 | +| `advantage_player_0_sample_seconds` | 1.652320 | 0.054032 | +| `advantage_player_1_sample_seconds` | 1.625927 | 0.053600 | +| `strategy_sample_seconds` | 0.061489 | 0.063131 | + +Evaluation iterations were iterations 5 and 10: + +| Metric | Before | After | +| --- | ---: | ---: | +| `iteration_seconds` | 22.676999 | 16.568693 | +| `traversal_seconds` | 3.233497 | 3.051501 | +| `memory_add_seconds` | 0.145742 | 0.146042 | +| `advantage_train_seconds` | 7.502700 | 1.778785 | +| `strategy_train_seconds` | 0.915639 | 0.930619 | +| `evaluation_seconds` | 11.004282 | 10.786219 | +| `checkpoint_seconds` | 0.019737 | 0.020205 | +| `batch_tensor_seconds` | 1.704181 | 1.633090 | +| `advantage_player_0_sample_seconds` | 2.839880 | 0.060441 | +| `advantage_player_1_sample_seconds` | 2.778888 | 0.061004 | +| `strategy_sample_seconds` | 0.067247 | 0.068568 | + +## Per-Iteration Notes + +At iteration 10, `advantage_player_0_sample_seconds + +advantage_player_1_sample_seconds` changed from about 8.231s to about 0.124s. + +At iteration 10, `advantage_train_seconds` changed from about 10.230s to about +1.782s. + +Traversal time stayed close to the previous profile. diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py index 02cfa7d..8a46c6f 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py @@ -150,7 +150,9 @@ class DeepCFRTrainer: lr=self.config.optimization.learning_rate, weight_decay=self.config.optimization.weight_decay, ) - self.advantage_memory = ReservoirMemory(self.config.memory.advantage_capacity) + self.advantage_memories = [ + ReservoirMemory(self.config.memory.advantage_capacity) for _ in range(2) + ] self.strategy_memory = ReservoirMemory(self.config.memory.strategy_capacity) self.rng = np.random.default_rng(self.config.run.seed + 101) self.iteration = 0 @@ -217,6 +219,13 @@ class DeepCFRTrainer: self.strategy_optimizer.load_state_dict(payload["strategy_optimizer"]) self.self_play_league_snapshots = payload.get("self_play_league_snapshots", []) + def _advantage_memory_size(self) -> int: + return sum(len(memory) for memory in self.advantage_memories) + + def _add_advantage_samples(self, samples: list[TrainingSample]) -> None: + for sample in samples: + self.advantage_memories[sample.player].add(sample, self.rng) + def run_iteration(self, iteration: int) -> IterationMetrics: self.iteration = iteration self._runtime_metrics = {} @@ -238,11 +247,13 @@ class DeepCFRTrainer: eval_started = time.perf_counter() eval_metrics = self._evaluate(iteration) self._runtime_metrics["evaluation_seconds"] = time.perf_counter() - eval_started - self._runtime_metrics["advantage_memory_size"] = len(self.advantage_memory) + self._runtime_metrics["advantage_memory_size"] = self._advantage_memory_size() + for player, memory in enumerate(self.advantage_memories): + self._runtime_metrics[f"advantage_player_{player}_memory_size"] = len(memory) self._runtime_metrics["strategy_memory_size"] = len(self.strategy_memory) return IterationMetrics( iteration=iteration, - advantage_samples=len(self.advantage_memory), + advantage_samples=self._advantage_memory_size(), strategy_samples=len(self.strategy_memory), advantage_loss=advantage_loss, strategy_loss=strategy_loss, @@ -313,7 +324,7 @@ class DeepCFRTrainer: ) total_stats.accumulate(stats) memory_add_started = time.perf_counter() - self.advantage_memory.add_many(advantage_samples, self.rng) + self._add_advantage_samples(advantage_samples) self.strategy_memory.add_many(strategy_samples, self.rng) self._runtime_metrics["memory_add_seconds"] = ( float(self._runtime_metrics.get("memory_add_seconds", 0.0)) @@ -363,7 +374,7 @@ class DeepCFRTrainer: result = future.result() total_stats.accumulate(result.stats) memory_add_started = time.perf_counter() - self.advantage_memory.add_many(result.advantage_samples, self.rng) + self._add_advantage_samples(result.advantage_samples) self.strategy_memory.add_many(result.strategy_samples, self.rng) self._runtime_metrics["memory_add_seconds"] = ( float(self._runtime_metrics.get("memory_add_seconds", 0.0)) @@ -546,24 +557,20 @@ class DeepCFRTrainer: def _train_advantage_networks(self) -> float: losses: list[float] = [] - for player, network in enumerate(self.advantage_networks): - filter_started = time.perf_counter() - samples = [sample for sample in self.advantage_memory.all() if sample.player == player] - self._runtime_metrics[f"advantage_player_{player}_filter_seconds"] = ( - time.perf_counter() - filter_started - ) - self._runtime_metrics[f"advantage_player_{player}_sample_count"] = len(samples) - if not samples: + for player, (network, memory) in enumerate( + zip(self.advantage_networks, self.advantage_memories, strict=True) + ): + self._runtime_metrics[f"advantage_player_{player}_sample_count"] = len(memory) + if len(memory) == 0: continue losses.append(self._train_advantage(player, network, self.advantage_optimizers[player])) return float(np.mean(losses)) if losses else 0.0 def _train_strategy_network(self) -> float: - samples = self.strategy_memory.all() - self._runtime_metrics["strategy_sample_count"] = len(samples) - if not samples: + self._runtime_metrics["strategy_sample_count"] = len(self.strategy_memory) + if len(self.strategy_memory) == 0: return 0.0 - return self._train_strategy(self.strategy_network, self.strategy_optimizer, samples) + return self._train_strategy(self.strategy_network, self.strategy_optimizer) def _batch_tensors( self, @@ -602,10 +609,9 @@ class DeepCFRTrainer: network.train() for _step in range(self.config.optimization.resolved_advantage_train_steps()): sample_started = time.perf_counter() - batch = self.advantage_memory.sample( + batch = self.advantage_memories[player].sample( self.config.optimization.resolved_advantage_batch_size(), self.rng, - player=player, ) self._runtime_metrics[f"advantage_player_{player}_sample_seconds"] = ( float(self._runtime_metrics.get(f"advantage_player_{player}_sample_seconds", 0.0)) @@ -630,7 +636,6 @@ class DeepCFRTrainer: self, network: nn.Module, optimizer: torch.optim.Optimizer, - samples: list[TrainingSample], ) -> float: last_loss = 0.0 network.train()