coolguy/coorl-lost-cities
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Deep CFR advantage memory를 player별로 분리

Browse Source
This commit is contained in:
coolguy
2026-05-07 02:59:03 +09:00
parent e411ea4a15
commit 9acf9ed261
2 changed files with 86 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/deep-cfr-profile-advantage-memory-split-2026-05-07.md
+61
View File
@@ -0,0 +1,61 @@
# Deep CFR Advantage Memory Split Profile 2026-05-07
Run directory:
`/mnt/2tbhdd/coolrl-lost-cities-runs/2026-05-07_025639_deep_cfr_profile_adv_memory_split_10iter`
Command:
```bash
uv run python -m coolrl_lost_cities.games.classic.deep_cfr.cli train \
--config configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml \
--checkpoint-dir /mnt/2tbhdd/coolrl-lost-cities-runs/2026-05-07_025639_deep_cfr_profile_adv_memory_split_10iter \
--max-iterations 10 \
--save-latest-only
```
## Summary
The run completed 10 iterations. Loss values stayed finite.
Non-evaluation iterations were iterations 1-4 and 6-9:
| Metric | Before | After |
| --- | ---: | ---: |
| `iteration_seconds` | 9.132692 | 5.832958 |
| `traversal_seconds` | 3.143286 | 3.160975 |
| `memory_add_seconds` | 0.171213 | 0.157649 |
| `advantage_train_seconds` | 5.061859 | 1.742895 |
| `strategy_train_seconds` | 0.911080 | 0.912324 |
| `evaluation_seconds` | 0.000004 | 0.000004 |
| `checkpoint_seconds` | 0.015469 | 0.015756 |
| `batch_tensor_seconds` | 1.613572 | 1.553183 |
| `advantage_player_0_sample_seconds` | 1.652320 | 0.054032 |
| `advantage_player_1_sample_seconds` | 1.625927 | 0.053600 |
| `strategy_sample_seconds` | 0.061489 | 0.063131 |
Evaluation iterations were iterations 5 and 10:
| Metric | Before | After |
| --- | ---: | ---: |
| `iteration_seconds` | 22.676999 | 16.568693 |
| `traversal_seconds` | 3.233497 | 3.051501 |
| `memory_add_seconds` | 0.145742 | 0.146042 |
| `advantage_train_seconds` | 7.502700 | 1.778785 |
| `strategy_train_seconds` | 0.915639 | 0.930619 |
| `evaluation_seconds` | 11.004282 | 10.786219 |
| `checkpoint_seconds` | 0.019737 | 0.020205 |
| `batch_tensor_seconds` | 1.704181 | 1.633090 |
| `advantage_player_0_sample_seconds` | 2.839880 | 0.060441 |
| `advantage_player_1_sample_seconds` | 2.778888 | 0.061004 |
| `strategy_sample_seconds` | 0.067247 | 0.068568 |
## Per-Iteration Notes
At iteration 10, `advantage_player_0_sample_seconds +
advantage_player_1_sample_seconds` changed from about 8.231s to about 0.124s.
At iteration 10, `advantage_train_seconds` changed from about 10.230s to about
1.782s.
Traversal time stayed close to the previous profile.
src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py
+25 -20
View File
@@ -150,7 +150,9 @@ class DeepCFRTrainer:
lr=self.config.optimization.learning_rate,
weight_decay=self.config.optimization.weight_decay,
)
self.advantage_memory = ReservoirMemory(self.config.memory.advantage_capacity)
self.advantage_memories = [
ReservoirMemory(self.config.memory.advantage_capacity) for _ in range(2)
]
self.strategy_memory = ReservoirMemory(self.config.memory.strategy_capacity)
self.rng = np.random.default_rng(self.config.run.seed + 101)
self.iteration = 0
@@ -217,6 +219,13 @@ class DeepCFRTrainer:
self.strategy_optimizer.load_state_dict(payload["strategy_optimizer"])
self.self_play_league_snapshots = payload.get("self_play_league_snapshots", [])
def _advantage_memory_size(self) -> int:
return sum(len(memory) for memory in self.advantage_memories)
def _add_advantage_samples(self, samples: list[TrainingSample]) -> None:
for sample in samples:
self.advantage_memories[sample.player].add(sample, self.rng)
def run_iteration(self, iteration: int) -> IterationMetrics:
self.iteration = iteration
self._runtime_metrics = {}
@@ -238,11 +247,13 @@ class DeepCFRTrainer:
eval_started = time.perf_counter()
eval_metrics = self._evaluate(iteration)
self._runtime_metrics["evaluation_seconds"] = time.perf_counter() - eval_started
self._runtime_metrics["advantage_memory_size"] = len(self.advantage_memory)
self._runtime_metrics["advantage_memory_size"] = self._advantage_memory_size()
for player, memory in enumerate(self.advantage_memories):
self._runtime_metrics[f"advantage_player_{player}_memory_size"] = len(memory)
self._runtime_metrics["strategy_memory_size"] = len(self.strategy_memory)
return IterationMetrics(
iteration=iteration,
advantage_samples=len(self.advantage_memory),
advantage_samples=self._advantage_memory_size(),
strategy_samples=len(self.strategy_memory),
advantage_loss=advantage_loss,
strategy_loss=strategy_loss,
@@ -313,7 +324,7 @@ class DeepCFRTrainer:
)
total_stats.accumulate(stats)
memory_add_started = time.perf_counter()
self.advantage_memory.add_many(advantage_samples, self.rng)
self._add_advantage_samples(advantage_samples)
self.strategy_memory.add_many(strategy_samples, self.rng)
self._runtime_metrics["memory_add_seconds"] = (
float(self._runtime_metrics.get("memory_add_seconds", 0.0))
@@ -363,7 +374,7 @@ class DeepCFRTrainer:
result = future.result()
total_stats.accumulate(result.stats)
memory_add_started = time.perf_counter()
self.advantage_memory.add_many(result.advantage_samples, self.rng)
self._add_advantage_samples(result.advantage_samples)
self.strategy_memory.add_many(result.strategy_samples, self.rng)
self._runtime_metrics["memory_add_seconds"] = (
float(self._runtime_metrics.get("memory_add_seconds", 0.0))
@@ -546,24 +557,20 @@ class DeepCFRTrainer:
def _train_advantage_networks(self) -> float:
losses: list[float] = []
for player, network in enumerate(self.advantage_networks):
filter_started = time.perf_counter()
samples = [sample for sample in self.advantage_memory.all() if sample.player == player]
self._runtime_metrics[f"advantage_player_{player}_filter_seconds"] = (
time.perf_counter() - filter_started
)
self._runtime_metrics[f"advantage_player_{player}_sample_count"] = len(samples)
if not samples:
for player, (network, memory) in enumerate(
zip(self.advantage_networks, self.advantage_memories, strict=True)
):
self._runtime_metrics[f"advantage_player_{player}_sample_count"] = len(memory)
if len(memory) == 0:
continue
losses.append(self._train_advantage(player, network, self.advantage_optimizers[player]))
return float(np.mean(losses)) if losses else 0.0
def _train_strategy_network(self) -> float:
samples = self.strategy_memory.all()
self._runtime_metrics["strategy_sample_count"] = len(samples)
if not samples:
self._runtime_metrics["strategy_sample_count"] = len(self.strategy_memory)
if len(self.strategy_memory) == 0:
return 0.0
return self._train_strategy(self.strategy_network, self.strategy_optimizer, samples)
return self._train_strategy(self.strategy_network, self.strategy_optimizer)
def _batch_tensors(
self,
@@ -602,10 +609,9 @@ class DeepCFRTrainer:
network.train()
for _step in range(self.config.optimization.resolved_advantage_train_steps()):
sample_started = time.perf_counter()
batch = self.advantage_memory.sample(
batch = self.advantage_memories[player].sample(
self.config.optimization.resolved_advantage_batch_size(),
self.rng,
player=player,
)
self._runtime_metrics[f"advantage_player_{player}_sample_seconds"] = (
float(self._runtime_metrics.get(f"advantage_player_{player}_sample_seconds", 0.0))
@@ -630,7 +636,6 @@ class DeepCFRTrainer:
self,
network: nn.Module,
optimizer: torch.optim.Optimizer,
samples: list[TrainingSample],
) -> float:
last_loss = 0.0
network.train()