coolguy/coorl-lost-cities
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Clean up Deep CFR config schema

Browse Source 
Remove legacy aliases, rename max_hours to max_minutes, collapse the
four checkpoint save flags into save_every + save_latest, and change
defaults to safer values (opponent_policy=self_play_league,
device=auto, eval_every=50, max_depth=null). Migrate all archived
yaml configs and tests to the new schema.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
coolguy
2026-05-07 16:09:19 +09:00
co-authored by Claude Opus 4.7
parent c2b88d7c9c
commit a177031963
21 changed files with 257 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+12 -16
View File
@@ -66,11 +66,9 @@ Short fixed-iteration run:
```bash ```bash
uv run lost-cities-deep-cfr train \ uv run lost-cities-deep-cfr train \
--config configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml \ --config configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml \
--set run.iterations=100 \ --set run.max_iterations=100 \
--set run.max_hours=null \ --set run.max_minutes=null \
--set run.max_iterations=null \ --set checkpoint.save_every=0
--set checkpoint.save_latest_only=true \
--set checkpoint.save_every_iteration=false
``` ```
Use explicit run directories for experiments. Put Deep CFR runs under Use explicit run directories for experiments. Put Deep CFR runs under
@@ -95,17 +93,16 @@ Useful train controls:
- `--resume PATH`: resume from a specific checkpoint. - `--resume PATH`: resume from a specific checkpoint.
- `--set PATH=VALUE`: override config fields. It is repeatable and parses - `--set PATH=VALUE`: override config fields. It is repeatable and parses
values as YAML, e.g. `--set traversal.num_workers=4` or values as YAML, e.g. `--set traversal.num_workers=4` or
`--set run.max_hours=null`. `--set run.max_minutes=null`.
Common `--set` overrides: Common `--set` overrides:
- `--set run.device=cuda`: set the trainer device. - `--set run.device=cuda`: set the trainer device.
- `--set checkpoint.exact_resume=true`: require checkpoint config compatibility. - `--set checkpoint.exact_resume=true`: require checkpoint config compatibility.
- `--set checkpoint.save_latest=false --set checkpoint.save_every_iteration=false - `--set checkpoint.save_latest=false --set checkpoint.save_every=0`:
--set checkpoint.save_iteration_interval=0`: disable checkpoint writes. disable checkpoint writes.
- `--set checkpoint.save_latest_only=true --set checkpoint.save_every_iteration=false`: - `--set checkpoint.save_every=0`: keep only `latest.pt` (no archives).
keep only `latest.pt`. - `--set checkpoint.save_every=N`: archive every N iterations.
- `--set checkpoint.save_iteration_interval=N`: archive every N iterations.
## Long Runs ## Long Runs
@@ -149,16 +146,15 @@ The unbounded config intentionally has:
```yaml ```yaml
run: run:
iterations: null
max_iterations: null max_iterations: null
max_hours: null max_minutes: null
checkpoint: checkpoint:
save_iteration_interval: 100 save_every: 100
``` ```
`latest.pt` is updated continuously; archive checkpoints are written every 100 `latest.pt` is updated continuously; archive checkpoints are written every 100
iterations. If disk is tight, prefer `--set checkpoint.save_latest_only=true` or iterations. If disk is tight, set `--set checkpoint.save_every=0` (keep only
increase `save_iteration_interval`. `latest.pt`) or increase `save_every`.
## Evaluation And Analysis ## Evaluation And Analysis
configs/deep_cfr/deep_cfr_color_shared_512x3.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_color_shared_512x3 experiment_name: lost_cities_deep_cfr_color_shared_512x3
iterations: null
seed: 42 seed: 42
max_iterations: 10000 max_iterations: 10000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -30,10 +29,10 @@ network:
color_attention_heads: 4 color_attention_heads: 4
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -49,7 +48,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -71,9 +70,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 512 advantage_updates_per_iteration: 512
@@ -89,9 +87,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_color_shared_512x3 directory: runs/deep_cfr/deep_cfr_color_shared_512x3
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_color_shared_attention_512x3.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_color_shared_attention_512x3 experiment_name: lost_cities_deep_cfr_color_shared_attention_512x3
iterations: null
seed: 42 seed: 42
max_iterations: 10000 max_iterations: 10000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -30,10 +29,10 @@ network:
color_attention_heads: 8 color_attention_heads: 8
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -49,7 +48,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -71,9 +70,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 512 advantage_updates_per_iteration: 512
@@ -89,9 +87,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_color_shared_attention_512x3 directory: runs/deep_cfr/deep_cfr_color_shared_attention_512x3
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_color_shared_attention_exp_1000iter.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_color_shared_attention_1000iter experiment_name: lost_cities_deep_cfr_color_shared_attention_1000iter
iterations: null
seed: 42 seed: 42
max_iterations: 1000 max_iterations: 1000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -30,10 +29,10 @@ network:
color_attention_heads: 8 color_attention_heads: 8
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -49,7 +48,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -71,9 +70,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 256 advantage_updates_per_iteration: 256
@@ -89,9 +87,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_color_shared_attention_exp_1000iter directory: runs/deep_cfr/deep_cfr_color_shared_attention_exp_1000iter
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_opponent_average_strategy_512x3_1000iter.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_opponent_average_strategy_512x3_1000iter experiment_name: lost_cities_deep_cfr_opponent_average_strategy_512x3_1000iter
iterations: null
seed: 79 seed: 79
max_iterations: 1000 max_iterations: 1000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -27,10 +26,10 @@ network:
activation: relu activation: relu
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -46,7 +45,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -68,9 +67,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 512 advantage_updates_per_iteration: 512
@@ -86,9 +84,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_opponent_average_strategy_512x3_1000iter directory: runs/deep_cfr/deep_cfr_opponent_average_strategy_512x3_1000iter
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_opponent_network_1024x4_1000iter.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_opponent_network_1024x4_1000iter experiment_name: lost_cities_deep_cfr_opponent_network_1024x4_1000iter
iterations: null
seed: 79 seed: 79
max_iterations: 1000 max_iterations: 1000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -27,10 +26,10 @@ network:
activation: relu activation: relu
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -46,7 +45,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -68,9 +67,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 512 advantage_updates_per_iteration: 512
@@ -86,9 +84,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_opponent_network_1024x4_1000iter directory: runs/deep_cfr/deep_cfr_opponent_network_1024x4_1000iter
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_opponent_network_512x3_1000iter.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_opponent_network_512x3_1000iter experiment_name: lost_cities_deep_cfr_opponent_network_512x3_1000iter
iterations: null
seed: 79 seed: 79
max_iterations: 1000 max_iterations: 1000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -27,10 +26,10 @@ network:
activation: relu activation: relu
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -46,7 +45,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -68,9 +67,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 512 advantage_updates_per_iteration: 512
@@ -86,9 +84,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_opponent_network_512x3_1000iter directory: runs/deep_cfr/deep_cfr_opponent_network_512x3_1000iter
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter experiment_name: lost_cities_deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter
iterations: null
seed: 79 seed: 79
max_iterations: 10000 max_iterations: 10000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -27,10 +26,10 @@ network:
activation: relu activation: relu
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -46,7 +45,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -68,9 +67,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 512 advantage_updates_per_iteration: 512
@@ -86,9 +84,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter directory: runs/deep_cfr/deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter experiment_name: lost_cities_deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter
iterations: null
seed: 79 seed: 79
max_iterations: 10000 max_iterations: 10000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -27,10 +26,10 @@ network:
activation: relu activation: relu
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -46,7 +45,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -68,9 +67,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 512 advantage_updates_per_iteration: 512
@@ -86,9 +84,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter directory: runs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml
+3 -5
View File
@@ -2,7 +2,7 @@ run:
experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability
seed: 79 seed: 79
max_iterations: null max_iterations: null
max_hours: 4 max_minutes: 240
device: cuda device: cuda
use_amp: false use_amp: false
@@ -39,7 +39,7 @@ traversal:
cutoff_rollout_max_steps: 300 cutoff_rollout_max_steps: 300
progress_every_traversals: 10 progress_every_traversals: 10
num_workers: 8 num_workers: 8
traversal_worker_chunk_size: 8 worker_chunk_size: 8
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
outcome_sampling_value_clip: 500 outcome_sampling_value_clip: 500
@@ -87,7 +87,5 @@ evaluation:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability
save_every_iteration: false save_every: 10
save_iteration_interval: 10
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter
iterations: null
seed: 79 seed: 79
max_iterations: 10000 max_iterations: 10000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -27,10 +26,10 @@ network:
activation: relu activation: relu
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -46,7 +45,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -68,9 +67,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 512 advantage_updates_per_iteration: 512
@@ -86,9 +84,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter
iterations: null
seed: 79 seed: 79
max_iterations: 10000 max_iterations: 10000
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -27,10 +26,10 @@ network:
activation: relu activation: relu
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -46,7 +45,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -69,9 +68,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 256 advantage_updates_per_iteration: 256
@@ -87,9 +85,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded.yaml
+7 -11
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded
iterations: null
seed: 79 seed: 79
max_iterations: null max_iterations: null
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -27,10 +26,10 @@ network:
activation: relu activation: relu
traversal: traversal:
traversals_per_iteration: 2 traversals_per_player: 2
traversals_per_player: 70 traversals_per_player: 70
max_depth: null max_depth: null
max_nodes: 10000 max_nodes_per_traversal: 10000
max_nodes_per_traversal: 1000 max_nodes_per_traversal: 1000
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
@@ -46,7 +45,7 @@ traversal:
store_strategy_on_opponent_nodes: false store_strategy_on_opponent_nodes: false
num_workers: 8 num_workers: 8
worker_chunk_size: 4 worker_chunk_size: 4
traversal_worker_chunk_size: 8 worker_chunk_size: 8
progress_every_traversals: 10 progress_every_traversals: 10
endpoint_depth_bucket_width: 100 endpoint_depth_bucket_width: 100
endpoint_depth_bucket_max: 1000 endpoint_depth_bucket_max: 1000
@@ -68,9 +67,8 @@ self_play:
recent_window: 5 recent_window: 5
optimization: optimization:
advantage_train_steps: 1 advantage_updates_per_iteration: 1
strategy_train_steps: 1 strategy_updates_per_iteration: 1
batch_size: 32
advantage_batch_size: 1024 advantage_batch_size: 1024
strategy_batch_size: 1024 strategy_batch_size: 1024
advantage_updates_per_iteration: 256 advantage_updates_per_iteration: 256
@@ -86,9 +84,7 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded
save_latest: true save_latest: true
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
exact_resume: false exact_resume: false
configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_unbounded.yaml
+3 -6
View File
@@ -1,9 +1,8 @@
run: run:
experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_unbounded experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_unbounded
iterations: null
seed: 79 seed: 79
max_iterations: null max_iterations: null
max_hours: null max_minutes: null
device: cuda device: cuda
use_amp: false use_amp: false
@@ -40,7 +39,7 @@ traversal:
cutoff_rollout_max_steps: 300 cutoff_rollout_max_steps: 300
progress_every_traversals: 10 progress_every_traversals: 10
num_workers: 8 num_workers: 8
traversal_worker_chunk_size: 8 worker_chunk_size: 8
regret_matching_epsilon: 0.0001 regret_matching_epsilon: 0.0001
outcome_sampling_epsilon: 0.2 outcome_sampling_epsilon: 0.2
outcome_sampling_value_clip: 500 outcome_sampling_value_clip: 500
@@ -88,7 +87,5 @@ evaluation:
checkpoint: checkpoint:
directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_unbounded directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_unbounded
save_every_iteration: false save_every: 100
save_iteration_interval: 100
save_latest_only: false
progress_interval_seconds: 20.0 progress_interval_seconds: 20.0
configs/deep_cfr/smoke.yaml
+9 -7
View File
@@ -1,5 +1,5 @@
run: run:
iterations: 1 max_iterations: 1
seed: 1 seed: 1
device: cpu device: cpu
@@ -7,16 +7,17 @@ network:
hidden_size: 16 hidden_size: 16
traversal: traversal:
traversals_per_iteration: 1 traversals_per_player: 1
max_depth: 2 max_depth: 2
max_nodes: 64 max_nodes_per_traversal: 64
num_workers: 0 num_workers: 0
worker_chunk_size: 1 worker_chunk_size: 1
optimization: optimization:
advantage_train_steps: 1 advantage_batch_size: 2
strategy_train_steps: 1 strategy_batch_size: 2
batch_size: 2 advantage_updates_per_iteration: 1
strategy_updates_per_iteration: 1
learning_rate: 0.001 learning_rate: 0.001
memory: memory:
@@ -25,7 +26,8 @@ memory:
checkpoint: checkpoint:
directory: runs/deep_cfr/smoke directory: runs/deep_cfr/smoke
save_every_iteration: false save_every: 0
save_latest: false
evaluation: evaluation:
eval_every: 0 eval_every: 0
src/coolrl_lost_cities/games/classic/deep_cfr/benchmark.py
+5 -4
View File
@@ -13,14 +13,15 @@ def benchmark_traversal(
) -> dict[str, float | int]: ) -> dict[str, float | int]:
base = config or DeepCFRConfig.model_validate( base = config or DeepCFRConfig.model_validate(
{ {
"run": {"iterations": 1}, "run": {"max_iterations": 1},
"traversal": {"traversals_per_iteration": 8, "max_depth": 4}, "traversal": {"traversals_per_player": 8, "max_depth": 4},
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0, "save_latest": False},
} }
) )
data = base.model_dump(mode="python") data = base.model_dump(mode="python")
data["traversal"]["num_workers"] = num_workers data["traversal"]["num_workers"] = num_workers
data["checkpoint"]["save_every_iteration"] = False data["checkpoint"]["save_every"] = 0
data["checkpoint"]["save_latest"] = False
data["evaluation"]["eval_every"] = 0 data["evaluation"]["eval_every"] = 0
cfg = DeepCFRConfig.model_validate(data) cfg = DeepCFRConfig.model_validate(data)
trainer = DeepCFRTrainer(cfg, cfg.rules.to_lost_cities_config(seed=cfg.run.seed)) trainer = DeepCFRTrainer(cfg, cfg.rules.to_lost_cities_config(seed=cfg.run.seed))
src/coolrl_lost_cities/games/classic/deep_cfr/cli.py
+3 -3
View File
@@ -136,10 +136,10 @@ def benchmark_command(args: argparse.Namespace) -> None:
{ {
"run": {"seed": args.seed}, "run": {"seed": args.seed},
"traversal": { "traversal": {
"traversals_per_iteration": args.traversals, "traversals_per_player": args.traversals,
"max_depth": args.depth, "max_depth": args.depth,
}, },
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
} }
) )
if args.compare: if args.compare:
@@ -212,7 +212,7 @@ def main(argv: list[str] | None = None) -> None:
metavar="PATH=VALUE", metavar="PATH=VALUE",
help=( help=(
"Override config fields using dotted paths. Repeatable. VALUE is parsed as " "Override config fields using dotted paths. Repeatable. VALUE is parsed as "
"YAML, e.g. --set traversal.num_workers=4 --set run.max_hours=null." "YAML, e.g. --set traversal.num_workers=4 --set run.max_minutes=null."
), ),
) )
train.add_argument( train.add_argument(
src/coolrl_lost_cities/games/classic/deep_cfr/config.py
+15 -66
View File
@@ -18,23 +18,18 @@ class StrictModel(BaseModel):
class RunConfig(StrictModel): class RunConfig(StrictModel):
experiment_name: str = "deep_cfr" experiment_name: str = "deep_cfr"
iterations: int | None = 1
max_iterations: int | None = None max_iterations: int | None = None
max_hours: float | None = None max_minutes: float | None = None
seed: int = 1 seed: int = 1
device: str = "cpu" device: str = "auto"
use_amp: bool = False use_amp: bool = False
@field_validator("device") @field_validator("device")
@classmethod @classmethod
def _normalize_device(cls, value: str) -> str: def _normalize_device(cls, value: str) -> str:
token = value.strip().lower() token = value.strip().lower()
if token == "cuda": if token in {"cuda", "cpu", "auto"}:
return "cuda" return token
if token == "cpu":
return "cpu"
if token == "auto":
return "auto"
return token return token
@@ -95,12 +90,10 @@ class NetworkConfig(StrictModel):
class TraversalConfig(StrictModel): class TraversalConfig(StrictModel):
traversals_per_iteration: int = 2 traversals_per_player: int = 8
traversals_per_player: int | None = None
sampling_mode: str = "outcome" sampling_mode: str = "outcome"
max_depth: int | None = 8 max_depth: int | None = None
max_nodes: int | None = 10_000 max_nodes_per_traversal: int | None = 10_000
max_nodes_per_traversal: int | None = None
regret_matching_epsilon: float = 1.0e-8 regret_matching_epsilon: float = 1.0e-8
outcome_sampling_epsilon: float = 0.0 outcome_sampling_epsilon: float = 0.0
outcome_sampling_value_clip: float | None = None outcome_sampling_value_clip: float | None = None
@@ -109,13 +102,12 @@ class TraversalConfig(StrictModel):
cutoff_rollouts: int = 0 cutoff_rollouts: int = 0
cutoff_rollout_policy: str = "random" cutoff_rollout_policy: str = "random"
cutoff_rollout_max_steps: int = 10_000 cutoff_rollout_max_steps: int = 10_000
opponent_policy: str = "network" opponent_policy: str = "self_play_league"
strategy_sample_interval: int = 1 strategy_sample_interval: int = 1
store_strategy_on_traverser_nodes: bool = True store_strategy_on_traverser_nodes: bool = True
store_strategy_on_opponent_nodes: bool = True store_strategy_on_opponent_nodes: bool = True
num_workers: int | str = 0 num_workers: int | str = 0
worker_chunk_size: int = 4 worker_chunk_size: int = 4
traversal_worker_chunk_size: int | None = None
progress_every_traversals: int = 0 progress_every_traversals: int = 0
endpoint_depth_bucket_width: int = 100 endpoint_depth_bucket_width: int = 100
endpoint_depth_bucket_max: int = 1000 endpoint_depth_bucket_max: int = 1000
@@ -169,21 +161,6 @@ class TraversalConfig(StrictModel):
workers = max(0, int(self.num_workers)) workers = max(0, int(self.num_workers))
return min(workers, batches) if batches is not None and batches > 0 else workers return min(workers, batches) if batches is not None and batches > 0 else workers
def resolved_traversals_per_player(self) -> int:
if self.traversals_per_player is not None:
return max(0, int(self.traversals_per_player))
return max(0, int(self.traversals_per_iteration))
def resolved_max_nodes(self) -> int | None:
if self.max_nodes_per_traversal is not None:
return self.max_nodes_per_traversal
return self.max_nodes
def resolved_worker_chunk_size(self) -> int:
if self.traversal_worker_chunk_size is not None:
return max(1, int(self.traversal_worker_chunk_size))
return max(1, int(self.worker_chunk_size))
class RegretMatchingConfig(StrictModel): class RegretMatchingConfig(StrictModel):
all_negative_fallback: str = "uniform" all_negative_fallback: str = "uniform"
@@ -225,37 +202,14 @@ class SelfPlayLeagueConfig(StrictModel):
class OptimizationConfig(StrictModel): class OptimizationConfig(StrictModel):
advantage_train_steps: int = 1 advantage_batch_size: int = 256
strategy_train_steps: int = 1 strategy_batch_size: int = 256
batch_size: int = 32 advantage_updates_per_iteration: int = 64
advantage_batch_size: int | None = None strategy_updates_per_iteration: int = 64
strategy_batch_size: int | None = None
advantage_updates_per_iteration: int | None = None
strategy_updates_per_iteration: int | None = None
learning_rate: float = 1.0e-3 learning_rate: float = 1.0e-3
weight_decay: float = 0.0 weight_decay: float = 0.0
grad_clip: float = 0.0 grad_clip: float = 0.0
def resolved_advantage_batch_size(self) -> int:
if self.advantage_batch_size is not None:
return max(1, int(self.advantage_batch_size))
return max(1, int(self.batch_size))
def resolved_strategy_batch_size(self) -> int:
if self.strategy_batch_size is not None:
return max(1, int(self.strategy_batch_size))
return max(1, int(self.batch_size))
def resolved_advantage_train_steps(self) -> int:
if self.advantage_updates_per_iteration is not None:
return max(0, int(self.advantage_updates_per_iteration))
return max(0, int(self.advantage_train_steps))
def resolved_strategy_train_steps(self) -> int:
if self.strategy_updates_per_iteration is not None:
return max(0, int(self.strategy_updates_per_iteration))
return max(0, int(self.strategy_train_steps))
class MemoryConfig(StrictModel): class MemoryConfig(StrictModel):
advantage_capacity: int = 2_000_000 advantage_capacity: int = 2_000_000
@@ -264,10 +218,8 @@ class MemoryConfig(StrictModel):
class CheckpointConfig(StrictModel): class CheckpointConfig(StrictModel):
directory: str = "runs/deep_cfr/default" directory: str = "runs/deep_cfr/default"
save_every: int = 1
save_latest: bool = True save_latest: bool = True
save_every_iteration: bool = True
save_iteration_interval: int = 0
save_latest_only: bool = False
progress_interval_seconds: float = 20.0 progress_interval_seconds: float = 20.0
exact_resume: bool = False exact_resume: bool = False
@@ -277,9 +229,9 @@ class CheckpointConfig(StrictModel):
class EvaluationConfig(StrictModel): class EvaluationConfig(StrictModel):
eval_every: int = 0 eval_every: int = 50
games: int = 10 games: int = 10
opponents: tuple[str, ...] = ("random",) opponents: tuple[str, ...] = ("random", "safe_heuristic")
max_steps: int = 10_000 max_steps: int = 10_000
on_max_steps: str = "score_diff" on_max_steps: str = "score_diff"
batch_size: int = 64 batch_size: int = 64
@@ -302,9 +254,6 @@ class EvaluationConfig(StrictModel):
raise ValueError("must be 'trainer', 'auto', 'cpu', or 'cuda'") raise ValueError("must be 'trainer', 'auto', 'cpu', or 'cuda'")
return token return token
def resolved_batch_size(self) -> int:
return max(1, int(self.batch_size))
def resolved_num_workers(self, opponent_count: int | None = None) -> int: def resolved_num_workers(self, opponent_count: int | None = None) -> int:
workers = max(1, int(self.num_workers)) workers = max(1, int(self.num_workers))
if opponent_count is not None: if opponent_count is not None:
src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py
+18 -24
View File
@@ -329,7 +329,7 @@ class DeepCFRTrainer:
progress_every = int(self.config.traversal.progress_every_traversals) progress_every = int(self.config.traversal.progress_every_traversals)
completed = 0 completed = 0
progress_started = time.perf_counter() progress_started = time.perf_counter()
traversals_per_player = self.config.traversal.resolved_traversals_per_player() traversals_per_player = self.config.traversal.traversals_per_player
for player in range(2): for player in range(2):
seeds = [ seeds = [
self.config.run.seed + iteration * 10_000 + traversal_index * 10 + player self.config.run.seed + iteration * 10_000 + traversal_index * 10 + player
@@ -358,7 +358,7 @@ class DeepCFRTrainer:
self.config.traversal.store_strategy_on_opponent_nodes self.config.traversal.store_strategy_on_opponent_nodes
), ),
max_depth=self.config.traversal.max_depth, max_depth=self.config.traversal.max_depth,
max_nodes=self.config.traversal.resolved_max_nodes(), max_nodes=self.config.traversal.max_nodes_per_traversal,
sampling_mode=self.config.traversal.sampling_mode, sampling_mode=self.config.traversal.sampling_mode,
outcome_sampling_epsilon=self.config.traversal.outcome_sampling_epsilon, outcome_sampling_epsilon=self.config.traversal.outcome_sampling_epsilon,
outcome_sampling_value_clip=self.config.traversal.outcome_sampling_value_clip, outcome_sampling_value_clip=self.config.traversal.outcome_sampling_value_clip,
@@ -409,7 +409,7 @@ class DeepCFRTrainer:
self.tracker.log_event( self.tracker.log_event(
f"Traversal multiprocessing enabled iteration={iteration} " f"Traversal multiprocessing enabled iteration={iteration} "
f"requested_workers={requested_workers} effective_workers={max_workers} " f"requested_workers={requested_workers} effective_workers={max_workers} "
f"batches={len(batches)} chunk_size={self.config.traversal.resolved_worker_chunk_size()}" f"batches={len(batches)} chunk_size={self.config.traversal.worker_chunk_size}"
) )
if max_workers < requested_workers: if max_workers < requested_workers:
self.tracker.log_event( self.tracker.log_event(
@@ -481,12 +481,12 @@ class DeepCFRTrainer:
name: value.detach().cpu() name: value.detach().cpu()
for name, value in self.strategy_network.state_dict().items() for name, value in self.strategy_network.state_dict().items()
} }
chunk_size = self.config.traversal.resolved_worker_chunk_size() chunk_size = self.config.traversal.worker_chunk_size
batch_index = 0 batch_index = 0
for player in range(2): for player in range(2):
seeds = [ seeds = [
self.config.run.seed + iteration * 10_000 + index * 10 + player self.config.run.seed + iteration * 10_000 + index * 10 + player
for index in range(self.config.traversal.resolved_traversals_per_player()) for index in range(self.config.traversal.traversals_per_player)
] ]
for start in range(0, len(seeds), chunk_size): for start in range(0, len(seeds), chunk_size):
chunk = seeds[start : start + chunk_size] chunk = seeds[start : start + chunk_size]
@@ -574,27 +574,21 @@ class DeepCFRTrainer:
def _stop_iteration(self) -> int: def _stop_iteration(self) -> int:
if self.config.run.max_iterations is not None: if self.config.run.max_iterations is not None:
return max(self.iteration, int(self.config.run.max_iterations)) return max(self.iteration, int(self.config.run.max_iterations))
if self.config.run.max_hours is not None: return 2**31 - 1
return 2**31 - 1
if self.config.run.iterations is None:
return 2**31 - 1
return self.iteration + self.config.run.iterations
def _time_limit_reached(self, run_started: float) -> bool: def _time_limit_reached(self, run_started: float) -> bool:
if self.config.run.max_hours is None: if self.config.run.max_minutes is None:
return False return False
elapsed_hours = (time.perf_counter() - run_started) / 3600.0 elapsed_minutes = (time.perf_counter() - run_started) / 60.0
return elapsed_hours >= self.config.run.max_hours return elapsed_minutes >= self.config.run.max_minutes
def _should_save_iteration(self, iteration: int) -> bool: def _should_save_iteration(self, iteration: int) -> bool:
if self.config.checkpoint.save_every_iteration: interval = int(self.config.checkpoint.save_every)
return True
interval = int(self.config.checkpoint.save_iteration_interval)
return interval > 0 and iteration % interval == 0 return interval > 0 and iteration % interval == 0
def _save_iteration_checkpoints(self, iteration: int, item: IterationMetrics) -> None: def _save_iteration_checkpoints(self, iteration: int, item: IterationMetrics) -> None:
checkpoint_dir = self.run_dir checkpoint_dir = self.run_dir
if self._should_save_iteration(iteration) and not self.config.checkpoint.save_latest_only: if self._should_save_iteration(iteration):
self.save_checkpoint(checkpoint_dir / f"iteration_{iteration:05d}.pt", item) self.save_checkpoint(checkpoint_dir / f"iteration_{iteration:05d}.pt", item)
if self.config.checkpoint.save_latest: if self.config.checkpoint.save_latest:
self.save_checkpoint(checkpoint_dir / "latest.pt", item) self.save_checkpoint(checkpoint_dir / "latest.pt", item)
@@ -641,7 +635,7 @@ class DeepCFRTrainer:
device=eval_device, device=eval_device,
max_steps=self.config.evaluation.max_steps, max_steps=self.config.evaluation.max_steps,
encoding=self.config.encoding, encoding=self.config.encoding,
batch_size=self.config.evaluation.resolved_batch_size(), batch_size=self.config.evaluation.batch_size,
) )
for key, value in result.items(): for key, value in result.items():
results[f"eval_{opponent}_{key}"] = value results[f"eval_{opponent}_{key}"] = value
@@ -657,7 +651,7 @@ class DeepCFRTrainer:
self.tracker.log_event( self.tracker.log_event(
f"Evaluation multiprocessing enabled iteration={iteration} " f"Evaluation multiprocessing enabled iteration={iteration} "
f"effective_workers={max_workers} opponents={len(opponents)} " f"effective_workers={max_workers} opponents={len(opponents)} "
f"batch_size={self.config.evaluation.resolved_batch_size()} device={eval_device}" f"batch_size={self.config.evaluation.batch_size} device={eval_device}"
) )
state_dict = self._strategy_state_dict_cpu() state_dict = self._strategy_state_dict_cpu()
jobs = [ jobs = [
@@ -673,7 +667,7 @@ class DeepCFRTrainer:
seed=self.config.run.seed + iteration * 1000, seed=self.config.run.seed + iteration * 1000,
device=str(eval_device), device=str(eval_device),
max_steps=self.config.evaluation.max_steps, max_steps=self.config.evaluation.max_steps,
batch_size=self.config.evaluation.resolved_batch_size(), batch_size=self.config.evaluation.batch_size,
) )
for opponent in opponents for opponent in opponents
] ]
@@ -773,10 +767,10 @@ class DeepCFRTrainer:
) -> float: ) -> float:
losses: list[float] = [] losses: list[float] = []
network.train() network.train()
for _step in range(self.config.optimization.resolved_advantage_train_steps()): for _step in range(self.config.optimization.advantage_updates_per_iteration):
sample_started = time.perf_counter() sample_started = time.perf_counter()
batch = self.advantage_memories[player].sample( batch = self.advantage_memories[player].sample(
self.config.optimization.resolved_advantage_batch_size(), self.config.optimization.advantage_batch_size,
self.rng, self.rng,
) )
self._runtime_metrics[f"advantage_player_{player}_sample_seconds"] = ( self._runtime_metrics[f"advantage_player_{player}_sample_seconds"] = (
@@ -828,10 +822,10 @@ class DeepCFRTrainer:
) -> float: ) -> float:
last_loss = 0.0 last_loss = 0.0
network.train() network.train()
for _step in range(self.config.optimization.resolved_strategy_train_steps()): for _step in range(self.config.optimization.strategy_updates_per_iteration):
sample_started = time.perf_counter() sample_started = time.perf_counter()
batch = self.strategy_memory.sample( batch = self.strategy_memory.sample(
self.config.optimization.resolved_strategy_batch_size(), self.rng self.config.optimization.strategy_batch_size, self.rng
) )
self._runtime_metrics["strategy_sample_seconds"] = ( self._runtime_metrics["strategy_sample_seconds"] = (
float(self._runtime_metrics.get("strategy_sample_seconds", 0.0)) float(self._runtime_metrics.get("strategy_sample_seconds", 0.0))
src/coolrl_lost_cities/games/classic/deep_cfr/workers.py
+1 -1
View File
@@ -100,7 +100,7 @@ def run_traversal_worker_batch(batch: TraversalWorkerBatch) -> TraversalWorkerRe
store_strategy_on_traverser_nodes=cfg.traversal.store_strategy_on_traverser_nodes, store_strategy_on_traverser_nodes=cfg.traversal.store_strategy_on_traverser_nodes,
store_strategy_on_opponent_nodes=cfg.traversal.store_strategy_on_opponent_nodes, store_strategy_on_opponent_nodes=cfg.traversal.store_strategy_on_opponent_nodes,
max_depth=cfg.traversal.max_depth, max_depth=cfg.traversal.max_depth,
max_nodes=cfg.traversal.resolved_max_nodes(), max_nodes=cfg.traversal.max_nodes_per_traversal,
sampling_mode=cfg.traversal.sampling_mode, sampling_mode=cfg.traversal.sampling_mode,
outcome_sampling_epsilon=cfg.traversal.outcome_sampling_epsilon, outcome_sampling_epsilon=cfg.traversal.outcome_sampling_epsilon,
outcome_sampling_value_clip=cfg.traversal.outcome_sampling_value_clip, outcome_sampling_value_clip=cfg.traversal.outcome_sampling_value_clip,
tests/games/classic/test_deep_cfr_trainer.py
+111 -114
View File
@@ -33,9 +33,9 @@ def _deep_cfr_config(data: dict) -> DeepCFRConfig:
def test_deep_cfr_loads_smoke_yaml_config() -> None: def test_deep_cfr_loads_smoke_yaml_config() -> None:
config = load_config("configs/deep_cfr/smoke.yaml") config = load_config("configs/deep_cfr/smoke.yaml")
assert config.run.iterations == 1 assert config.run.max_iterations == 1
assert config.network.hidden_size == 16 assert config.network.hidden_size == 16
assert config.traversal.traversals_per_iteration == 1 assert config.traversal.traversals_per_player == 1
assert config.checkpoint.directory == "runs/deep_cfr/smoke" assert config.checkpoint.directory == "runs/deep_cfr/smoke"
@@ -45,30 +45,30 @@ def test_deep_cfr_loads_mapped_legacy_reproduction_config() -> None:
assert config.run.experiment_name.endswith("slot_playability") assert config.run.experiment_name.endswith("slot_playability")
assert config.run.seed == 79 assert config.run.seed == 79
assert config.run.max_iterations is None assert config.run.max_iterations is None
assert config.run.max_hours == 4 assert config.run.max_minutes == 240
assert config.encoding.derived_playability is True assert config.encoding.derived_playability is True
assert config.encoding.slot_aware_playability is True assert config.encoding.slot_aware_playability is True
assert config.network.hidden_size == 256 assert config.network.hidden_size == 256
assert config.network.num_layers == 3 assert config.network.num_layers == 3
assert config.traversal.resolved_traversals_per_player() == 70 assert config.traversal.traversals_per_player == 70
assert config.traversal.sampling_mode == "outcome" assert config.traversal.sampling_mode == "outcome"
assert config.traversal.max_depth is None assert config.traversal.max_depth is None
assert config.traversal.resolved_max_nodes() == 1000 assert config.traversal.max_nodes_per_traversal == 1000
assert config.traversal.resolved_worker_chunk_size() == 8 assert config.traversal.worker_chunk_size == 8
assert config.traversal.progress_every_traversals == 10 assert config.traversal.progress_every_traversals == 10
assert config.optimization.resolved_advantage_batch_size() == 1024 assert config.optimization.advantage_batch_size == 1024
assert config.optimization.resolved_strategy_batch_size() == 1024 assert config.optimization.strategy_batch_size == 1024
assert config.optimization.resolved_advantage_train_steps() == 256 assert config.optimization.advantage_updates_per_iteration == 256
assert config.optimization.resolved_strategy_train_steps() == 256 assert config.optimization.strategy_updates_per_iteration == 256
assert config.optimization.weight_decay == 0.0001 assert config.optimization.weight_decay == 0.0001
assert config.optimization.grad_clip == 1.0 assert config.optimization.grad_clip == 1.0
assert config.evaluation.on_max_steps == "score_diff" assert config.evaluation.on_max_steps == "score_diff"
assert config.evaluation.resolved_batch_size() == 64 assert config.evaluation.batch_size == 64
assert config.evaluation.device == "trainer" assert config.evaluation.device == "trainer"
assert config.evaluation.resolved_num_workers() == 4 assert config.evaluation.resolved_num_workers() == 4
assert config.regret_matching.all_negative_fallback == "uniform" assert config.regret_matching.all_negative_fallback == "uniform"
assert config.training_weighting.mode == "none" assert config.training_weighting.mode == "none"
assert config.checkpoint.save_iteration_interval == 10 assert config.checkpoint.save_every == 10
assert ( assert (
config.checkpoint.directory == "runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability" config.checkpoint.directory == "runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability"
) )
@@ -80,17 +80,14 @@ def test_deep_cfr_train_cli_accepts_run_and_traversal_config_overrides() -> None
(), (),
{ {
"config_overrides": [ "config_overrides": [
"run.iterations=1", "run.max_iterations=1",
"run.max_hours=null", "run.max_minutes=null",
"run.max_iterations=null", "traversal.traversals_per_player=1",
"traversal.traversals_per_iteration=1",
"traversal.traversals_per_player=null",
"traversal.num_workers=0", "traversal.num_workers=0",
"regret_matching.all_negative_fallback=argmax_tiebreak", "regret_matching.all_negative_fallback=argmax_tiebreak",
"training_weighting.mode=lcfr", "training_weighting.mode=lcfr",
"checkpoint.save_latest=false", "checkpoint.save_latest=false",
"checkpoint.save_every_iteration=false", "checkpoint.save_every=0",
"checkpoint.save_iteration_interval=0",
], ],
}, },
)() )()
@@ -98,15 +95,13 @@ def test_deep_cfr_train_cli_accepts_run_and_traversal_config_overrides() -> None
overridden = _with_overrides(config, _train_overrides_from_args(args)) overridden = _with_overrides(config, _train_overrides_from_args(args))
assert overridden.run.iterations == 1 assert overridden.run.max_iterations == 1
assert overridden.run.max_hours is None assert overridden.run.max_minutes is None
assert overridden.run.max_iterations is None assert overridden.traversal.traversals_per_player == 1
assert overridden.traversal.traversals_per_player is None
assert overridden.traversal.resolved_traversals_per_player() == 1
assert overridden.traversal.resolved_num_workers() == 0 assert overridden.traversal.resolved_num_workers() == 0
assert overridden.regret_matching.all_negative_fallback == "argmax_tiebreak" assert overridden.regret_matching.all_negative_fallback == "argmax_tiebreak"
assert overridden.training_weighting.mode == "lcfr" assert overridden.training_weighting.mode == "lcfr"
assert overridden.checkpoint.save_every_iteration is False assert overridden.checkpoint.save_every == 0
assert overridden.checkpoint.save_latest is False assert overridden.checkpoint.save_latest is False
@@ -123,9 +118,7 @@ def test_deep_cfr_train_cli_checkpoint_save_overrides() -> None:
{ {
"config_overrides": [ "config_overrides": [
"checkpoint.save_latest=true", "checkpoint.save_latest=true",
"checkpoint.save_latest_only=true", "checkpoint.save_every=1",
"checkpoint.save_every_iteration=false",
"checkpoint.save_iteration_interval=1",
], ],
}, },
)() )()
@@ -133,9 +126,7 @@ def test_deep_cfr_train_cli_checkpoint_save_overrides() -> None:
overridden = _with_overrides(DeepCFRConfig(), _train_overrides_from_args(args)) overridden = _with_overrides(DeepCFRConfig(), _train_overrides_from_args(args))
assert overridden.checkpoint.save_latest is True assert overridden.checkpoint.save_latest is True
assert overridden.checkpoint.save_latest_only is True assert overridden.checkpoint.save_every == 1
assert overridden.checkpoint.save_every_iteration is False
assert overridden.checkpoint.save_iteration_interval == 1
def test_deep_cfr_train_cli_accepts_generic_config_overrides() -> None: def test_deep_cfr_train_cli_accepts_generic_config_overrides() -> None:
@@ -146,8 +137,8 @@ def test_deep_cfr_train_cli_accepts_generic_config_overrides() -> None:
"config_overrides": [ "config_overrides": [
"traversal.sampling_mode=external", "traversal.sampling_mode=external",
"traversal.max_depth=null", "traversal.max_depth=null",
"optimization.batch_size=64", "optimization.advantage_batch_size=64",
"checkpoint.save_latest_only=true", "checkpoint.save_latest=true",
], ],
}, },
)() )()
@@ -156,17 +147,17 @@ def test_deep_cfr_train_cli_accepts_generic_config_overrides() -> None:
assert overridden.traversal.sampling_mode == "external" assert overridden.traversal.sampling_mode == "external"
assert overridden.traversal.max_depth is None assert overridden.traversal.max_depth is None
assert overridden.optimization.batch_size == 64 assert overridden.optimization.advantage_batch_size == 64
assert overridden.checkpoint.save_latest_only is True assert overridden.checkpoint.save_latest is True
def test_deep_cfr_iteration_weights_use_sample_age() -> None: def test_deep_cfr_iteration_weights_use_sample_age() -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 12}, "run": {"max_iterations": 1, "seed": 12},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
"training_weighting": {"mode": "lcfr", "lcfr_alpha": 1.0}, "training_weighting": {"mode": "lcfr", "lcfr_alpha": 1.0},
} }
), ),
@@ -279,16 +270,21 @@ def test_deep_cfr_slot_aware_playability_encoding_zero_fills_empty_hand_slots()
def test_deep_cfr_trainer_uses_playability_encoding() -> None: def test_deep_cfr_trainer_uses_playability_encoding() -> None:
config = _deep_cfr_config( config = _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 62}, "run": {"max_iterations": 1, "seed": 62},
"encoding": {"derived_playability": True, "slot_aware_playability": True}, "encoding": {"derived_playability": True, "slot_aware_playability": True},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": {"traversals_per_iteration": 1, "max_depth": 1, "max_nodes": 16}, "traversal": {
"optimization": { "traversals_per_player": 1,
"advantage_train_steps": 1, "max_depth": 1,
"strategy_train_steps": 1, "max_nodes_per_traversal": 16,
"batch_size": 2,
}, },
"checkpoint": {"save_every_iteration": False}, "optimization": {
"advantage_updates_per_iteration": 1,
"strategy_updates_per_iteration": 1,
"advantage_batch_size": 2,
"strategy_batch_size": 2,
},
"checkpoint": {"save_every": 0},
} }
) )
game_config = LostCitiesConfig(seed=62) game_config = LostCitiesConfig(seed=62)
@@ -317,17 +313,17 @@ def test_deep_cfr_trainer_forwards_metrics_to_extra_trackers(tmp_path) -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 71}, "run": {"max_iterations": 1, "seed": 71},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"max_depth": 2, "max_depth": 2,
"max_nodes": 16, "max_nodes_per_traversal": 16,
}, },
"optimization": {"batch_size": 2}, "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2},
"checkpoint": { "checkpoint": {
"directory": str(tmp_path / "extra-tracker"), "directory": str(tmp_path / "extra-tracker"),
"save_every_iteration": False, "save_every": 0,
}, },
} }
), ),
@@ -347,19 +343,20 @@ def test_deep_cfr_trainer_smoke_run() -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 23}, "run": {"max_iterations": 1, "seed": 23},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"max_depth": 3, "max_depth": 3,
"max_nodes": 64, "max_nodes_per_traversal": 64,
}, },
"optimization": { "optimization": {
"advantage_train_steps": 1, "advantage_updates_per_iteration": 1,
"strategy_train_steps": 1, "strategy_updates_per_iteration": 1,
"batch_size": 2, "advantage_batch_size": 2,
"strategy_batch_size": 2,
}, },
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
} }
), ),
LostCitiesConfig(seed=23), LostCitiesConfig(seed=23),
@@ -383,20 +380,21 @@ def test_deep_cfr_trainer_supports_lcfr_and_dcfr_loss_weighting() -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 24}, "run": {"max_iterations": 1, "seed": 24},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"max_depth": 2, "max_depth": 2,
"max_nodes": 32, "max_nodes_per_traversal": 32,
}, },
"optimization": { "optimization": {
"advantage_train_steps": 1, "advantage_updates_per_iteration": 1,
"strategy_train_steps": 1, "strategy_updates_per_iteration": 1,
"batch_size": 2, "advantage_batch_size": 2,
"strategy_batch_size": 2,
}, },
"training_weighting": {"mode": mode}, "training_weighting": {"mode": mode},
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
} }
), ),
LostCitiesConfig(seed=24), LostCitiesConfig(seed=24),
@@ -413,15 +411,15 @@ def test_deep_cfr_cython_traverser_restores_state_and_collects_samples() -> None
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 29}, "run": {"max_iterations": 1, "seed": 29},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"max_depth": 2, "max_depth": 2,
"max_nodes": 32, "max_nodes_per_traversal": 32,
}, },
"optimization": {"batch_size": 2}, "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2},
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
} }
), ),
LostCitiesConfig(seed=29), LostCitiesConfig(seed=29),
@@ -457,12 +455,12 @@ def test_deep_cfr_cython_traverser_supports_outcome_sampling_and_rollout_cutoffs
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 31}, "run": {"max_iterations": 1, "seed": 31},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"max_depth": 1, "max_depth": 1,
"max_nodes": 32, "max_nodes_per_traversal": 32,
"outcome_sampling_epsilon": 0.25, "outcome_sampling_epsilon": 0.25,
"outcome_sampling_value_clip": 100.0, "outcome_sampling_value_clip": 100.0,
"outcome_unsampled_regret": "zero", "outcome_unsampled_regret": "zero",
@@ -471,8 +469,8 @@ def test_deep_cfr_cython_traverser_supports_outcome_sampling_and_rollout_cutoffs
"cutoff_rollout_policy": "random", "cutoff_rollout_policy": "random",
"cutoff_rollout_max_steps": 16, "cutoff_rollout_max_steps": 16,
}, },
"optimization": {"batch_size": 2}, "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2},
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
} }
), ),
LostCitiesConfig(seed=31), LostCitiesConfig(seed=31),
@@ -512,16 +510,16 @@ def test_deep_cfr_cython_traverser_supports_external_sampling() -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 33}, "run": {"max_iterations": 1, "seed": 33},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"sampling_mode": "external", "sampling_mode": "external",
"max_depth": 1, "max_depth": 1,
"max_nodes": 64, "max_nodes_per_traversal": 64,
}, },
"optimization": {"batch_size": 2}, "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2},
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
} }
), ),
LostCitiesConfig(seed=33), LostCitiesConfig(seed=33),
@@ -563,11 +561,11 @@ def test_deep_cfr_cython_traverser_records_regret_fallback_metrics() -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 37}, "run": {"max_iterations": 1, "seed": 37},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": {"traversals_per_iteration": 1, "max_depth": 1}, "traversal": {"traversals_per_player": 1, "max_depth": 1},
"optimization": {"batch_size": 2}, "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2},
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
"regret_matching": {"all_negative_fallback": "argmax_tiebreak"}, "regret_matching": {"all_negative_fallback": "argmax_tiebreak"},
} }
), ),
@@ -637,17 +635,17 @@ def test_deep_cfr_trainer_saves_loads_and_evaluates_checkpoint(tmp_path) -> None
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 41}, "run": {"max_iterations": 1, "seed": 41},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"max_depth": 2, "max_depth": 2,
"max_nodes": 32, "max_nodes_per_traversal": 32,
}, },
"optimization": {"batch_size": 2}, "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2},
"checkpoint": { "checkpoint": {
"directory": str(checkpoint_dir), "directory": str(checkpoint_dir),
"save_every_iteration": True, "save_every": 1,
}, },
"evaluation": {"eval_every": 1, "games": 2, "opponents": ("random",)}, "evaluation": {"eval_every": 1, "games": 2, "opponents": ("random",)},
} }
@@ -664,7 +662,7 @@ def test_deep_cfr_trainer_saves_loads_and_evaluates_checkpoint(tmp_path) -> None
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"checkpoint": { "checkpoint": {
"directory": str(checkpoint_dir), "directory": str(checkpoint_dir),
"save_every_iteration": False, "save_every": 0,
}, },
} }
), ),
@@ -700,13 +698,12 @@ def test_deep_cfr_trainer_always_saves_latest_checkpoint(tmp_path) -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 42}, "run": {"max_iterations": 1, "seed": 42},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": {"traversals_per_iteration": 1, "max_depth": 1}, "traversal": {"traversals_per_player": 1, "max_depth": 1},
"checkpoint": { "checkpoint": {
"directory": str(checkpoint_dir), "directory": str(checkpoint_dir),
"save_every_iteration": False, "save_every": 10,
"save_iteration_interval": 10,
}, },
} }
), ),
@@ -724,10 +721,10 @@ def test_deep_cfr_exact_resume_is_explicitly_not_implemented(tmp_path) -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 44}, "run": {"max_iterations": 1, "seed": 44},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": {"traversals_per_iteration": 1, "max_depth": 1}, "traversal": {"traversals_per_player": 1, "max_depth": 1},
"checkpoint": {"directory": str(checkpoint_dir), "save_every_iteration": True}, "checkpoint": {"directory": str(checkpoint_dir), "save_every": 1},
} }
), ),
LostCitiesConfig(seed=44), LostCitiesConfig(seed=44),
@@ -755,20 +752,20 @@ def test_deep_cfr_trainer_multiprocessing_smoke_run(tmp_path) -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 1, "seed": 43}, "run": {"max_iterations": 1, "seed": 43},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"max_depth": 2, "max_depth": 2,
"max_nodes": 32, "max_nodes_per_traversal": 32,
"num_workers": 8, "num_workers": 8,
"worker_chunk_size": 1, "worker_chunk_size": 1,
"progress_every_traversals": 1, "progress_every_traversals": 1,
}, },
"optimization": {"batch_size": 2}, "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2},
"checkpoint": { "checkpoint": {
"directory": str(tmp_path / "mp"), "directory": str(tmp_path / "mp"),
"save_every_iteration": False, "save_every": 0,
}, },
} }
), ),
@@ -791,8 +788,8 @@ def test_deep_cfr_traversal_benchmark_smoke() -> None:
{ {
"run": {"seed": 47}, "run": {"seed": 47},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": {"traversals_per_iteration": 1, "max_depth": 2}, "traversal": {"traversals_per_player": 1, "max_depth": 2},
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
} }
) )
) )
@@ -804,8 +801,8 @@ def test_deep_cfr_traversal_benchmark_smoke() -> None:
{ {
"run": {"seed": 48}, "run": {"seed": 48},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": {"traversals_per_iteration": 1, "max_depth": 2}, "traversal": {"traversals_per_player": 1, "max_depth": 2},
"checkpoint": {"save_every_iteration": False}, "checkpoint": {"save_every": 0},
} }
) )
) )
@@ -816,12 +813,12 @@ def test_deep_cfr_self_play_league_records_snapshots(tmp_path) -> None:
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 2, "seed": 53}, "run": {"max_iterations": 2, "seed": 53},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"max_depth": 2, "max_depth": 2,
"max_nodes": 32, "max_nodes_per_traversal": 32,
"opponent_policy": "self_play_league", "opponent_policy": "self_play_league",
}, },
"self_play": { "self_play": {
@@ -829,10 +826,10 @@ def test_deep_cfr_self_play_league_records_snapshots(tmp_path) -> None:
"max_snapshots": 1, "max_snapshots": 1,
"anchor_probability": 1.0, "anchor_probability": 1.0,
}, },
"optimization": {"batch_size": 2}, "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2},
"checkpoint": { "checkpoint": {
"directory": str(tmp_path / "league"), "directory": str(tmp_path / "league"),
"save_every_iteration": False, "save_every": 0,
}, },
} }
), ),
@@ -849,12 +846,12 @@ def test_deep_cfr_weighted_self_play_league_uses_snapshot_bucket(tmp_path) -> No
trainer = DeepCFRTrainer( trainer = DeepCFRTrainer(
_deep_cfr_config( _deep_cfr_config(
{ {
"run": {"iterations": 2, "seed": 59}, "run": {"max_iterations": 2, "seed": 59},
"network": {"hidden_size": 16}, "network": {"hidden_size": 16},
"traversal": { "traversal": {
"traversals_per_iteration": 1, "traversals_per_player": 1,
"max_depth": 2, "max_depth": 2,
"max_nodes": 32, "max_nodes_per_traversal": 32,
"opponent_policy": "self_play_league", "opponent_policy": "self_play_league",
}, },
"self_play": { "self_play": {
@@ -866,10 +863,10 @@ def test_deep_cfr_weighted_self_play_league_uses_snapshot_bucket(tmp_path) -> No
"anchor_weight": 0.0, "anchor_weight": 0.0,
"recent_window": 1, "recent_window": 1,
}, },
"optimization": {"batch_size": 2}, "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2},
"checkpoint": { "checkpoint": {
"directory": str(tmp_path / "weighted-league"), "directory": str(tmp_path / "weighted-league"),
"save_every_iteration": False, "save_every": 0,
}, },
} }
), ),