diff --git a/AGENTS.md b/AGENTS.md index ef1d69f..887cd16 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,11 +66,9 @@ Short fixed-iteration run: ```bash uv run lost-cities-deep-cfr train \ --config configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml \ - --set run.iterations=100 \ - --set run.max_hours=null \ - --set run.max_iterations=null \ - --set checkpoint.save_latest_only=true \ - --set checkpoint.save_every_iteration=false + --set run.max_iterations=100 \ + --set run.max_minutes=null \ + --set checkpoint.save_every=0 ``` Use explicit run directories for experiments. Put Deep CFR runs under @@ -95,17 +93,16 @@ Useful train controls: - `--resume PATH`: resume from a specific checkpoint. - `--set PATH=VALUE`: override config fields. It is repeatable and parses values as YAML, e.g. `--set traversal.num_workers=4` or - `--set run.max_hours=null`. + `--set run.max_minutes=null`. Common `--set` overrides: - `--set run.device=cuda`: set the trainer device. - `--set checkpoint.exact_resume=true`: require checkpoint config compatibility. -- `--set checkpoint.save_latest=false --set checkpoint.save_every_iteration=false - --set checkpoint.save_iteration_interval=0`: disable checkpoint writes. -- `--set checkpoint.save_latest_only=true --set checkpoint.save_every_iteration=false`: - keep only `latest.pt`. -- `--set checkpoint.save_iteration_interval=N`: archive every N iterations. +- `--set checkpoint.save_latest=false --set checkpoint.save_every=0`: + disable checkpoint writes. +- `--set checkpoint.save_every=0`: keep only `latest.pt` (no archives). +- `--set checkpoint.save_every=N`: archive every N iterations. ## Long Runs @@ -149,16 +146,15 @@ The unbounded config intentionally has: ```yaml run: - iterations: null max_iterations: null - max_hours: null + max_minutes: null checkpoint: - save_iteration_interval: 100 + save_every: 100 ``` `latest.pt` is updated continuously; archive checkpoints are written every 100 -iterations. If disk is tight, prefer `--set checkpoint.save_latest_only=true` or -increase `save_iteration_interval`. +iterations. If disk is tight, set `--set checkpoint.save_every=0` (keep only +`latest.pt`) or increase `save_every`. ## Evaluation And Analysis diff --git a/configs/deep_cfr/deep_cfr_color_shared_512x3.yaml b/configs/deep_cfr/deep_cfr_color_shared_512x3.yaml index 1754052..5227752 100644 --- a/configs/deep_cfr/deep_cfr_color_shared_512x3.yaml +++ b/configs/deep_cfr/deep_cfr_color_shared_512x3.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_color_shared_512x3 - iterations: null seed: 42 max_iterations: 10000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -30,10 +29,10 @@ network: color_attention_heads: 4 traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -49,7 +48,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -71,9 +70,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 512 @@ -89,9 +87,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_color_shared_512x3 save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_color_shared_attention_512x3.yaml b/configs/deep_cfr/deep_cfr_color_shared_attention_512x3.yaml index 74b25ec..0487c68 100644 --- a/configs/deep_cfr/deep_cfr_color_shared_attention_512x3.yaml +++ b/configs/deep_cfr/deep_cfr_color_shared_attention_512x3.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_color_shared_attention_512x3 - iterations: null seed: 42 max_iterations: 10000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -30,10 +29,10 @@ network: color_attention_heads: 8 traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -49,7 +48,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -71,9 +70,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 512 @@ -89,9 +87,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_color_shared_attention_512x3 save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_color_shared_attention_exp_1000iter.yaml b/configs/deep_cfr/deep_cfr_color_shared_attention_exp_1000iter.yaml index 4e73f84..752c7b7 100644 --- a/configs/deep_cfr/deep_cfr_color_shared_attention_exp_1000iter.yaml +++ b/configs/deep_cfr/deep_cfr_color_shared_attention_exp_1000iter.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_color_shared_attention_1000iter - iterations: null seed: 42 max_iterations: 1000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -30,10 +29,10 @@ network: color_attention_heads: 8 traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -49,7 +48,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -71,9 +70,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 256 @@ -89,9 +87,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_color_shared_attention_exp_1000iter save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_opponent_average_strategy_512x3_1000iter.yaml b/configs/deep_cfr/deep_cfr_opponent_average_strategy_512x3_1000iter.yaml index d8d42a5..3045dad 100644 --- a/configs/deep_cfr/deep_cfr_opponent_average_strategy_512x3_1000iter.yaml +++ b/configs/deep_cfr/deep_cfr_opponent_average_strategy_512x3_1000iter.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_opponent_average_strategy_512x3_1000iter - iterations: null seed: 79 max_iterations: 1000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -27,10 +26,10 @@ network: activation: relu traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -46,7 +45,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -68,9 +67,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 512 @@ -86,9 +84,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_opponent_average_strategy_512x3_1000iter save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_opponent_network_1024x4_1000iter.yaml b/configs/deep_cfr/deep_cfr_opponent_network_1024x4_1000iter.yaml index 124f60e..39aed53 100644 --- a/configs/deep_cfr/deep_cfr_opponent_network_1024x4_1000iter.yaml +++ b/configs/deep_cfr/deep_cfr_opponent_network_1024x4_1000iter.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_opponent_network_1024x4_1000iter - iterations: null seed: 79 max_iterations: 1000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -27,10 +26,10 @@ network: activation: relu traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -46,7 +45,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -68,9 +67,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 512 @@ -86,9 +84,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_opponent_network_1024x4_1000iter save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_opponent_network_512x3_1000iter.yaml b/configs/deep_cfr/deep_cfr_opponent_network_512x3_1000iter.yaml index 78f6f73..9676101 100644 --- a/configs/deep_cfr/deep_cfr_opponent_network_512x3_1000iter.yaml +++ b/configs/deep_cfr/deep_cfr_opponent_network_512x3_1000iter.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_opponent_network_512x3_1000iter - iterations: null seed: 79 max_iterations: 1000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -27,10 +26,10 @@ network: activation: relu traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -46,7 +45,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -68,9 +67,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 512 @@ -86,9 +84,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_opponent_network_512x3_1000iter save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml b/configs/deep_cfr/deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml index f4e1436..3fde4b8 100644 --- a/configs/deep_cfr/deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml +++ b/configs/deep_cfr/deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter - iterations: null seed: 79 max_iterations: 10000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -27,10 +26,10 @@ network: activation: relu traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -46,7 +45,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -68,9 +67,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 512 @@ -86,9 +84,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_pure_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml b/configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml index 702d8ed..4501282 100644 --- a/configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml +++ b/configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter - iterations: null seed: 79 max_iterations: 10000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -27,10 +26,10 @@ network: activation: relu traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -46,7 +45,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -68,9 +67,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 512 @@ -86,9 +84,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml index 109eb89..10c350a 100644 --- a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml +++ b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability.yaml @@ -2,7 +2,7 @@ run: experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability seed: 79 max_iterations: null - max_hours: 4 + max_minutes: 240 device: cuda use_amp: false @@ -39,7 +39,7 @@ traversal: cutoff_rollout_max_steps: 300 progress_every_traversals: 10 num_workers: 8 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 outcome_sampling_value_clip: 500 @@ -87,7 +87,5 @@ evaluation: checkpoint: directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability - save_every_iteration: false - save_iteration_interval: 10 - save_latest_only: false + save_every: 10 progress_interval_seconds: 20.0 diff --git a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml index 1eedccf..bb44490 100644 --- a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml +++ b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter - iterations: null seed: 79 max_iterations: 10000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -27,10 +26,10 @@ network: activation: relu traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -46,7 +45,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -68,9 +67,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 512 @@ -86,9 +84,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_2x_updates_10000iter save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter.yaml b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter.yaml index c1dd3a9..1d3ff0d 100644 --- a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter.yaml +++ b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter - iterations: null seed: 79 max_iterations: 10000 - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -27,10 +26,10 @@ network: activation: relu traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -46,7 +45,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -69,9 +68,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 256 @@ -87,9 +85,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_lcfr_10000iter save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded.yaml b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded.yaml index e8e3def..fdd5919 100644 --- a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded.yaml +++ b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded - iterations: null seed: 79 max_iterations: null - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -27,10 +26,10 @@ network: activation: relu traversal: - traversals_per_iteration: 2 + traversals_per_player: 2 traversals_per_player: 70 max_depth: null - max_nodes: 10000 + max_nodes_per_traversal: 10000 max_nodes_per_traversal: 1000 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 @@ -46,7 +45,7 @@ traversal: store_strategy_on_opponent_nodes: false num_workers: 8 worker_chunk_size: 4 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 progress_every_traversals: 10 endpoint_depth_bucket_width: 100 endpoint_depth_bucket_max: 1000 @@ -68,9 +67,8 @@ self_play: recent_window: 5 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 32 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 advantage_batch_size: 1024 strategy_batch_size: 1024 advantage_updates_per_iteration: 256 @@ -86,9 +84,7 @@ memory: checkpoint: directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_512x3_unbounded save_latest: true - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 exact_resume: false diff --git a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_unbounded.yaml b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_unbounded.yaml index 941bec1..f17446c 100644 --- a/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_unbounded.yaml +++ b/configs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_unbounded.yaml @@ -1,9 +1,8 @@ run: experiment_name: lost_cities_deep_cfr_selfplay_full_depth_slot_playability_unbounded - iterations: null seed: 79 max_iterations: null - max_hours: null + max_minutes: null device: cuda use_amp: false @@ -40,7 +39,7 @@ traversal: cutoff_rollout_max_steps: 300 progress_every_traversals: 10 num_workers: 8 - traversal_worker_chunk_size: 8 + worker_chunk_size: 8 regret_matching_epsilon: 0.0001 outcome_sampling_epsilon: 0.2 outcome_sampling_value_clip: 500 @@ -88,7 +87,5 @@ evaluation: checkpoint: directory: runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability_unbounded - save_every_iteration: false - save_iteration_interval: 100 - save_latest_only: false + save_every: 100 progress_interval_seconds: 20.0 diff --git a/configs/deep_cfr/smoke.yaml b/configs/deep_cfr/smoke.yaml index ea1d77c..d3cfe74 100644 --- a/configs/deep_cfr/smoke.yaml +++ b/configs/deep_cfr/smoke.yaml @@ -1,5 +1,5 @@ run: - iterations: 1 + max_iterations: 1 seed: 1 device: cpu @@ -7,16 +7,17 @@ network: hidden_size: 16 traversal: - traversals_per_iteration: 1 + traversals_per_player: 1 max_depth: 2 - max_nodes: 64 + max_nodes_per_traversal: 64 num_workers: 0 worker_chunk_size: 1 optimization: - advantage_train_steps: 1 - strategy_train_steps: 1 - batch_size: 2 + advantage_batch_size: 2 + strategy_batch_size: 2 + advantage_updates_per_iteration: 1 + strategy_updates_per_iteration: 1 learning_rate: 0.001 memory: @@ -25,7 +26,8 @@ memory: checkpoint: directory: runs/deep_cfr/smoke - save_every_iteration: false + save_every: 0 + save_latest: false evaluation: eval_every: 0 diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/benchmark.py b/src/coolrl_lost_cities/games/classic/deep_cfr/benchmark.py index f26101f..529167a 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/benchmark.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/benchmark.py @@ -13,14 +13,15 @@ def benchmark_traversal( ) -> dict[str, float | int]: base = config or DeepCFRConfig.model_validate( { - "run": {"iterations": 1}, - "traversal": {"traversals_per_iteration": 8, "max_depth": 4}, - "checkpoint": {"save_every_iteration": False}, + "run": {"max_iterations": 1}, + "traversal": {"traversals_per_player": 8, "max_depth": 4}, + "checkpoint": {"save_every": 0, "save_latest": False}, } ) data = base.model_dump(mode="python") data["traversal"]["num_workers"] = num_workers - data["checkpoint"]["save_every_iteration"] = False + data["checkpoint"]["save_every"] = 0 + data["checkpoint"]["save_latest"] = False data["evaluation"]["eval_every"] = 0 cfg = DeepCFRConfig.model_validate(data) trainer = DeepCFRTrainer(cfg, cfg.rules.to_lost_cities_config(seed=cfg.run.seed)) diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py b/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py index 9c6aca5..485eff6 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py @@ -136,10 +136,10 @@ def benchmark_command(args: argparse.Namespace) -> None: { "run": {"seed": args.seed}, "traversal": { - "traversals_per_iteration": args.traversals, + "traversals_per_player": args.traversals, "max_depth": args.depth, }, - "checkpoint": {"save_every_iteration": False}, + "checkpoint": {"save_every": 0}, } ) if args.compare: @@ -212,7 +212,7 @@ def main(argv: list[str] | None = None) -> None: metavar="PATH=VALUE", help=( "Override config fields using dotted paths. Repeatable. VALUE is parsed as " - "YAML, e.g. --set traversal.num_workers=4 --set run.max_hours=null." + "YAML, e.g. --set traversal.num_workers=4 --set run.max_minutes=null." ), ) train.add_argument( diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/config.py b/src/coolrl_lost_cities/games/classic/deep_cfr/config.py index 263fc8b..7e33037 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/config.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/config.py @@ -18,23 +18,18 @@ class StrictModel(BaseModel): class RunConfig(StrictModel): experiment_name: str = "deep_cfr" - iterations: int | None = 1 max_iterations: int | None = None - max_hours: float | None = None + max_minutes: float | None = None seed: int = 1 - device: str = "cpu" + device: str = "auto" use_amp: bool = False @field_validator("device") @classmethod def _normalize_device(cls, value: str) -> str: token = value.strip().lower() - if token == "cuda": - return "cuda" - if token == "cpu": - return "cpu" - if token == "auto": - return "auto" + if token in {"cuda", "cpu", "auto"}: + return token return token @@ -95,12 +90,10 @@ class NetworkConfig(StrictModel): class TraversalConfig(StrictModel): - traversals_per_iteration: int = 2 - traversals_per_player: int | None = None + traversals_per_player: int = 8 sampling_mode: str = "outcome" - max_depth: int | None = 8 - max_nodes: int | None = 10_000 - max_nodes_per_traversal: int | None = None + max_depth: int | None = None + max_nodes_per_traversal: int | None = 10_000 regret_matching_epsilon: float = 1.0e-8 outcome_sampling_epsilon: float = 0.0 outcome_sampling_value_clip: float | None = None @@ -109,13 +102,12 @@ class TraversalConfig(StrictModel): cutoff_rollouts: int = 0 cutoff_rollout_policy: str = "random" cutoff_rollout_max_steps: int = 10_000 - opponent_policy: str = "network" + opponent_policy: str = "self_play_league" strategy_sample_interval: int = 1 store_strategy_on_traverser_nodes: bool = True store_strategy_on_opponent_nodes: bool = True num_workers: int | str = 0 worker_chunk_size: int = 4 - traversal_worker_chunk_size: int | None = None progress_every_traversals: int = 0 endpoint_depth_bucket_width: int = 100 endpoint_depth_bucket_max: int = 1000 @@ -169,21 +161,6 @@ class TraversalConfig(StrictModel): workers = max(0, int(self.num_workers)) return min(workers, batches) if batches is not None and batches > 0 else workers - def resolved_traversals_per_player(self) -> int: - if self.traversals_per_player is not None: - return max(0, int(self.traversals_per_player)) - return max(0, int(self.traversals_per_iteration)) - - def resolved_max_nodes(self) -> int | None: - if self.max_nodes_per_traversal is not None: - return self.max_nodes_per_traversal - return self.max_nodes - - def resolved_worker_chunk_size(self) -> int: - if self.traversal_worker_chunk_size is not None: - return max(1, int(self.traversal_worker_chunk_size)) - return max(1, int(self.worker_chunk_size)) - class RegretMatchingConfig(StrictModel): all_negative_fallback: str = "uniform" @@ -225,37 +202,14 @@ class SelfPlayLeagueConfig(StrictModel): class OptimizationConfig(StrictModel): - advantage_train_steps: int = 1 - strategy_train_steps: int = 1 - batch_size: int = 32 - advantage_batch_size: int | None = None - strategy_batch_size: int | None = None - advantage_updates_per_iteration: int | None = None - strategy_updates_per_iteration: int | None = None + advantage_batch_size: int = 256 + strategy_batch_size: int = 256 + advantage_updates_per_iteration: int = 64 + strategy_updates_per_iteration: int = 64 learning_rate: float = 1.0e-3 weight_decay: float = 0.0 grad_clip: float = 0.0 - def resolved_advantage_batch_size(self) -> int: - if self.advantage_batch_size is not None: - return max(1, int(self.advantage_batch_size)) - return max(1, int(self.batch_size)) - - def resolved_strategy_batch_size(self) -> int: - if self.strategy_batch_size is not None: - return max(1, int(self.strategy_batch_size)) - return max(1, int(self.batch_size)) - - def resolved_advantage_train_steps(self) -> int: - if self.advantage_updates_per_iteration is not None: - return max(0, int(self.advantage_updates_per_iteration)) - return max(0, int(self.advantage_train_steps)) - - def resolved_strategy_train_steps(self) -> int: - if self.strategy_updates_per_iteration is not None: - return max(0, int(self.strategy_updates_per_iteration)) - return max(0, int(self.strategy_train_steps)) - class MemoryConfig(StrictModel): advantage_capacity: int = 2_000_000 @@ -264,10 +218,8 @@ class MemoryConfig(StrictModel): class CheckpointConfig(StrictModel): directory: str = "runs/deep_cfr/default" + save_every: int = 1 save_latest: bool = True - save_every_iteration: bool = True - save_iteration_interval: int = 0 - save_latest_only: bool = False progress_interval_seconds: float = 20.0 exact_resume: bool = False @@ -277,9 +229,9 @@ class CheckpointConfig(StrictModel): class EvaluationConfig(StrictModel): - eval_every: int = 0 + eval_every: int = 50 games: int = 10 - opponents: tuple[str, ...] = ("random",) + opponents: tuple[str, ...] = ("random", "safe_heuristic") max_steps: int = 10_000 on_max_steps: str = "score_diff" batch_size: int = 64 @@ -302,9 +254,6 @@ class EvaluationConfig(StrictModel): raise ValueError("must be 'trainer', 'auto', 'cpu', or 'cuda'") return token - def resolved_batch_size(self) -> int: - return max(1, int(self.batch_size)) - def resolved_num_workers(self, opponent_count: int | None = None) -> int: workers = max(1, int(self.num_workers)) if opponent_count is not None: diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py index 9ea9db8..6af1d8b 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py @@ -329,7 +329,7 @@ class DeepCFRTrainer: progress_every = int(self.config.traversal.progress_every_traversals) completed = 0 progress_started = time.perf_counter() - traversals_per_player = self.config.traversal.resolved_traversals_per_player() + traversals_per_player = self.config.traversal.traversals_per_player for player in range(2): seeds = [ self.config.run.seed + iteration * 10_000 + traversal_index * 10 + player @@ -358,7 +358,7 @@ class DeepCFRTrainer: self.config.traversal.store_strategy_on_opponent_nodes ), max_depth=self.config.traversal.max_depth, - max_nodes=self.config.traversal.resolved_max_nodes(), + max_nodes=self.config.traversal.max_nodes_per_traversal, sampling_mode=self.config.traversal.sampling_mode, outcome_sampling_epsilon=self.config.traversal.outcome_sampling_epsilon, outcome_sampling_value_clip=self.config.traversal.outcome_sampling_value_clip, @@ -409,7 +409,7 @@ class DeepCFRTrainer: self.tracker.log_event( f"Traversal multiprocessing enabled iteration={iteration} " f"requested_workers={requested_workers} effective_workers={max_workers} " - f"batches={len(batches)} chunk_size={self.config.traversal.resolved_worker_chunk_size()}" + f"batches={len(batches)} chunk_size={self.config.traversal.worker_chunk_size}" ) if max_workers < requested_workers: self.tracker.log_event( @@ -481,12 +481,12 @@ class DeepCFRTrainer: name: value.detach().cpu() for name, value in self.strategy_network.state_dict().items() } - chunk_size = self.config.traversal.resolved_worker_chunk_size() + chunk_size = self.config.traversal.worker_chunk_size batch_index = 0 for player in range(2): seeds = [ self.config.run.seed + iteration * 10_000 + index * 10 + player - for index in range(self.config.traversal.resolved_traversals_per_player()) + for index in range(self.config.traversal.traversals_per_player) ] for start in range(0, len(seeds), chunk_size): chunk = seeds[start : start + chunk_size] @@ -574,27 +574,21 @@ class DeepCFRTrainer: def _stop_iteration(self) -> int: if self.config.run.max_iterations is not None: return max(self.iteration, int(self.config.run.max_iterations)) - if self.config.run.max_hours is not None: - return 2**31 - 1 - if self.config.run.iterations is None: - return 2**31 - 1 - return self.iteration + self.config.run.iterations + return 2**31 - 1 def _time_limit_reached(self, run_started: float) -> bool: - if self.config.run.max_hours is None: + if self.config.run.max_minutes is None: return False - elapsed_hours = (time.perf_counter() - run_started) / 3600.0 - return elapsed_hours >= self.config.run.max_hours + elapsed_minutes = (time.perf_counter() - run_started) / 60.0 + return elapsed_minutes >= self.config.run.max_minutes def _should_save_iteration(self, iteration: int) -> bool: - if self.config.checkpoint.save_every_iteration: - return True - interval = int(self.config.checkpoint.save_iteration_interval) + interval = int(self.config.checkpoint.save_every) return interval > 0 and iteration % interval == 0 def _save_iteration_checkpoints(self, iteration: int, item: IterationMetrics) -> None: checkpoint_dir = self.run_dir - if self._should_save_iteration(iteration) and not self.config.checkpoint.save_latest_only: + if self._should_save_iteration(iteration): self.save_checkpoint(checkpoint_dir / f"iteration_{iteration:05d}.pt", item) if self.config.checkpoint.save_latest: self.save_checkpoint(checkpoint_dir / "latest.pt", item) @@ -641,7 +635,7 @@ class DeepCFRTrainer: device=eval_device, max_steps=self.config.evaluation.max_steps, encoding=self.config.encoding, - batch_size=self.config.evaluation.resolved_batch_size(), + batch_size=self.config.evaluation.batch_size, ) for key, value in result.items(): results[f"eval_{opponent}_{key}"] = value @@ -657,7 +651,7 @@ class DeepCFRTrainer: self.tracker.log_event( f"Evaluation multiprocessing enabled iteration={iteration} " f"effective_workers={max_workers} opponents={len(opponents)} " - f"batch_size={self.config.evaluation.resolved_batch_size()} device={eval_device}" + f"batch_size={self.config.evaluation.batch_size} device={eval_device}" ) state_dict = self._strategy_state_dict_cpu() jobs = [ @@ -673,7 +667,7 @@ class DeepCFRTrainer: seed=self.config.run.seed + iteration * 1000, device=str(eval_device), max_steps=self.config.evaluation.max_steps, - batch_size=self.config.evaluation.resolved_batch_size(), + batch_size=self.config.evaluation.batch_size, ) for opponent in opponents ] @@ -773,10 +767,10 @@ class DeepCFRTrainer: ) -> float: losses: list[float] = [] network.train() - for _step in range(self.config.optimization.resolved_advantage_train_steps()): + for _step in range(self.config.optimization.advantage_updates_per_iteration): sample_started = time.perf_counter() batch = self.advantage_memories[player].sample( - self.config.optimization.resolved_advantage_batch_size(), + self.config.optimization.advantage_batch_size, self.rng, ) self._runtime_metrics[f"advantage_player_{player}_sample_seconds"] = ( @@ -828,10 +822,10 @@ class DeepCFRTrainer: ) -> float: last_loss = 0.0 network.train() - for _step in range(self.config.optimization.resolved_strategy_train_steps()): + for _step in range(self.config.optimization.strategy_updates_per_iteration): sample_started = time.perf_counter() batch = self.strategy_memory.sample( - self.config.optimization.resolved_strategy_batch_size(), self.rng + self.config.optimization.strategy_batch_size, self.rng ) self._runtime_metrics["strategy_sample_seconds"] = ( float(self._runtime_metrics.get("strategy_sample_seconds", 0.0)) diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py b/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py index 4f94407..634b1b0 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/workers.py @@ -100,7 +100,7 @@ def run_traversal_worker_batch(batch: TraversalWorkerBatch) -> TraversalWorkerRe store_strategy_on_traverser_nodes=cfg.traversal.store_strategy_on_traverser_nodes, store_strategy_on_opponent_nodes=cfg.traversal.store_strategy_on_opponent_nodes, max_depth=cfg.traversal.max_depth, - max_nodes=cfg.traversal.resolved_max_nodes(), + max_nodes=cfg.traversal.max_nodes_per_traversal, sampling_mode=cfg.traversal.sampling_mode, outcome_sampling_epsilon=cfg.traversal.outcome_sampling_epsilon, outcome_sampling_value_clip=cfg.traversal.outcome_sampling_value_clip, diff --git a/tests/games/classic/test_deep_cfr_trainer.py b/tests/games/classic/test_deep_cfr_trainer.py index 8b1ad57..0f4aaef 100644 --- a/tests/games/classic/test_deep_cfr_trainer.py +++ b/tests/games/classic/test_deep_cfr_trainer.py @@ -33,9 +33,9 @@ def _deep_cfr_config(data: dict) -> DeepCFRConfig: def test_deep_cfr_loads_smoke_yaml_config() -> None: config = load_config("configs/deep_cfr/smoke.yaml") - assert config.run.iterations == 1 + assert config.run.max_iterations == 1 assert config.network.hidden_size == 16 - assert config.traversal.traversals_per_iteration == 1 + assert config.traversal.traversals_per_player == 1 assert config.checkpoint.directory == "runs/deep_cfr/smoke" @@ -45,30 +45,30 @@ def test_deep_cfr_loads_mapped_legacy_reproduction_config() -> None: assert config.run.experiment_name.endswith("slot_playability") assert config.run.seed == 79 assert config.run.max_iterations is None - assert config.run.max_hours == 4 + assert config.run.max_minutes == 240 assert config.encoding.derived_playability is True assert config.encoding.slot_aware_playability is True assert config.network.hidden_size == 256 assert config.network.num_layers == 3 - assert config.traversal.resolved_traversals_per_player() == 70 + assert config.traversal.traversals_per_player == 70 assert config.traversal.sampling_mode == "outcome" assert config.traversal.max_depth is None - assert config.traversal.resolved_max_nodes() == 1000 - assert config.traversal.resolved_worker_chunk_size() == 8 + assert config.traversal.max_nodes_per_traversal == 1000 + assert config.traversal.worker_chunk_size == 8 assert config.traversal.progress_every_traversals == 10 - assert config.optimization.resolved_advantage_batch_size() == 1024 - assert config.optimization.resolved_strategy_batch_size() == 1024 - assert config.optimization.resolved_advantage_train_steps() == 256 - assert config.optimization.resolved_strategy_train_steps() == 256 + assert config.optimization.advantage_batch_size == 1024 + assert config.optimization.strategy_batch_size == 1024 + assert config.optimization.advantage_updates_per_iteration == 256 + assert config.optimization.strategy_updates_per_iteration == 256 assert config.optimization.weight_decay == 0.0001 assert config.optimization.grad_clip == 1.0 assert config.evaluation.on_max_steps == "score_diff" - assert config.evaluation.resolved_batch_size() == 64 + assert config.evaluation.batch_size == 64 assert config.evaluation.device == "trainer" assert config.evaluation.resolved_num_workers() == 4 assert config.regret_matching.all_negative_fallback == "uniform" assert config.training_weighting.mode == "none" - assert config.checkpoint.save_iteration_interval == 10 + assert config.checkpoint.save_every == 10 assert ( config.checkpoint.directory == "runs/deep_cfr/deep_cfr_selfplay_full_depth_slot_playability" ) @@ -80,17 +80,14 @@ def test_deep_cfr_train_cli_accepts_run_and_traversal_config_overrides() -> None (), { "config_overrides": [ - "run.iterations=1", - "run.max_hours=null", - "run.max_iterations=null", - "traversal.traversals_per_iteration=1", - "traversal.traversals_per_player=null", + "run.max_iterations=1", + "run.max_minutes=null", + "traversal.traversals_per_player=1", "traversal.num_workers=0", "regret_matching.all_negative_fallback=argmax_tiebreak", "training_weighting.mode=lcfr", "checkpoint.save_latest=false", - "checkpoint.save_every_iteration=false", - "checkpoint.save_iteration_interval=0", + "checkpoint.save_every=0", ], }, )() @@ -98,15 +95,13 @@ def test_deep_cfr_train_cli_accepts_run_and_traversal_config_overrides() -> None overridden = _with_overrides(config, _train_overrides_from_args(args)) - assert overridden.run.iterations == 1 - assert overridden.run.max_hours is None - assert overridden.run.max_iterations is None - assert overridden.traversal.traversals_per_player is None - assert overridden.traversal.resolved_traversals_per_player() == 1 + assert overridden.run.max_iterations == 1 + assert overridden.run.max_minutes is None + assert overridden.traversal.traversals_per_player == 1 assert overridden.traversal.resolved_num_workers() == 0 assert overridden.regret_matching.all_negative_fallback == "argmax_tiebreak" assert overridden.training_weighting.mode == "lcfr" - assert overridden.checkpoint.save_every_iteration is False + assert overridden.checkpoint.save_every == 0 assert overridden.checkpoint.save_latest is False @@ -123,9 +118,7 @@ def test_deep_cfr_train_cli_checkpoint_save_overrides() -> None: { "config_overrides": [ "checkpoint.save_latest=true", - "checkpoint.save_latest_only=true", - "checkpoint.save_every_iteration=false", - "checkpoint.save_iteration_interval=1", + "checkpoint.save_every=1", ], }, )() @@ -133,9 +126,7 @@ def test_deep_cfr_train_cli_checkpoint_save_overrides() -> None: overridden = _with_overrides(DeepCFRConfig(), _train_overrides_from_args(args)) assert overridden.checkpoint.save_latest is True - assert overridden.checkpoint.save_latest_only is True - assert overridden.checkpoint.save_every_iteration is False - assert overridden.checkpoint.save_iteration_interval == 1 + assert overridden.checkpoint.save_every == 1 def test_deep_cfr_train_cli_accepts_generic_config_overrides() -> None: @@ -146,8 +137,8 @@ def test_deep_cfr_train_cli_accepts_generic_config_overrides() -> None: "config_overrides": [ "traversal.sampling_mode=external", "traversal.max_depth=null", - "optimization.batch_size=64", - "checkpoint.save_latest_only=true", + "optimization.advantage_batch_size=64", + "checkpoint.save_latest=true", ], }, )() @@ -156,17 +147,17 @@ def test_deep_cfr_train_cli_accepts_generic_config_overrides() -> None: assert overridden.traversal.sampling_mode == "external" assert overridden.traversal.max_depth is None - assert overridden.optimization.batch_size == 64 - assert overridden.checkpoint.save_latest_only is True + assert overridden.optimization.advantage_batch_size == 64 + assert overridden.checkpoint.save_latest is True def test_deep_cfr_iteration_weights_use_sample_age() -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 12}, + "run": {"max_iterations": 1, "seed": 12}, "network": {"hidden_size": 16}, - "checkpoint": {"save_every_iteration": False}, + "checkpoint": {"save_every": 0}, "training_weighting": {"mode": "lcfr", "lcfr_alpha": 1.0}, } ), @@ -279,16 +270,21 @@ def test_deep_cfr_slot_aware_playability_encoding_zero_fills_empty_hand_slots() def test_deep_cfr_trainer_uses_playability_encoding() -> None: config = _deep_cfr_config( { - "run": {"iterations": 1, "seed": 62}, + "run": {"max_iterations": 1, "seed": 62}, "encoding": {"derived_playability": True, "slot_aware_playability": True}, "network": {"hidden_size": 16}, - "traversal": {"traversals_per_iteration": 1, "max_depth": 1, "max_nodes": 16}, - "optimization": { - "advantage_train_steps": 1, - "strategy_train_steps": 1, - "batch_size": 2, + "traversal": { + "traversals_per_player": 1, + "max_depth": 1, + "max_nodes_per_traversal": 16, }, - "checkpoint": {"save_every_iteration": False}, + "optimization": { + "advantage_updates_per_iteration": 1, + "strategy_updates_per_iteration": 1, + "advantage_batch_size": 2, + "strategy_batch_size": 2, + }, + "checkpoint": {"save_every": 0}, } ) game_config = LostCitiesConfig(seed=62) @@ -317,17 +313,17 @@ def test_deep_cfr_trainer_forwards_metrics_to_extra_trackers(tmp_path) -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 71}, + "run": {"max_iterations": 1, "seed": 71}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "max_depth": 2, - "max_nodes": 16, + "max_nodes_per_traversal": 16, }, - "optimization": {"batch_size": 2}, + "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2}, "checkpoint": { "directory": str(tmp_path / "extra-tracker"), - "save_every_iteration": False, + "save_every": 0, }, } ), @@ -347,19 +343,20 @@ def test_deep_cfr_trainer_smoke_run() -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 23}, + "run": {"max_iterations": 1, "seed": 23}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "max_depth": 3, - "max_nodes": 64, + "max_nodes_per_traversal": 64, }, "optimization": { - "advantage_train_steps": 1, - "strategy_train_steps": 1, - "batch_size": 2, + "advantage_updates_per_iteration": 1, + "strategy_updates_per_iteration": 1, + "advantage_batch_size": 2, + "strategy_batch_size": 2, }, - "checkpoint": {"save_every_iteration": False}, + "checkpoint": {"save_every": 0}, } ), LostCitiesConfig(seed=23), @@ -383,20 +380,21 @@ def test_deep_cfr_trainer_supports_lcfr_and_dcfr_loss_weighting() -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 24}, + "run": {"max_iterations": 1, "seed": 24}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "max_depth": 2, - "max_nodes": 32, + "max_nodes_per_traversal": 32, }, "optimization": { - "advantage_train_steps": 1, - "strategy_train_steps": 1, - "batch_size": 2, + "advantage_updates_per_iteration": 1, + "strategy_updates_per_iteration": 1, + "advantage_batch_size": 2, + "strategy_batch_size": 2, }, "training_weighting": {"mode": mode}, - "checkpoint": {"save_every_iteration": False}, + "checkpoint": {"save_every": 0}, } ), LostCitiesConfig(seed=24), @@ -413,15 +411,15 @@ def test_deep_cfr_cython_traverser_restores_state_and_collects_samples() -> None trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 29}, + "run": {"max_iterations": 1, "seed": 29}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "max_depth": 2, - "max_nodes": 32, + "max_nodes_per_traversal": 32, }, - "optimization": {"batch_size": 2}, - "checkpoint": {"save_every_iteration": False}, + "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2}, + "checkpoint": {"save_every": 0}, } ), LostCitiesConfig(seed=29), @@ -457,12 +455,12 @@ def test_deep_cfr_cython_traverser_supports_outcome_sampling_and_rollout_cutoffs trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 31}, + "run": {"max_iterations": 1, "seed": 31}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "max_depth": 1, - "max_nodes": 32, + "max_nodes_per_traversal": 32, "outcome_sampling_epsilon": 0.25, "outcome_sampling_value_clip": 100.0, "outcome_unsampled_regret": "zero", @@ -471,8 +469,8 @@ def test_deep_cfr_cython_traverser_supports_outcome_sampling_and_rollout_cutoffs "cutoff_rollout_policy": "random", "cutoff_rollout_max_steps": 16, }, - "optimization": {"batch_size": 2}, - "checkpoint": {"save_every_iteration": False}, + "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2}, + "checkpoint": {"save_every": 0}, } ), LostCitiesConfig(seed=31), @@ -512,16 +510,16 @@ def test_deep_cfr_cython_traverser_supports_external_sampling() -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 33}, + "run": {"max_iterations": 1, "seed": 33}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "sampling_mode": "external", "max_depth": 1, - "max_nodes": 64, + "max_nodes_per_traversal": 64, }, - "optimization": {"batch_size": 2}, - "checkpoint": {"save_every_iteration": False}, + "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2}, + "checkpoint": {"save_every": 0}, } ), LostCitiesConfig(seed=33), @@ -563,11 +561,11 @@ def test_deep_cfr_cython_traverser_records_regret_fallback_metrics() -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 37}, + "run": {"max_iterations": 1, "seed": 37}, "network": {"hidden_size": 16}, - "traversal": {"traversals_per_iteration": 1, "max_depth": 1}, - "optimization": {"batch_size": 2}, - "checkpoint": {"save_every_iteration": False}, + "traversal": {"traversals_per_player": 1, "max_depth": 1}, + "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2}, + "checkpoint": {"save_every": 0}, "regret_matching": {"all_negative_fallback": "argmax_tiebreak"}, } ), @@ -637,17 +635,17 @@ def test_deep_cfr_trainer_saves_loads_and_evaluates_checkpoint(tmp_path) -> None trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 41}, + "run": {"max_iterations": 1, "seed": 41}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "max_depth": 2, - "max_nodes": 32, + "max_nodes_per_traversal": 32, }, - "optimization": {"batch_size": 2}, + "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2}, "checkpoint": { "directory": str(checkpoint_dir), - "save_every_iteration": True, + "save_every": 1, }, "evaluation": {"eval_every": 1, "games": 2, "opponents": ("random",)}, } @@ -664,7 +662,7 @@ def test_deep_cfr_trainer_saves_loads_and_evaluates_checkpoint(tmp_path) -> None "network": {"hidden_size": 16}, "checkpoint": { "directory": str(checkpoint_dir), - "save_every_iteration": False, + "save_every": 0, }, } ), @@ -700,13 +698,12 @@ def test_deep_cfr_trainer_always_saves_latest_checkpoint(tmp_path) -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 42}, + "run": {"max_iterations": 1, "seed": 42}, "network": {"hidden_size": 16}, - "traversal": {"traversals_per_iteration": 1, "max_depth": 1}, + "traversal": {"traversals_per_player": 1, "max_depth": 1}, "checkpoint": { "directory": str(checkpoint_dir), - "save_every_iteration": False, - "save_iteration_interval": 10, + "save_every": 10, }, } ), @@ -724,10 +721,10 @@ def test_deep_cfr_exact_resume_is_explicitly_not_implemented(tmp_path) -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 44}, + "run": {"max_iterations": 1, "seed": 44}, "network": {"hidden_size": 16}, - "traversal": {"traversals_per_iteration": 1, "max_depth": 1}, - "checkpoint": {"directory": str(checkpoint_dir), "save_every_iteration": True}, + "traversal": {"traversals_per_player": 1, "max_depth": 1}, + "checkpoint": {"directory": str(checkpoint_dir), "save_every": 1}, } ), LostCitiesConfig(seed=44), @@ -755,20 +752,20 @@ def test_deep_cfr_trainer_multiprocessing_smoke_run(tmp_path) -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 1, "seed": 43}, + "run": {"max_iterations": 1, "seed": 43}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "max_depth": 2, - "max_nodes": 32, + "max_nodes_per_traversal": 32, "num_workers": 8, "worker_chunk_size": 1, "progress_every_traversals": 1, }, - "optimization": {"batch_size": 2}, + "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2}, "checkpoint": { "directory": str(tmp_path / "mp"), - "save_every_iteration": False, + "save_every": 0, }, } ), @@ -791,8 +788,8 @@ def test_deep_cfr_traversal_benchmark_smoke() -> None: { "run": {"seed": 47}, "network": {"hidden_size": 16}, - "traversal": {"traversals_per_iteration": 1, "max_depth": 2}, - "checkpoint": {"save_every_iteration": False}, + "traversal": {"traversals_per_player": 1, "max_depth": 2}, + "checkpoint": {"save_every": 0}, } ) ) @@ -804,8 +801,8 @@ def test_deep_cfr_traversal_benchmark_smoke() -> None: { "run": {"seed": 48}, "network": {"hidden_size": 16}, - "traversal": {"traversals_per_iteration": 1, "max_depth": 2}, - "checkpoint": {"save_every_iteration": False}, + "traversal": {"traversals_per_player": 1, "max_depth": 2}, + "checkpoint": {"save_every": 0}, } ) ) @@ -816,12 +813,12 @@ def test_deep_cfr_self_play_league_records_snapshots(tmp_path) -> None: trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 2, "seed": 53}, + "run": {"max_iterations": 2, "seed": 53}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "max_depth": 2, - "max_nodes": 32, + "max_nodes_per_traversal": 32, "opponent_policy": "self_play_league", }, "self_play": { @@ -829,10 +826,10 @@ def test_deep_cfr_self_play_league_records_snapshots(tmp_path) -> None: "max_snapshots": 1, "anchor_probability": 1.0, }, - "optimization": {"batch_size": 2}, + "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2}, "checkpoint": { "directory": str(tmp_path / "league"), - "save_every_iteration": False, + "save_every": 0, }, } ), @@ -849,12 +846,12 @@ def test_deep_cfr_weighted_self_play_league_uses_snapshot_bucket(tmp_path) -> No trainer = DeepCFRTrainer( _deep_cfr_config( { - "run": {"iterations": 2, "seed": 59}, + "run": {"max_iterations": 2, "seed": 59}, "network": {"hidden_size": 16}, "traversal": { - "traversals_per_iteration": 1, + "traversals_per_player": 1, "max_depth": 2, - "max_nodes": 32, + "max_nodes_per_traversal": 32, "opponent_policy": "self_play_league", }, "self_play": { @@ -866,10 +863,10 @@ def test_deep_cfr_weighted_self_play_league_uses_snapshot_bucket(tmp_path) -> No "anchor_weight": 0.0, "recent_window": 1, }, - "optimization": {"batch_size": 2}, + "optimization": {"advantage_batch_size": 2, "strategy_batch_size": 2}, "checkpoint": { "directory": str(tmp_path / "weighted-league"), - "save_every_iteration": False, + "save_every": 0, }, } ),