From fd99d3bb4a909db4e8b6f43b00b499e1138a82fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=8B=9C=EC=9B=90?= Date: Thu, 7 May 2026 09:14:23 +0900 Subject: [PATCH] =?UTF-8?q?Deep=20CFR=20self-play=20anchor=20safe=20512x3?= =?UTF-8?q?=202x=20updates=2010000=20iter=20config=20=EB=B0=8F=20=EA=B4=80?= =?UTF-8?q?=EB=A0=A8=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Haiku 4.5 --- ...nchor_safe_512x3_2x_updates_10000iter.yaml | 109 ++++++++++++++++++ .../games/classic/deep_cfr/analyze.py | 41 ++++++- .../games/classic/deep_cfr/cli.py | 7 +- 3 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml diff --git a/configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml b/configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml new file mode 100644 index 0000000..702d8ed --- /dev/null +++ b/configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml @@ -0,0 +1,109 @@ +run: + experiment_name: lost_cities_deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter + iterations: null + seed: 79 + max_iterations: 10000 + max_hours: null + device: cuda + use_amp: false + +rules: + n_colors: 5 + n_ranks: 9 + min_rank: 2 + n_handshakes: 3 + hand_size: 8 + expedition_penalty: -20 + bonus_threshold: 8 + bonus_amount: 20 + +encoding: + derived_playability: true + slot_aware_playability: true + +network: + hidden_size: 512 + num_layers: 3 + activation: relu + +traversal: + traversals_per_iteration: 2 + traversals_per_player: 70 + max_depth: null + max_nodes: 10000 + max_nodes_per_traversal: 1000 + regret_matching_epsilon: 0.0001 + outcome_sampling_epsilon: 0.2 + outcome_sampling_value_clip: 500.0 + outcome_unsampled_regret: zero + cutoff_value_mode: score_diff + cutoff_rollouts: 0 + cutoff_rollout_policy: random + cutoff_rollout_max_steps: 300 + opponent_policy: self_play_league + strategy_sample_interval: 1 + store_strategy_on_traverser_nodes: true + store_strategy_on_opponent_nodes: false + num_workers: 8 + worker_chunk_size: 4 + traversal_worker_chunk_size: 8 + progress_every_traversals: 10 + endpoint_depth_bucket_width: 100 + endpoint_depth_bucket_max: 1000 + +regret_matching: + all_negative_fallback: argmax_tiebreak + +training_weighting: + mode: none + +self_play: + snapshot_every: 1 + max_snapshots: 20 + anchor_probability: 0.0 + current_weight: 0.45 + recent_weight: 0.30 + older_weight: 0.15 + anchor_weight: 0.10 + recent_window: 5 + +optimization: + advantage_train_steps: 1 + strategy_train_steps: 1 + batch_size: 32 + advantage_batch_size: 1024 + strategy_batch_size: 1024 + advantage_updates_per_iteration: 512 + strategy_updates_per_iteration: 512 + learning_rate: 0.00003 + weight_decay: 0.0001 + grad_clip: 1.0 + +memory: + advantage_capacity: 2000000 + strategy_capacity: 2000000 + +checkpoint: + directory: runs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter + save_latest: true + save_every_iteration: false + save_iteration_interval: 100 + save_latest_only: false + progress_interval_seconds: 20.0 + exact_resume: false + +evaluation: + eval_every: 5 + games: 100 + opponents: + - random + - passive_discard + - safe_heuristic + - safe_heuristic_loose + - safe_heuristic_strict + - noisy_safe + max_steps: 10000 + on_max_steps: score_diff + batch_size: 64 + device: trainer + num_workers: 4 diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/analyze.py b/src/coolrl_lost_cities/games/classic/deep_cfr/analyze.py index a4baab9..d9130c3 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/analyze.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/analyze.py @@ -523,28 +523,51 @@ def analyze_run( output_dir: Path | None = None, *, smoothing_window: int = DEFAULT_SMOOTHING_WINDOW, + max_iteration: int | None = None, ) -> list[Path]: metrics_path = run_dir / "metrics.jsonl" rows = load_metrics(metrics_path) + if max_iteration is not None: + rows = [ + row for row in rows if "iteration" in row and int(row["iteration"]) <= max_iteration + ] output_dir = output_dir or run_dir output_dir.mkdir(parents=True, exist_ok=True) written: list[Path] = [] + filename_suffix = _iteration_filename_suffix(max_iteration) for section in SECTIONS: - path = output_dir / section.filename + path = output_dir / _with_filename_suffix(section.filename, filename_suffix) if plot_section(rows, section, path, smoothing_window=smoothing_window): written.append(path) - selectivity_path = output_dir / "analysis_09_selectivity.png" + selectivity_path = output_dir / _with_filename_suffix( + "analysis_09_selectivity.png", filename_suffix + ) if plot_selectivity(rows, selectivity_path, smoothing_window=smoothing_window): written.append(selectivity_path) - final_eval_path = output_dir / "analysis_final_eval_summary.png" + final_eval_path = output_dir / _with_filename_suffix( + "analysis_final_eval_summary.png", filename_suffix + ) if plot_final_eval_summary(rows, final_eval_path): written.append(final_eval_path) return written +def _iteration_filename_suffix(max_iteration: int | None) -> str: + if max_iteration is None: + return "" + return f"_upto_{max_iteration:05d}" + + +def _with_filename_suffix(filename: str, suffix: str) -> str: + if not suffix: + return filename + path = Path(filename) + return f"{path.stem}{suffix}{path.suffix}" + + def plot_selectivity( rows: list[dict[str, Any]], output: Path, @@ -979,9 +1002,19 @@ def main(argv: list[str] | None = None) -> None: action="store_true", help="Disable moving-average smoothing.", ) + parser.add_argument( + "--max-iteration", + type=int, + help="Only plot metrics up to and including this iteration.", + ) args = parser.parse_args(argv) smoothing_window = 1 if args.no_smoothing else max(1, args.smoothing_window) - written = analyze_run(args.run, args.output_dir, smoothing_window=smoothing_window) + written = analyze_run( + args.run, + args.output_dir, + smoothing_window=smoothing_window, + max_iteration=args.max_iteration, + ) for path in written: print(path) diff --git a/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py b/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py index 527a37a..e7ea62d 100644 --- a/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py +++ b/src/coolrl_lost_cities/games/classic/deep_cfr/cli.py @@ -200,7 +200,7 @@ def policy_gradient_command(args: argparse.Namespace) -> None: def analyze_command(args: argparse.Namespace) -> None: - written = analyze_run(args.run, args.output_dir) + written = analyze_run(args.run, args.output_dir, max_iteration=args.max_iteration) for path in written: print(path) @@ -277,6 +277,11 @@ def main(argv: list[str] | None = None) -> None: analyze = subparsers.add_parser("analyze") analyze.add_argument("--run", required=True, type=Path) analyze.add_argument("--output-dir", type=Path) + analyze.add_argument( + "--max-iteration", + type=int, + help="Only plot metrics up to and including this iteration.", + ) analyze.set_defaults(func=analyze_command) args = parser.parse_args(argv)