Deep CFR self-play anchor safe 512x3 2x updates 10000 iter config 및 관련 변경

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>

configs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter.yaml

@@ -0,0 +1,109 @@
+run:
+  experiment_name: lost_cities_deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter
+  iterations: null
+  seed: 79
+  max_iterations: 10000
+  max_hours: null
+  device: cuda
+  use_amp: false
+
+rules:
+  n_colors: 5
+  n_ranks: 9
+  min_rank: 2
+  n_handshakes: 3
+  hand_size: 8
+  expedition_penalty: -20
+  bonus_threshold: 8
+  bonus_amount: 20
+
+encoding:
+  derived_playability: true
+  slot_aware_playability: true
+
+network:
+  hidden_size: 512
+  num_layers: 3
+  activation: relu
+
+traversal:
+  traversals_per_iteration: 2
+  traversals_per_player: 70
+  max_depth: null
+  max_nodes: 10000
+  max_nodes_per_traversal: 1000
+  regret_matching_epsilon: 0.0001
+  outcome_sampling_epsilon: 0.2
+  outcome_sampling_value_clip: 500.0
+  outcome_unsampled_regret: zero
+  cutoff_value_mode: score_diff
+  cutoff_rollouts: 0
+  cutoff_rollout_policy: random
+  cutoff_rollout_max_steps: 300
+  opponent_policy: self_play_league
+  strategy_sample_interval: 1
+  store_strategy_on_traverser_nodes: true
+  store_strategy_on_opponent_nodes: false
+  num_workers: 8
+  worker_chunk_size: 4
+  traversal_worker_chunk_size: 8
+  progress_every_traversals: 10
+  endpoint_depth_bucket_width: 100
+  endpoint_depth_bucket_max: 1000
+
+regret_matching:
+  all_negative_fallback: argmax_tiebreak
+
+training_weighting:
+  mode: none
+
+self_play:
+  snapshot_every: 1
+  max_snapshots: 20
+  anchor_probability: 0.0
+  current_weight: 0.45
+  recent_weight: 0.30
+  older_weight: 0.15
+  anchor_weight: 0.10
+  recent_window: 5
+
+optimization:
+  advantage_train_steps: 1
+  strategy_train_steps: 1
+  batch_size: 32
+  advantage_batch_size: 1024
+  strategy_batch_size: 1024
+  advantage_updates_per_iteration: 512
+  strategy_updates_per_iteration: 512
+  learning_rate: 0.00003
+  weight_decay: 0.0001
+  grad_clip: 1.0
+
+memory:
+  advantage_capacity: 2000000
+  strategy_capacity: 2000000
+
+checkpoint:
+  directory: runs/deep_cfr/deep_cfr_selfplay_anchor_safe_512x3_2x_updates_10000iter
+  save_latest: true
+  save_every_iteration: false
+  save_iteration_interval: 100
+  save_latest_only: false
+  progress_interval_seconds: 20.0
+  exact_resume: false
+
+evaluation:
+  eval_every: 5
+  games: 100
+  opponents:
+    - random
+    - passive_discard
+    - safe_heuristic
+    - safe_heuristic_loose
+    - safe_heuristic_strict
+    - noisy_safe
+  max_steps: 10000
+  on_max_steps: score_diff
+  batch_size: 64
+  device: trainer
+  num_workers: 4

src/coolrl_lost_cities/games/classic/deep_cfr/analyze.py

@@ -523,28 +523,51 @@ def analyze_run(
     output_dir: Path | None = None,
     *,
     smoothing_window: int = DEFAULT_SMOOTHING_WINDOW,
+    max_iteration: int | None = None,
 ) -> list[Path]:
     metrics_path = run_dir / "metrics.jsonl"
     rows = load_metrics(metrics_path)
+    if max_iteration is not None:
+        rows = [
+            row for row in rows if "iteration" in row and int(row["iteration"]) <= max_iteration
+        ]
     output_dir = output_dir or run_dir
     output_dir.mkdir(parents=True, exist_ok=True)
     written: list[Path] = []
+    filename_suffix = _iteration_filename_suffix(max_iteration)
 
     for section in SECTIONS:
-        path = output_dir / section.filename
+        path = output_dir / _with_filename_suffix(section.filename, filename_suffix)
         if plot_section(rows, section, path, smoothing_window=smoothing_window):
             written.append(path)
 
-    selectivity_path = output_dir / "analysis_09_selectivity.png"
+    selectivity_path = output_dir / _with_filename_suffix(
+        "analysis_09_selectivity.png", filename_suffix
+    )
     if plot_selectivity(rows, selectivity_path, smoothing_window=smoothing_window):
         written.append(selectivity_path)
 
-    final_eval_path = output_dir / "analysis_final_eval_summary.png"
+    final_eval_path = output_dir / _with_filename_suffix(
+        "analysis_final_eval_summary.png", filename_suffix
+    )
     if plot_final_eval_summary(rows, final_eval_path):
         written.append(final_eval_path)
     return written
 
 
+def _iteration_filename_suffix(max_iteration: int | None) -> str:
+    if max_iteration is None:
+        return ""
+    return f"_upto_{max_iteration:05d}"
+
+
+def _with_filename_suffix(filename: str, suffix: str) -> str:
+    if not suffix:
+        return filename
+    path = Path(filename)
+    return f"{path.stem}{suffix}{path.suffix}"
+
+
 def plot_selectivity(
     rows: list[dict[str, Any]],
     output: Path,
@@ -979,9 +1002,19 @@ def main(argv: list[str] | None = None) -> None:
         action="store_true",
         help="Disable moving-average smoothing.",
     )
+    parser.add_argument(
+        "--max-iteration",
+        type=int,
+        help="Only plot metrics up to and including this iteration.",
+    )
     args = parser.parse_args(argv)
     smoothing_window = 1 if args.no_smoothing else max(1, args.smoothing_window)
-    written = analyze_run(args.run, args.output_dir, smoothing_window=smoothing_window)
+    written = analyze_run(
+        args.run,
+        args.output_dir,
+        smoothing_window=smoothing_window,
+        max_iteration=args.max_iteration,
+    )
     for path in written:
         print(path)
 

src/coolrl_lost_cities/games/classic/deep_cfr/cli.py

@@ -200,7 +200,7 @@ def policy_gradient_command(args: argparse.Namespace) -> None:
 
 
 def analyze_command(args: argparse.Namespace) -> None:
-    written = analyze_run(args.run, args.output_dir)
+    written = analyze_run(args.run, args.output_dir, max_iteration=args.max_iteration)
     for path in written:
         print(path)
 
@@ -277,6 +277,11 @@ def main(argv: list[str] | None = None) -> None:
     analyze = subparsers.add_parser("analyze")
     analyze.add_argument("--run", required=True, type=Path)
     analyze.add_argument("--output-dir", type=Path)
+    analyze.add_argument(
+        "--max-iteration",
+        type=int,
+        help="Only plot metrics up to and including this iteration.",
+    )
     analyze.set_defaults(func=analyze_command)
 
     args = parser.parse_args(argv)