From f289997c1c19ae49e22702ae979266c875677a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=8B=9C=EC=9B=90?= Date: Mon, 11 May 2026 05:17:32 +0900 Subject: [PATCH] Add Dirichlet root noise + standalone eval CLI, fix self-play stall trap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trap diagnosis: agent learned to stall (avoid opening expeditions, draw from discard pile to extend deck) until max_steps timeout, then squeak by on opponents' negative scores. All eval wins were from timeouts; agent never won a naturally-terminating game. Self-play reinforced this because timeout games still got a positive value target. Fixes (no algorithm change, all MCTS hyperparameters or signal shaping): - Dirichlet noise at root prior (AlphaZero standard, was missing): mcts.pyx `_expand_with_prior` takes `is_root` flag; root expansion mixes prior with Dirichlet(α). Callers in interleaved_self_play and the internal evaluate_and_backup pass `not item.path`. - Default config strengthens exploration on the 50-sim batched search: c_puct 1.5 -> 3.0, virtual_loss_value 1.0 -> 5.0, plus new root_dirichlet_alpha=0.3 / root_dirichlet_epsilon=0.25. - Self-play timeout signal zeroed: `_finalize_context` sets v_target=0 if context.state is not terminal. Stops the network from learning "stall = positive value". New standalone evaluator: - `lost-cities-ismcts eval` subcommand (eval_checkpoint.py): loads a checkpoint, runs N games per opponent across a parallel pool, reports win/score with 95% CIs plus per-game logging via --verbose. Defaults cover heuristic-balanced/aggressive/cautious (rollout policy isn't in the training-eval opponent list, so this is the natural way to compare the trained policy against its rollout target). Tests (19) still pass; .so rebuilt. Co-Authored-By: Claude Opus 4.7 (1M context) --- configs/ismcts/default.yaml | 26 +- .../games/classic/ismcts/cli.py | 10 + .../games/classic/ismcts/config.py | 2 + .../games/classic/ismcts/eval_checkpoint.py | 323 ++++++++++++++++++ .../classic/ismcts/interleaved_self_play.py | 11 +- .../games/classic/ismcts/mcts.pyx | 19 ++ 6 files changed, 380 insertions(+), 11 deletions(-) create mode 100644 src/coolrl_lost_cities/games/classic/ismcts/eval_checkpoint.py diff --git a/configs/ismcts/default.yaml b/configs/ismcts/default.yaml index d76fc7b..798bdbf 100644 --- a/configs/ismcts/default.yaml +++ b/configs/ismcts/default.yaml @@ -1,8 +1,8 @@ run: experiment_name: ismcts-default - max_iterations: 100 + max_iterations: 500 seed: 1 - device: auto + device: cuda rules: n_colors: 5 n_ranks: 9 @@ -22,10 +22,14 @@ network: activation: relu mcts: n_simulations: 50 - c_puct: 1.5 + c_puct: 3.0 max_depth: 200 - parallel_simulations: 8 - virtual_loss_value: 1.0 + parallel_simulations: 64 + virtual_loss_value: 5.0 + eval_n_simulations: 16 + rollout_policy: heuristic_balanced + root_dirichlet_alpha: 0.3 + root_dirichlet_epsilon: 0.25 temperature: training: 1.0 eval: 0.0 @@ -36,15 +40,17 @@ training: replay_capacity: 100000 interleave_games: 8 interleave_max_batch: 64 + num_workers: 8 + worker_device: cuda optimization: learning_rate: 0.0003 grad_clip: 5.0 checkpoint: - save_every: 10 + save_every: 20 save_latest: true evaluation: - eval_every: 10 - games: 20 + eval_every: 5 + games: 5 opponents: [random, discard-only, heuristic-cautious] - max_steps: 10000 - num_workers: 1 + max_steps: 500 + num_workers: 8 diff --git a/src/coolrl_lost_cities/games/classic/ismcts/cli.py b/src/coolrl_lost_cities/games/classic/ismcts/cli.py index 3ed96fd..941f4ef 100644 --- a/src/coolrl_lost_cities/games/classic/ismcts/cli.py +++ b/src/coolrl_lost_cities/games/classic/ismcts/cli.py @@ -112,6 +112,16 @@ def main(argv: list[str] | None = None) -> None: train.add_argument("--wandb-tag", action="append", default=[]) train.add_argument("--wandb-notes") train.set_defaults(func=train_command) + + from .eval_checkpoint import add_eval_args, run_eval + + eval_cmd = subparsers.add_parser( + "eval", + help="Evaluate a saved checkpoint vs heuristic bots in parallel.", + ) + add_eval_args(eval_cmd) + eval_cmd.set_defaults(func=lambda a: run_eval(a)) + args = parser.parse_args(argv) args.func(args) diff --git a/src/coolrl_lost_cities/games/classic/ismcts/config.py b/src/coolrl_lost_cities/games/classic/ismcts/config.py index 0b849fa..8e3aeae 100644 --- a/src/coolrl_lost_cities/games/classic/ismcts/config.py +++ b/src/coolrl_lost_cities/games/classic/ismcts/config.py @@ -30,6 +30,8 @@ class MctsConfig(StrictModel): virtual_loss_value: float = 1.0 eval_with_mcts: bool = True eval_n_simulations: int = 0 + root_dirichlet_alpha: float = 0.0 + root_dirichlet_epsilon: float = 0.0 @field_validator("n_simulations", "max_depth", "parallel_simulations") @classmethod diff --git a/src/coolrl_lost_cities/games/classic/ismcts/eval_checkpoint.py b/src/coolrl_lost_cities/games/classic/ismcts/eval_checkpoint.py new file mode 100644 index 0000000..093d24a --- /dev/null +++ b/src/coolrl_lost_cities/games/classic/ismcts/eval_checkpoint.py @@ -0,0 +1,323 @@ +"""Standalone parallel evaluation of an ISMCTS checkpoint vs heuristic bots. + +Reuses the same MCTS / bot stack as the training-loop eval, but does not +interfere with a running training process. Useful for comparing a snapshot +against opponents that are not in `evaluation.opponents` (e.g. heuristic-balanced, +the rollout policy) and for running many more games than per-iter eval typically +allows. + +Invoked via ``lost-cities-ismcts eval`` (see ``cli.py``). +""" + +from __future__ import annotations + +import argparse +import multiprocessing as mp +import os +import random +import sys +import time +from concurrent.futures import ProcessPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path + +import torch + + +@dataclass(frozen=True) +class _WorkerJob: + ckpt_path: str + opponent: str + game_indices: tuple[int, ...] + seed: int + device: str + worker_index: int + verbose: bool + + +@dataclass +class _GameResult: + game_index: int + policy_player: int + score_diff: float + turns: int + policy_turns: int + play_actions: int + timed_out: bool + + +@dataclass +class _WorkerResult: + worker_index: int + opponent: str + games: list[_GameResult] = field(default_factory=list) + elapsed: float = 0.0 + + +def _run_games(job: _WorkerJob) -> _WorkerResult: + # Inside-worker imports + thread-cap to avoid CPU oversubscription + from coolrl_lost_cities.games.classic.bots.registry import build_bot + from coolrl_lost_cities.games.classic.deep_cfr.encoding import input_dim + from coolrl_lost_cities.games.classic.game import GameState, LostCitiesConfig + from coolrl_lost_cities.games.classic.ismcts.config import IsMctsConfig + from coolrl_lost_cities.games.classic.ismcts.mcts import IsMctsSearcher + from coolrl_lost_cities.games.classic.ismcts.network import AlphaZeroNet + + os.environ.setdefault("OMP_NUM_THREADS", "1") + os.environ.setdefault("MKL_NUM_THREADS", "1") + torch.set_num_threads(1) + + started = time.perf_counter() + print( + f" [worker {job.worker_index}] start ({len(job.game_indices)} games " + f"vs {job.opponent}, device={job.device})", + flush=True, + ) + + ckpt = torch.load(job.ckpt_path, map_location="cpu", weights_only=False) + cfg = IsMctsConfig.model_validate(ckpt["config"]) + game_config = LostCitiesConfig(**ckpt["game_config"]) + probe = GameState.new_game(game_config, seed=cfg.run.seed) + dim = input_dim(probe, cfg.encoding) + device = torch.device(job.device) + net = AlphaZeroNet.from_config(dim, probe.action_size, cfg).to(device) + net.load_state_dict(ckpt["network"]) + net.eval() + + eval_mcts_cfg = cfg.mcts.model_copy() + if cfg.mcts.eval_n_simulations > 0: + eval_mcts_cfg = eval_mcts_cfg.model_copy( + update={"n_simulations": cfg.mcts.eval_n_simulations} + ) + + rng = random.Random(job.seed + job.worker_index * 7919) + result = _WorkerResult(worker_index=job.worker_index, opponent=job.opponent) + + max_steps = 500 + for game_index in job.game_indices: + game_started = time.perf_counter() + policy_player = game_index % 2 + opps = [ + build_bot(job.opponent, seed=job.seed + game_index), + build_bot(job.opponent, seed=job.seed + game_index + 1), + ] + state = GameState.new_game(game_config, seed=job.seed + game_index) + turns = 0 + policy_turns = 0 + play_actions = 0 + timed_out = False + while True: + if state.terminal: + break + if turns >= max_steps: + timed_out = True + break + current = int(state.current_player) + if current == policy_player: + searcher = IsMctsSearcher( + net, + eval_mcts_cfg, + device=device, + encoding=cfg.encoding, + rng=random.Random(rng.randrange(2**31)), + ) + visits = searcher.search(state, current) + unified = ( + max(visits, key=visits.get) if visits else state.unified_legal_actions()[0] + ) + if state.phase == "card": + policy_turns += 1 + if unified % 2 == 0: + play_actions += 1 + state.apply_unified_action(unified) + else: + state.apply_action(opps[current].act(state)) + turns += 1 + + diff = float(state.score_diff(policy_player)) + gr = _GameResult( + game_index=game_index, + policy_player=policy_player, + score_diff=diff, + turns=turns, + policy_turns=policy_turns, + play_actions=play_actions, + timed_out=timed_out, + ) + result.games.append(gr) + if job.verbose: + elapsed_g = time.perf_counter() - game_started + pa = play_actions / policy_turns if policy_turns else 0.0 + print( + f" [worker {job.worker_index}] game {game_index:3d} " + f"as P{policy_player} | turns={turns:3d} " + f"score={diff:+6.1f} PA={pa:.2f}" + f"{' TIMEOUT' if timed_out else ''} " + f"({elapsed_g:.1f}s)", + flush=True, + ) + + result.elapsed = time.perf_counter() - started + won = sum(1 for g in result.games if g.score_diff > 0) + print( + f" [worker {job.worker_index}] done {len(result.games)} games " + f"vs {job.opponent} in {result.elapsed:.1f}s " + f"(W={won}/{len(result.games)})", + flush=True, + ) + return result + + +def _split_games(n_games: int, n_workers: int) -> list[tuple[int, ...]]: + n_workers = max(1, min(n_workers, n_games)) + base = n_games // n_workers + rem = n_games % n_workers + out: list[tuple[int, ...]] = [] + cursor = 0 + for i in range(n_workers): + count = base + (1 if i < rem else 0) + out.append(tuple(range(cursor, cursor + count))) + cursor += count + return out + + +def _summarize(games: list[_GameResult]) -> dict[str, float]: + n = len(games) + if n == 0: + return {} + wins = sum(1 for g in games if g.score_diff > 0) + losses = sum(1 for g in games if g.score_diff < 0) + draws = sum(1 for g in games if g.score_diff == 0) + timeouts = sum(1 for g in games if g.timed_out) + score_diffs = [g.score_diff for g in games] + avg = sum(score_diffs) / n + var = sum((d - avg) ** 2 for d in score_diffs) / n if n > 1 else 0.0 + std = var**0.5 + total_policy_turns = sum(g.policy_turns for g in games) + total_play_actions = sum(g.play_actions for g in games) + pa = total_play_actions / total_policy_turns if total_policy_turns else 0.0 + avg_turns = sum(g.turns for g in games) / n + z = 1.96 + # Wilson CI on win rate (half-width only; report as wr ± half) + wr = wins / n + denom = 1 + z * z / n + half = z / denom * ((wr * (1 - wr) / n + z * z / (4 * n * n)) ** 0.5) + score_ci = z * std / (n**0.5) if n > 1 else 0.0 + return { + "games": n, + "wins": wins, + "losses": losses, + "draws": draws, + "timeouts": timeouts, + "win_rate": wr, + "win_rate_ci_half": half, + "avg_score_diff": avg, + "score_std": std, + "score_ci_half": score_ci, + "play_action_rate": pa, + "avg_turns": avg_turns, + } + + +def _default_ckpt() -> Path: + """Find latest 'ismcts-overnight*' run's latest.pt, or fallback to newest run.""" + candidates = sorted(Path("runs").glob("*ismcts-overnight*")) + if candidates: + return candidates[-1] / "latest.pt" + candidates = sorted(Path("runs").iterdir()) + if not candidates: + raise SystemExit("no runs/ directory entries") + return candidates[-1] / "latest.pt" + + +def add_eval_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--ckpt", + default=None, + help="Path to checkpoint .pt. Default: latest overnight run latest.pt.", + ) + parser.add_argument( + "--opponents", + nargs="+", + default=["heuristic-balanced", "heuristic-aggressive", "heuristic-cautious"], + help="Opponent bot names from registry.", + ) + parser.add_argument("--games", type=int, default=50, help="Games per opponent (default: 50).") + parser.add_argument( + "--device", + choices=("cpu", "cuda"), + default="cpu", + help="Worker device (default: cpu; cuda may compete with running training).", + ) + parser.add_argument( + "--num-workers", type=int, default=8, help="Parallel worker count (default: 8)." + ) + parser.add_argument("--seed", type=int, default=99999) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-game result lines (turns/score/PA) in addition to per-worker summaries.", + ) + + +def run_eval(args: argparse.Namespace) -> None: + ckpt_path = Path(args.ckpt) if args.ckpt else _default_ckpt() + if not ckpt_path.exists(): + print(f"checkpoint not found: {ckpt_path}", file=sys.stderr) + raise SystemExit(1) + + ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False) + iteration = ckpt.get("iteration", "?") + print(f"checkpoint : {ckpt_path}") + print(f"iteration : {iteration}") + print(f"device : {args.device}") + print(f"workers : {args.num_workers}") + print(f"games/opp : {args.games}") + print(f"opponents : {args.opponents}") + print(f"seed : {args.seed}") + if args.verbose: + print("verbose : True (per-game logging)") + print() + + ctx = mp.get_context("spawn") + started_all = time.perf_counter() + + for opponent in args.opponents: + opp_started = time.perf_counter() + slices = _split_games(args.games, args.num_workers) + jobs = [ + _WorkerJob( + ckpt_path=str(ckpt_path), + opponent=opponent, + game_indices=tuple(slices[i]), + seed=args.seed, + device=args.device, + worker_index=i, + verbose=args.verbose, + ) + for i in range(len(slices)) + ] + all_games: list[_GameResult] = [] + with ProcessPoolExecutor(max_workers=len(jobs), mp_context=ctx) as ex: + futures = [ex.submit(_run_games, j) for j in jobs] + for f in as_completed(futures): + res = f.result() + all_games.extend(res.games) + + elapsed = time.perf_counter() - opp_started + summary = _summarize(all_games) + n = int(summary["games"]) + print() + print( + f"vs {opponent:22s} | W={summary['wins']}/{n} " + f"({summary['win_rate']:.2f} ± {summary['win_rate_ci_half']:.2f}) " + f"| S={summary['avg_score_diff']:+6.1f} ± {summary['score_ci_half']:5.1f} " + f"(σ={summary['score_std']:.1f}) | PA={summary['play_action_rate']:.2f} " + f"| turns={summary['avg_turns']:.0f} " + f"| timeouts={summary['timeouts']} " + f"| elapsed={elapsed:.1f}s" + ) + print() + + total = time.perf_counter() - started_all + print(f"total elapsed: {total:.1f}s") diff --git a/src/coolrl_lost_cities/games/classic/ismcts/interleaved_self_play.py b/src/coolrl_lost_cities/games/classic/ismcts/interleaved_self_play.py index 3e4b763..583e5b4 100644 --- a/src/coolrl_lost_cities/games/classic/ismcts/interleaved_self_play.py +++ b/src/coolrl_lost_cities/games/classic/ismcts/interleaved_self_play.py @@ -181,6 +181,7 @@ def _evaluate_global_batch( item.legal_actions, priors_by_id[id(item)], values_by_id[id(item)], + not item.path, # is_root for Dirichlet noise ) job.searcher._backup(item.path, value, item.leaf_player) @@ -222,7 +223,15 @@ def _finish_decision( def _finalize_context(context: _GameContext) -> list[ReplaySample]: - final_diff0 = float(context.state.score_diff(0)) + # If the game did not terminate naturally (hit max_steps), the score + # reflects an incomplete game — typically a "stall" outcome where both + # sides have under-developed expeditions. Treating that as a real win + # for either player creates a degenerate stall-and-pray learning + # signal. Zero it out so the trajectory is neutral. + if not context.state.terminal: + final_diff0 = 0.0 + else: + final_diff0 = float(context.state.score_diff(0)) samples: list[ReplaySample] = [] for decision in context.decisions: value = final_diff0 if decision.player == 0 else -final_diff0 diff --git a/src/coolrl_lost_cities/games/classic/ismcts/mcts.pyx b/src/coolrl_lost_cities/games/classic/ismcts/mcts.pyx index f142a70..a7cfc07 100644 --- a/src/coolrl_lost_cities/games/classic/ismcts/mcts.pyx +++ b/src/coolrl_lost_cities/games/classic/ismcts/mcts.pyx @@ -489,6 +489,7 @@ cdef class IsMctsSearcher: item.legal_actions, priors_by_id[id(item)], values_by_id[id(item)], + not item.path, # is_root: empty path means leaf == root ) self._backup(item.path, value, item.leaf_player) @@ -500,14 +501,32 @@ cdef class IsMctsSearcher: list legal_actions, object probs, double network_value, + bint is_root=False, ): cdef int action + cdef int i + cdef int n_legal + cdef double alpha + cdef double epsilon cdef object rollout_value + cdef object noise + cdef object np_rng legal_actions = self._unified_legal_actions_list_c(state) if not legal_actions: node.terminal = True return float(state.total_scores[player] - state.total_scores[1 - player]) node.expanded = True + # Dirichlet noise at root (AlphaZero pattern: force exploration of low-prior actions) + alpha = float(self.config.root_dirichlet_alpha) + epsilon = float(self.config.root_dirichlet_epsilon) + if is_root and alpha > 0.0 and epsilon > 0.0: + n_legal = len(legal_actions) + # Use numpy with seed from self.rng for reproducibility under fixed seeds + np_rng = np.random.default_rng(self.rng.randrange(2**31)) + noise = np_rng.dirichlet([alpha] * n_legal) + for i in range(n_legal): + action = legal_actions[i] + probs[action] = (1.0 - epsilon) * float(probs[action]) + epsilon * float(noise[i]) for action in legal_actions: (<_ArrayMap>node.priors).set_float(action, float(probs[action])) if not (<_ArrayMap>node.visits).has(action):