Add multi-process self-play, eval workers, MCTS Cython port
Key changes for ISMCTS speed and correctness: - Cython port: HeuristicBot helpers (`heuristic_cy.pyx` + new `.pxd`) and ISMCTS searcher (`mcts.pyx`) now run as cdef. Both share a fast unified-action path through GameState's C interface to avoid Python round-trips on hot rollout/tree-walk paths. - Multi-process self-play and eval: `workers.py`, `eval_worker.py`, `interleaved_self_play.py`, plus trainer wiring with ProcessPoolExecutor + spawn context. Eval inside `evaluate.py` is parallel per opponent. - ISMCTS-specific eval (`evaluate.py`) runs MCTS at decision time so the metric matches deploy mode; `evaluation.eval_with_mcts` flag preserves backwards-compatible policy-only eval when needed. - Trainer logs progress per phase (self-play start/done, eval per opponent), and value loss is now scaled by `value_scale` so policy and value losses sit on comparable magnitudes. - Compact info-set key (`info_set.py`) using packed-struct format and child-key reuse during MCTS descent to cut per-step canonicalization. Tests: 19 ISMCTS suite passing, including parity (Cython-vs-Python sequential, batched-vs-sequential visit counts, push/pop round-trip). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -25,14 +25,26 @@ class MctsConfig(StrictModel):
|
||||
c_puct: float = 1.5
|
||||
max_depth: int = 200
|
||||
use_rollout_value: bool = True
|
||||
rollout_policy: str = "random"
|
||||
parallel_simulations: int = 8
|
||||
virtual_loss_value: float = 1.0
|
||||
eval_with_mcts: bool = True
|
||||
eval_n_simulations: int = 0
|
||||
|
||||
@field_validator("n_simulations", "max_depth")
|
||||
@field_validator("n_simulations", "max_depth", "parallel_simulations")
|
||||
@classmethod
|
||||
def _positive_int(cls, value: int) -> int:
|
||||
if value <= 0:
|
||||
raise ValueError("must be positive")
|
||||
return value
|
||||
|
||||
@field_validator("rollout_policy")
|
||||
@classmethod
|
||||
def _rollout_policy(cls, value: str) -> str:
|
||||
if value not in {"random", "heuristic_balanced"}:
|
||||
raise ValueError("rollout_policy must be 'random' or 'heuristic_balanced'")
|
||||
return value
|
||||
|
||||
|
||||
class TemperatureConfig(StrictModel):
|
||||
training: float = 1.0
|
||||
@@ -44,8 +56,20 @@ class TrainingConfig(StrictModel):
|
||||
gradient_steps_per_iter: int = 10
|
||||
batch_size: int = 128
|
||||
replay_capacity: int = 100_000
|
||||
interleave_games: int = 8
|
||||
interleave_max_batch: int = 64
|
||||
num_workers: int = 1
|
||||
worker_device: str = "cpu"
|
||||
|
||||
@field_validator("games_per_iter", "gradient_steps_per_iter", "batch_size", "replay_capacity")
|
||||
@field_validator(
|
||||
"games_per_iter",
|
||||
"gradient_steps_per_iter",
|
||||
"batch_size",
|
||||
"replay_capacity",
|
||||
"interleave_games",
|
||||
"interleave_max_batch",
|
||||
"num_workers",
|
||||
)
|
||||
@classmethod
|
||||
def _positive_int(cls, value: int) -> int:
|
||||
if value <= 0:
|
||||
|
||||
Reference in New Issue
Block a user