From fa9a1c5286ce58ed40b462e6dff8f8d68dc72c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=8B=9C=EC=9B=90?= Date: Sun, 5 Jul 2026 17:55:22 +0900 Subject: [PATCH] Add gates 1-2 audit and repair workflow --- .../gates-1-2-exploiter-long-random.yaml | 40 + .../jax_ppo/gates-1-2-exploiter-replay.yaml | 40 + .../jax_ppo/gates-1-2-exploiter-smoke.yaml | 40 + .../gates-1-2-exploiter-warmstart.yaml | 40 + configs/jax_ppo/gates-1-2-smoke.yaml | 35 + configs/jax_ppo/gates-1-2.yaml | 44 + .../gates-1-2-2026-07-05-summary.jsonl | 19 + docs/reports/gates-1-2-2026-07-05.md | 67 + docs/reports/gates-1-2-delta-open-hist.png | Bin 0 -> 19387 bytes src/lost_cities_jax/gates.py | 1146 +++++++++++++++++ src/lost_cities_jax/league.py | 3 + src/lost_cities_jax/opponents.py | 19 +- src/lost_cities_jax/ppo.py | 21 +- tests/lost_cities_jax/test_gates.py | 19 + tests/lost_cities_jax/test_ppo_stack.py | 72 ++ 15 files changed, 1602 insertions(+), 3 deletions(-) create mode 100644 configs/jax_ppo/gates-1-2-exploiter-long-random.yaml create mode 100644 configs/jax_ppo/gates-1-2-exploiter-replay.yaml create mode 100644 configs/jax_ppo/gates-1-2-exploiter-smoke.yaml create mode 100644 configs/jax_ppo/gates-1-2-exploiter-warmstart.yaml create mode 100644 configs/jax_ppo/gates-1-2-smoke.yaml create mode 100644 configs/jax_ppo/gates-1-2.yaml create mode 100644 docs/reports/gates-1-2-2026-07-05-summary.jsonl create mode 100644 docs/reports/gates-1-2-2026-07-05.md create mode 100644 docs/reports/gates-1-2-delta-open-hist.png create mode 100644 src/lost_cities_jax/gates.py create mode 100644 tests/lost_cities_jax/test_gates.py diff --git a/configs/jax_ppo/gates-1-2-exploiter-long-random.yaml b/configs/jax_ppo/gates-1-2-exploiter-long-random.yaml new file mode 100644 index 0000000..178ecb1 --- /dev/null +++ b/configs/jax_ppo/gates-1-2-exploiter-long-random.yaml @@ -0,0 +1,40 @@ +run: + experiment_name: gates-1-2-long-random-exploiter + seed: 20260705 + learner_seat: 0 + total_updates: 1200 + log_every: 10 + checkpoint_every: 100 + artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/exploiters + +opponent: + name: league_v1_update_500_frozen + +network: + hidden_size: 512 + num_layers: 3 + +ppo: + batch_games: 8192 + rollout_steps: 400 + gamma: 1.0 + gae_lambda: 0.95 + clip_epsilon: 0.2 + entropy_coef: 0.01 + value_coef: 0.5 + max_grad_norm: 0.5 + learning_rate: 0.0003 + epochs: 4 + minibatches: 128 + +reward: + terminal_scale: 50.0 + potential_shaping_initial: 1.0 + potential_shaping_final: 0.0 + potential_shaping_anneal_steps: 5000000 + +evaluation: + games: 2000 + duplicate: true + shuffle_bank_seed: 20260704 + batch_games: 8192 diff --git a/configs/jax_ppo/gates-1-2-exploiter-replay.yaml b/configs/jax_ppo/gates-1-2-exploiter-replay.yaml new file mode 100644 index 0000000..4cdec2f --- /dev/null +++ b/configs/jax_ppo/gates-1-2-exploiter-replay.yaml @@ -0,0 +1,40 @@ +run: + experiment_name: gates-1-2-replay-exploiter + seed: 20260705 + learner_seat: 0 + total_updates: 900 + log_every: 10 + checkpoint_every: 100 + artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/exploiters + +opponent: + name: league_v1_update_500_frozen + +network: + hidden_size: 512 + num_layers: 3 + +ppo: + batch_games: 8192 + rollout_steps: 400 + gamma: 1.0 + gae_lambda: 0.95 + clip_epsilon: 0.2 + entropy_coef: 0.01 + value_coef: 0.5 + max_grad_norm: 0.5 + learning_rate: 0.0003 + epochs: 4 + minibatches: 128 + +reward: + terminal_scale: 50.0 + potential_shaping_initial: 0.0 + potential_shaping_final: 0.0 + potential_shaping_anneal_steps: 0 + +evaluation: + games: 2000 + duplicate: true + shuffle_bank_seed: 20260704 + batch_games: 8192 diff --git a/configs/jax_ppo/gates-1-2-exploiter-smoke.yaml b/configs/jax_ppo/gates-1-2-exploiter-smoke.yaml new file mode 100644 index 0000000..976543e --- /dev/null +++ b/configs/jax_ppo/gates-1-2-exploiter-smoke.yaml @@ -0,0 +1,40 @@ +run: + experiment_name: gates-1-2-smoke-exploiter + seed: 20260705 + learner_seat: 0 + total_updates: 1 + log_every: 1 + checkpoint_every: 1 + artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2-smoke/exploiters + +opponent: + name: league_v1_update_500_frozen + +network: + hidden_size: 512 + num_layers: 3 + +ppo: + batch_games: 8 + rollout_steps: 16 + gamma: 1.0 + gae_lambda: 0.95 + clip_epsilon: 0.2 + entropy_coef: 0.01 + value_coef: 0.5 + max_grad_norm: 0.5 + learning_rate: 0.0003 + epochs: 1 + minibatches: 2 + +reward: + terminal_scale: 50.0 + potential_shaping_initial: 1.0 + potential_shaping_final: 0.0 + potential_shaping_anneal_steps: 5000000 + +evaluation: + games: 2 + duplicate: true + shuffle_bank_seed: 20260704 + batch_games: 2 diff --git a/configs/jax_ppo/gates-1-2-exploiter-warmstart.yaml b/configs/jax_ppo/gates-1-2-exploiter-warmstart.yaml new file mode 100644 index 0000000..335278b --- /dev/null +++ b/configs/jax_ppo/gates-1-2-exploiter-warmstart.yaml @@ -0,0 +1,40 @@ +run: + experiment_name: gates-1-2-warmstart-gate3-exploiter + seed: 20260705 + learner_seat: 0 + total_updates: 900 + log_every: 10 + checkpoint_every: 100 + artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/exploiters + +opponent: + name: league_v1_update_500_frozen + +network: + hidden_size: 512 + num_layers: 3 + +ppo: + batch_games: 8192 + rollout_steps: 400 + gamma: 1.0 + gae_lambda: 0.95 + clip_epsilon: 0.2 + entropy_coef: 0.01 + value_coef: 0.5 + max_grad_norm: 0.5 + learning_rate: 0.0003 + epochs: 4 + minibatches: 128 + +reward: + terminal_scale: 50.0 + potential_shaping_initial: 0.0 + potential_shaping_final: 0.0 + potential_shaping_anneal_steps: 0 + +evaluation: + games: 2000 + duplicate: true + shuffle_bank_seed: 20260704 + batch_games: 8192 diff --git a/configs/jax_ppo/gates-1-2-smoke.yaml b/configs/jax_ppo/gates-1-2-smoke.yaml new file mode 100644 index 0000000..5be27bb --- /dev/null +++ b/configs/jax_ppo/gates-1-2-smoke.yaml @@ -0,0 +1,35 @@ +run: + artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2-smoke + report_path: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2-smoke/report.md + summary_path: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2-smoke/summary.jsonl + shuffle_bank_seed: 20260704 + gpu_budget_hours: 1.0 + +target: + config: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/main_ppo_config.json + checkpoint: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/snapshots/cycle_01_update_000500 + +gate1: + games: 2 + batch_games: 2 + delta_games: 2 + delta_target_events: 2 + delta_pairs: 2 + delta_batch_events: 1 + +gate2: + eval_games: 2 + pass_threshold: 0.55 + exploiters: + - name: long_random_smoke + config: configs/jax_ppo/gates-1-2-exploiter-smoke.yaml + +repair: + max_cycles: 1 + league_template: configs/jax_ppo/league-smoke.yaml + experiment_name: jax-ppo-gates-1-2-repair-smoke + artifact_subdir: gate2c + evaluation_games: 2 + evaluation_batch_games: 2 + guard_expert_ci_low: -1000.0 + guard_max_steps_rate: 1.0 diff --git a/configs/jax_ppo/gates-1-2.yaml b/configs/jax_ppo/gates-1-2.yaml new file mode 100644 index 0000000..97b0f34 --- /dev/null +++ b/configs/jax_ppo/gates-1-2.yaml @@ -0,0 +1,44 @@ +run: + artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2 + report_path: docs/reports/gates-1-2-2026-07-05.md + summary_path: docs/reports/gates-1-2-2026-07-05-summary.jsonl + shuffle_bank_seed: 20260704 + gpu_budget_hours: 8.0 + +target: + config: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/main_ppo_config.json + checkpoint: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/snapshots/cycle_01_update_000500 + +gate1: + games: 2000 + batch_games: 8192 + delta_games: 200 + delta_target_events: 500 + delta_pairs: 64 + delta_batch_events: 16 + +gate2: + eval_games: 2000 + pass_threshold: 0.55 + exploiters: + - name: long_random + config: configs/jax_ppo/gates-1-2-exploiter-long-random.yaml + notes: random init + shaping anneal, extended budget + - name: warmstart_gate3 + config: configs/jax_ppo/gates-1-2-exploiter-warmstart.yaml + resume: /mnt/2tbhdd/coolrl-lost-cities-artifacts/ladder-v2/2026-07-05_013223_jax-ppo-ladder-v2-expert/latest + notes: ladder v2 gate-3 warm start, shaping disabled + - name: replay_exploiter + config: configs/jax_ppo/gates-1-2-exploiter-replay.yaml + resume: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/exploiters/2026-07-05_060727_jax-ppo-league-v1-cycle-1-exploiter/latest + notes: league v1 exploiter warm start, shaping disabled + +repair: + max_cycles: 3 + league_template: configs/jax_ppo/league-v1.yaml + experiment_name: jax-ppo-gates-1-2-repair + artifact_subdir: gate2c + evaluation_games: 2000 + evaluation_batch_games: 8192 + guard_expert_ci_low: 0.0 + guard_max_steps_rate: 0.02 diff --git a/docs/reports/gates-1-2-2026-07-05-summary.jsonl b/docs/reports/gates-1-2-2026-07-05-summary.jsonl new file mode 100644 index 0000000..8539728 --- /dev/null +++ b/docs/reports/gates-1-2-2026-07-05-summary.jsonl @@ -0,0 +1,19 @@ +{"event": "gate1_tournament", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate1/expert_cap2_vs_expert_cap3.json", "learner": "expert_cap2", "losses": 2102.0, "max_steps_rate": 0.0, "mean_game_length": 46.549, "mean_score_diff": -1.89325, "opened_colors_per_game": 1.77925, "opponent": "expert_cap3", "play_action_rate": 0.2414775169804832, "positive_expeditions_per_game": 0.94175, "score_diff_ci95_high": -1.4203123965597364, "score_diff_ci95_low": -2.366187603440264, "score_diff_std": 15.261096936571331, "ties": 132.0, "wilson_high": 0.45693730157215046, "wilson_low": 0.42617495329274474, "win_rate": 0.4415, "wins": 1766.0} +{"event": "gate1_tournament", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate1/expert_cap2_vs_expert_capN.json", "learner": "expert_cap2", "losses": 2092.0, "max_steps_rate": 0.0, "mean_game_length": 46.354, "mean_score_diff": -1.88075, "opened_colors_per_game": 1.7815, "opponent": "expert_capN", "play_action_rate": 0.24223387949830538, "positive_expeditions_per_game": 0.93725, "score_diff_ci95_high": -1.4070318876291175, "score_diff_ci95_low": -2.3544681123708826, "score_diff_std": 15.286282970338549, "ties": 130.0, "wilson_high": 0.45994507591745637, "wilson_low": 0.4291614222877006, "win_rate": 0.4445, "wins": 1778.0} +{"event": "gate1_tournament", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate1/expert_cap3_vs_expert_capN.json", "learner": "expert_cap3", "losses": 1927.0, "max_steps_rate": 0.0, "mean_game_length": 45.35975, "mean_score_diff": 0.0285, "opened_colors_per_game": 2.21525, "opponent": "expert_capN", "play_action_rate": 0.3101634934459303, "positive_expeditions_per_game": 1.20625, "score_diff_ci95_high": 0.5258735279417411, "score_diff_ci95_low": -0.46887352794174114, "score_diff_std": 16.04961324366783, "ties": 137.0, "wilson_high": 0.49949486790031544, "wilson_low": 0.4685358342849551, "win_rate": 0.484, "wins": 1936.0} +{"event": "gate1_vs_league", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate1/expert_cap2_vs_league_v1_update_500.json", "learner": "expert_cap2", "losses": 2871.0, "max_steps_rate": 0.00025, "mean_game_length": 55.372, "mean_score_diff": -20.88625, "opened_colors_per_game": 1.9735, "opponent": "league_v1_update_500", "play_action_rate": 0.22526015834941476, "positive_expeditions_per_game": 1.09725, "score_diff_ci95_high": -19.787867065349186, "score_diff_ci95_low": -21.984632934650815, "score_diff_std": 35.44342491958145, "ties": 39.0, "wilson_high": 0.2865114339331819, "wilson_low": 0.2589251127636324, "win_rate": 0.2725, "wins": 1090.0} +{"event": "gate1_vs_league", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate1/expert_cap3_vs_league_v1_update_500.json", "learner": "expert_cap3", "losses": 2590.0, "max_steps_rate": 0.0, "mean_game_length": 53.52525, "mean_score_diff": -14.35125, "opened_colors_per_game": 2.7385, "opponent": "league_v1_update_500", "play_action_rate": 0.320897958422324, "positive_expeditions_per_game": 1.52675, "score_diff_ci95_high": -13.256641796659636, "score_diff_ci95_low": -15.445858203340364, "score_diff_std": 35.32161912528796, "ties": 50.0, "wilson_high": 0.3548273870268761, "wilson_low": 0.32547963482582853, "win_rate": 0.34, "wins": 1360.0} +{"event": "gate1_vs_league", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate1/expert_capN_vs_league_v1_update_500.json", "learner": "expert_capN", "losses": 2585.0, "max_steps_rate": 0.0, "mean_game_length": 53.29175, "mean_score_diff": -13.967, "opened_colors_per_game": 3.004, "opponent": "league_v1_update_500", "play_action_rate": 0.3410701746172932, "positive_expeditions_per_game": 1.56475, "score_diff_ci95_high": -12.879677994093045, "score_diff_ci95_low": -15.054322005906956, "score_diff_std": 35.086502770569304, "ties": 49.0, "wilson_high": 0.3563415432992672, "wilson_low": 0.3269626002235683, "win_rate": 0.3415, "wins": 1366.0} +{"ci95_high": 0.3559042062015777, "ci95_low": -0.32834170620157765, "deltas_npy": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate1/delta_open_deltas.npy", "event": "gate1_delta_open", "histogram": {"counts": [107, 1643, 5209, 7192, 10627, 5447, 1662, 113], "edges": [-200.0, -100.0, -50.0, -20.0, 0.0, 20.0, 50.0, 100.0, 200.0]}, "histogram_png": "docs/reports/gates-1-2-delta-open-hist.png", "judgment": "near_zero", "mean_delta": 0.01378125, "pairs": 64, "quantiles": {"max": 149.0, "min": -167.0, "p05": -52.0, "p25": -17.0, "p50": 0.0, "p75": 17.0, "p95": 51.0}, "samples": 32000, "states": 500, "std_delta": 31.225479356323433} +{"checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_094311_gates-1-2-long-random-exploiter/latest", "event": "gate2_exploiter", "exploiter": "long_random", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_094311_gates-1-2-long-random-exploiter/eval_vs_league_v1_update_500_duplicate.json", "losses": 1610.0, "max_steps_rate": 0.0, "mean_game_length": 50.74375, "mean_score_diff": 10.95125, "notes": "random init + shaping anneal, extended budget", "opened_colors_per_game": 4.99575, "play_action_rate": 0.8010700772505124, "positive_expeditions_per_game": 2.53625, "resume": null, "score_diff_ci95_high": 12.423541316332845, "score_diff_ci95_low": 9.478958683667155, "score_diff_std": 47.50897440589277, "ties": 42.0, "train_dir": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_094311_gates-1-2-long-random-exploiter", "wilson_high": 0.6021679468487495, "wilson_low": 0.5716651100188423, "win_rate": 0.587, "wins": 2348.0} +{"checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_105902_gates-1-2-warmstart-gate3-exploiter/latest", "event": "gate2_exploiter", "exploiter": "warmstart_gate3", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_105902_gates-1-2-warmstart-gate3-exploiter/eval_vs_league_v1_update_500_duplicate.json", "losses": 1605.0, "max_steps_rate": 0.0, "mean_game_length": 50.8715, "mean_score_diff": 11.38075, "notes": "ladder v2 gate-3 warm start, shaping disabled", "opened_colors_per_game": 4.9945, "play_action_rate": 0.798003736847281, "positive_expeditions_per_game": 2.5195, "resume": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/ladder-v2/2026-07-05_013223_jax-ppo-ladder-v2-expert/latest", "score_diff_ci95_high": 12.872234578510747, "score_diff_ci95_low": 9.889265421489254, "score_diff_std": 48.128316645747006, "ties": 35.0, "train_dir": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_105902_gates-1-2-warmstart-gate3-exploiter", "wilson_high": 0.6051483732088389, "wilson_low": 0.5746789269990148, "win_rate": 0.59, "wins": 2360.0} +{"checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_115602_gates-1-2-replay-exploiter/latest", "event": "gate2_exploiter", "exploiter": "replay_exploiter", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_115602_gates-1-2-replay-exploiter/eval_vs_league_v1_update_500_duplicate.json", "losses": 1638.0, "max_steps_rate": 0.0, "mean_game_length": 51.4305, "mean_score_diff": 10.42525, "notes": "league v1 exploiter warm start, shaping disabled", "opened_colors_per_game": 4.99775, "play_action_rate": 0.7904596181514173, "positive_expeditions_per_game": 2.54225, "resume": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/exploiters/2026-07-05_060727_jax-ppo-league-v1-cycle-1-exploiter/latest", "score_diff_ci95_high": 11.913719629678091, "score_diff_ci95_low": 8.936780370321909, "score_diff_std": 48.0310280688623, "ties": 34.0, "train_dir": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_115602_gates-1-2-replay-exploiter", "wilson_high": 0.5971992743148633, "wilson_low": 0.5666433769856256, "win_rate": 0.582, "wins": 2328.0} +{"event": "gate2_judgment", "passed": false, "threshold": 0.55, "worst_exploiter": "warmstart_gate3", "worst_win_rate": 0.59} +{"elapsed_seconds": 11425.08623591601, "event": "gates_complete", "run_dir": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2"} +{"cycle": 1, "event": "gate2c_league_cycle", "league_config": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/league_config.yaml", "league_run_dir": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01", "target_checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/snapshots/cycle_01_update_000500", "target_config": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/main_ppo_config.json", "worst_protocol": "warmstart_gate3"} +{"cycle": 1, "event": "gate2c_guard_expert", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/guard_vs_heuristic_expert.json", "losses": 1188.0, "max_steps_rate": 0.0, "mean_game_length": 54.61975, "mean_score_diff": 19.654, "opened_colors_per_game": 4.7015, "passed": true, "play_action_rate": 0.6708126417051431, "positive_expeditions_per_game": 2.1505, "score_diff_ci95_high": 20.81558482366198, "score_diff_ci95_low": 18.49241517633802, "score_diff_std": 37.482869759149246, "target_checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/snapshots/cycle_01_update_000500", "target_config": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/main_ppo_config.json", "ties": 41.0, "wilson_high": 0.7068566468456556, "wilson_low": 0.6782734877661645, "win_rate": 0.69275, "wins": 2771.0} +{"checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/battery/2026-07-05_143807_gates-1-2-long-random-exploiter/latest", "cycle": 1, "event": "gate2c_exploiter", "exploiter": "long_random", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/battery/2026-07-05_143807_gates-1-2-long-random-exploiter/eval_vs_repair_cycle_01_duplicate.json", "losses": 1810.0, "max_steps_rate": 0.0, "mean_game_length": 51.4185, "mean_score_diff": 6.27925, "notes": "random init + shaping anneal, extended budget", "opened_colors_per_game": 4.99425, "play_action_rate": 0.7816023796562719, "positive_expeditions_per_game": 2.52, "resume": null, "score_diff_ci95_high": 7.76660187726116, "score_diff_ci95_low": 4.79189812273884, "score_diff_std": 47.99495961530356, "target_checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/snapshots/cycle_01_update_000500", "target_config": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/main_ppo_config.json", "target_name": "repair_cycle_01", "ties": 34.0, "train_dir": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/battery/2026-07-05_143807_gates-1-2-long-random-exploiter", "wilson_high": 0.5544028831760525, "wilson_low": 0.5235222802473508, "win_rate": 0.539, "wins": 2156.0} +{"checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/battery/2026-07-05_155430_gates-1-2-warmstart-gate3-exploiter/latest", "cycle": 1, "event": "gate2c_exploiter", "exploiter": "warmstart_gate3", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/battery/2026-07-05_155430_gates-1-2-warmstart-gate3-exploiter/eval_vs_repair_cycle_01_duplicate.json", "losses": 1763.0, "max_steps_rate": 0.0, "mean_game_length": 50.434, "mean_score_diff": 7.0285, "notes": "ladder v2 gate-3 warm start, shaping disabled", "opened_colors_per_game": 4.9935, "play_action_rate": 0.8006960073764884, "positive_expeditions_per_game": 2.4845, "resume": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/ladder-v2/2026-07-05_013223_jax-ppo-ladder-v2-expert/latest", "score_diff_ci95_high": 8.490821414519859, "score_diff_ci95_low": 5.5661785854801415, "score_diff_std": 47.18725831288327, "target_checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/snapshots/cycle_01_update_000500", "target_config": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/main_ppo_config.json", "target_name": "repair_cycle_01", "ties": 41.0, "train_dir": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/battery/2026-07-05_155430_gates-1-2-warmstart-gate3-exploiter", "wilson_high": 0.5643659496573598, "wilson_low": 0.5335400249002497, "win_rate": 0.549, "wins": 2196.0} +{"checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/battery/2026-07-05_165151_gates-1-2-replay-exploiter/latest", "cycle": 1, "event": "gate2c_exploiter", "exploiter": "replay_exploiter", "games": 4000, "json": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/battery/2026-07-05_165151_gates-1-2-replay-exploiter/eval_vs_repair_cycle_01_duplicate.json", "losses": 1798.0, "max_steps_rate": 0.0, "mean_game_length": 51.116, "mean_score_diff": 6.10225, "notes": "league v1 exploiter warm start, shaping disabled", "opened_colors_per_game": 4.99425, "play_action_rate": 0.7845902569268536, "positive_expeditions_per_game": 2.50025, "resume": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/exploiters/2026-07-05_060727_jax-ppo-league-v1-cycle-1-exploiter/latest", "score_diff_ci95_high": 7.583110013023179, "score_diff_ci95_low": 4.62138998697682, "score_diff_std": 47.785475385853, "target_checkpoint": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/snapshots/cycle_01_update_000500", "target_config": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/main_ppo_config.json", "target_name": "repair_cycle_01", "ties": 45.0, "train_dir": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/cycle_01/battery/2026-07-05_165151_gates-1-2-replay-exploiter", "wilson_high": 0.5546520360903574, "wilson_low": 0.523772647611401, "win_rate": 0.53925, "wins": 2157.0} +{"cycle": 1, "event": "gate2c_judgment", "passed": true, "threshold": 0.55, "worst_exploiter": "warmstart_gate3", "worst_win_rate": 0.549} +{"elapsed_seconds_total": 28945.184503759025, "event": "gates_repair_complete", "run_dir": "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2"} diff --git a/docs/reports/gates-1-2-2026-07-05.md b/docs/reports/gates-1-2-2026-07-05.md new file mode 100644 index 0000000..a28dea7 --- /dev/null +++ b/docs/reports/gates-1-2-2026-07-05.md @@ -0,0 +1,67 @@ +# Gates 1-2 Report - 2026-07-05 + +**Run dir:** `/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2`. +**Target:** `/mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/snapshots/cycle_01_update_000500`. + +## Gate 1A - Policy Class Tournament + +| Learner | Opponent | Win rate | Mean diff | CI low | Opened colors | Max-step | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| `expert_cap2` | `expert_cap3` | 0.4415 | -1.8933 | -2.3662 | 1.7792 | 0.0000 | +| `expert_cap2` | `expert_capN` | 0.4445 | -1.8807 | -2.3545 | 1.7815 | 0.0000 | +| `expert_cap3` | `expert_capN` | 0.4840 | +0.0285 | -0.4689 | 2.2153 | 0.0000 | + +## Gate 1A - Variants vs League v1 + +| Learner | Opponent | Win rate | Mean diff | CI low | Opened colors | Max-step | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| `expert_cap2` | `league_v1_update_500` | 0.2725 | -20.8863 | -21.9846 | 1.9735 | 0.0003 | +| `expert_cap3` | `league_v1_update_500` | 0.3400 | -14.3513 | -15.4459 | 2.7385 | 0.0000 | +| `expert_capN` | `league_v1_update_500` | 0.3415 | -13.9670 | -15.0543 | 3.0040 | 0.0000 | + +## Gate 1B - Delta Open Audit + +- States: 500 +- Paired samples: 32000 +- Mean delta: +0.0138 +- CI95: [-0.3283, +0.3559] +- Judgment: `near_zero` +- Histogram: `docs/reports/gates-1-2-delta-open-hist.png` + +## Gate 1C - Selectivity Judgment + +통념 기각/미결: capN은 집중 변형보다 유의하게 나쁘지 않고, 리그 정책의 4번째+ 오픈 delta는 0 근처다. selectivity는 현재 주요 성능 병목으로 보이지 않는다. + +## Gate 2A - Strengthened Exploiter Battery + +| Exploiter | Win rate | Mean diff | CI low | Opened colors | Max-step | Run | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| `long_random` | 0.5870 | +10.9512 | +9.4790 | 4.9958 | 0.0000 | `/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_094311_gates-1-2-long-random-exploiter` | +| `warmstart_gate3` | 0.5900 | +11.3808 | +9.8893 | 4.9945 | 0.0000 | `/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_105902_gates-1-2-warmstart-gate3-exploiter` | +| `replay_exploiter` | 0.5820 | +10.4253 | +8.9368 | 4.9977 | 0.0000 | `/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/exploiters/2026-07-05_115602_gates-1-2-replay-exploiter` | + +## Gate 2B - Robustness Judgment + +보수 필요: worst exploiter `warmstart_gate3` win rate 0.5900 vs threshold 0.5500. + +## Gate 2C - Conditional Repair League + +| Cycle | Worst protocol used | Guard pass | Battery worst | Passed | Checkpoint | +| ---: | --- | ---: | ---: | ---: | --- | +| 1 | `warmstart_gate3` | True | 0.5490 | True | `/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/2026-07-05_094238_gates-1-2/gate2c/league/2026-07-05_125709_jax-ppo-gates-1-2-repair-c01/snapshots/cycle_01_update_000500` | + +| Cycle | Exploiter | Win rate | Mean diff | CI low | Opened colors | Max-step | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | +| 1 | `long_random` | 0.5390 | +6.2793 | +4.7919 | 4.9943 | 0.0000 | +| 1 | `warmstart_gate3` | 0.5490 | +7.0285 | +5.5662 | 4.9935 | 0.0000 | +| 1 | `replay_exploiter` | 0.5393 | +6.1022 | +4.6214 | 4.9943 | 0.0000 | + +## Human Play Recommendation + +조건부 예: 강화 exploiter 관문은 통과했다. 다만 selectivity 관문이 미결/부분 지지이면 인간 대전은 실력 인증이 아니라 행동 양식 진단으로 시작해야 한다. + +## Decisions + +- Existing `heuristic_expert` remains unchanged. The cap variants add only a hard `max_open_colors` gate around new-color openings. +- `expert_capN` means no hard cap; EV thresholds and the existing soft concentration penalties are retained. +- Warm-started exploiters use shaping coefficient 0 to measure target-specific exploitation without reintroducing early shaping rewards. diff --git a/docs/reports/gates-1-2-delta-open-hist.png b/docs/reports/gates-1-2-delta-open-hist.png new file mode 100644 index 0000000000000000000000000000000000000000..574f56a1bf228d9a29a4df2a8a8140671c7e4745 GIT binary patch literal 19387 zcmb`vbzD_h`#!wEE*OJRP{C0w5Kw6(Y&~?Bgo<=a2~sAbVxfpiDo7nd1tg>tMWrRB zQKVDp{#^@a=9!r%zwhV$gVB+5&fa^)eP6ZC6-D_Y%NDaOrYLF|?dU;eiemJnD2Aqm z^YJf&AM3v24>9{gn)WKzM)po8P8(8kC+ux3tnDpKPi}BDJZ)!cZN<;KYbUShjtwUE z_BM86e0-LFyn@&Iv@ze7!)03dltng2wd^R0`2_i4h?I&jr6`%Lw1fLp&jj?;J8Rw@ znx6T#dSWAE+z!*D;g=`aPMAL6yyy5lCbWgdQ}%O7c}(Kz5Avqk+V@NY<-f$Mm^x_7 z77EDn9OS=RXCv`Y`>`+M{=D(Asn2hx98TvP4>&h5mJ*j5_h4eWz%5IPKd1j;afb66 z18>~Q@wGv5y!ew^y_(@Cipn-x=!yRkIzY{(sHd!q8z{=N@F$9aqAV^`3-NOPxqI<@ z|6eFZiu$zl-@fI{%QVl6M3R*r>1#Q>jEQLr#VC?I>Dn>grO;5Ef5)uc z#!`ZczL=GEjdj5}-2Q*K@!y`cESvJ&)Bo-E7Wv7n399KJp4w(XW@c#^EiR7HQ|{@a zHDs*TFwCW#6t%ewje^oSR&JWQbnWGwz0S)ihV1q0qDJIyFnyV?7HcT?u$$8n?HpT<_R zq@}S@73==({QYIhbrv&Gb?1_k2TKl&dzeZ^y}2Xviou5WFnP|=FR1x_dEMzJ$cv8%f^Ja zxB6SFpE?yLE-pU!VQa8RfpD(L>x+x7{`#xp-o2h*nwyjC+T^{2j@(%H(UW1$s&(sP zN=g)$vF<+T?Ci{BDP&a_$*Y_8zOhk%*;=9UMAI^Pddmf7$r2U~onJgWJd(^Rq-Feg zV+W_Pgmo;eR#O850~O(NYYT%UCcFnCr%C|+%wF|BOhJ7I5$wl_O%)R z@odVn`uQPl^_Y?`Z?7d86|&(91rLM${psuyF59@cq&qq~9!k2UZIOEzCfsl)$1K6I zN4ws+Es$;ZsbAekKQ9>SC_CadKKQY<&_A%s8O!}zcFvj+iMoQEoC6#llh4KnJ2rKE zZP&-svpauXew~&7u+QB)cS4^&-MM+|R=UT`bY4M$>OgB@(uth;4&9%Wd=f3Hch}pO z$%)xCd2ZaaNzk}xoy6GJJ9&9|KJoGKoTlBMo{fLKqmXD>D@0Sya-X^z9321l+UjFN zIt~?TX0dvwJqg!6}0@H4+YTn?K_8!yxt0%vo~CN zUUp2(4w|y2=KY*OJh{1^y86A&s`$R0jkUGMlFX|dP3*!A1JZ6JByIQi*rF0Ht910} zuMXy)o_@aMxZG3M`K-)^`5ig;>Qh&(qax_a3?=o~*9yyvii%cRqPWEIF50X1xozv_ zt>&`vN?6>UWol8>8g_PGnwWk2t(?IFUQAj8uXA!7M*B~AjlF&QR_yfGUk~{6^E8(` z1jNc!Px5~o7*MgXd785fRiIuKZ(hGXw!h!zNm!U>8J;7_eaZ!IqxpRK^5t$|VC>+a zJ-sl`ms?F+JNnZnO?u%&L334A)l*4Y&rV*#-@~<&%(Q+V3~KL_-G}pvHTHg9#5-?6 zc1ImA)!Ndsebc4`tgNg)^6aCxZ{L38HZlAGOJ>uN8|6|nV+uUliAVi~EK9S`&0%3> zHC?}b-O81rK@w-p+Djgd57%1zK71G*7REW$*Qk$o`-HvhO_RyY%p|Mwg+=w_OG{RF zw)p8+wtozb6=QcFJ;8JAg)Cp@=;Ifbwc)foTRL9$WvVDEm%eqUQJ-FMcpTlgFE7A( z52sF|r&rpOZ2lqq0@u*jWKpVZBgKfKKjJ4jy7!8KUrO2vN+`Qn z_OI=(Zl8PR*UyhKibo`!B~XSp{%Dm*s6N?*bVw#+KvRTHS z4?}6Qyq|yNUpuhVtzf>2$VajK5VCi0jI7v55jRrbJ*G;KV-awkUcPi?+;gOq->hC)X|?-yko zD&U42D!uOHuzzPt+I;Fo_QDN+efi%}!4ZZz7{|f8VvglE7o1KZoACE*3R#r!V8>-2 z%0>_SqwW(NQAgI*b+!(F(UH8YUuPT4Fln#Hy$8z_%r<<;Nl9BnRjl9SiMRa+c(QKZ zQmQvQ=YUy*&K{p@IZigv+&QEK$Syn%0{|^`E(KZ?LXSV%gq|v`#SgEOye<*tQ zzddvRt;dh;o7!#vF~5v+o@AVLST;MqzoCX`9cst+?H@czRNTR&|=i;sf5^ zV_EzWYz6`+Ue7&r@Zh!$8}@$t_N^w>E+Xgjjdh}r8RpF2CL*Hw=+PsxZ@+*4zOKfi zE=5T<&Hi>)Ru-F(MWkt2xb9s^ta=d@-zA>KQZwnT1rI_s^xf1RJb2(TJ=P{@StF3~ zJ&#-6A1^hnjL~|y$0@EdUO%2q&}`c;zwFy8KiSygJAdh#5bP|Iw^v!T)6YD^%R*1& zT-F*ukEjSe!hGn^q3se9x@8dxRbjL{gSl(1b;_q1pI&Kx_e4H$uwzo`J2#h&gCG?x z%52wJqQz43(#iVLrAr-+Gt)umakv8 z?bspv;lqc{#;mN}{QMQ^&UTicp6*+}$MG&FDM`Zh_IH@piG%<=7CzuVf(j7!{Q2{* zU0vKXwmpt9l4D<2TB4)rB(=7-a-kui;xA@miuti^u1if&R#r|lc*jI5y(u*vX`MNC7(G0c)AS#0ZYbc^t)JfJ z=5q1y$obs8cP|{kgY-0XKye8PQugU`RdKqp{aa~^7A>mDbbEONz#z9~@#4jI{r$Ua z0Dx_T)q1xzJu`U6B`mC=p<{a~Jw07jS^2SXu$%gv`Actc8X6kXd|tnPy)J0=&p#U# z=A*$*jSsDFJ$v>n-La>J$?|b%sKT+s)#+#&f$Ft)x2p_dhiL5Q14s{u$h%Hu&P-3Z zv;{g1_P*@voaoD{nj9M#dcB0tWO{Nej#sWZ_qvL_{0)cNERQU@3|TC4czoBvk4FR@ zyERFR@F|a0~Jm^mCZt!!| z9)*RSJ?`$^Rp#ziRFEA{jfpbD0?Yt55@%dTI&dNoD} z2uu+jqbOx;HA;MR+9_sScK7bx zu-^KY`DJA)=O(fcFntfT z1+!H=@^l+RjPz{pt9bu`;TGCo#f2U;L=@|*;A@_HMBH`Kdc}CoIPIA2+-g=Fd#T8K z(lSpf=nlS|3d02qO{cv73$e3VwUbJ@mc4gMN%c!77xjrErF-9XHf8tBhjyrxs~P+~ z|H>9^7HpM2n&HwB%e>`7d`d}01r^!)X#sf`?MkA3RGi;eE$ts9<-c9~&)h6h>A>#8 z*U?)%uX!%t`=?fd%ZCI6q&io|rzrCOoE2H^b9lu4;Jk)R`D3do(lV{G6B1Q3UPMG! z4XgfMr~hM1`N4QUi=+9@zcDeo-KgBF(v7m&g3g~Zhnac($dHo*3uSR$E}OKb9CV5l zhQ==&B}U%TeZsb`@KJ5=Y5sDRQZ>f&d9Ll};`!L)uVO#Zy2{!2UP{`!|CxpbkU`6~ z<&a9^n7-ivN_B-$D)hP|j0bdd=TZK@_3YkHM;sZs#?rg?iwJ&ean|GM=w!C^>ys6>v-x7IBb&b$$*-#ZekL7 zc`57T^mK7Ap|!$Rs(!q>U;FyvFE3kbR+DJD`iTeTsz}w8q5pY4Z&_c~jC;*scejPE zW3{1g65DzS7gco4Bv)70!Lhbr^RCKRmv3)Zw!#Tw4kNWAkM4@wV83>Z@rmUhe zG*qP@Y@503FhIP^*V{(~j0)D!lu^vlF75^d#7s|3aP#x46#5HS;X%uv862UNqP~C~ z3U`1ST=n)Y4-mB*1Rdn!<|YK7GEM5=^vsJ5b8Ma*Ht-X%tY!~a^_zL2?~_QkI>5@@ zW)V`oC&^Fc-jb#-(I&Z({Z~R(^Jt%{pZRi)Z`FzwCnb2dZ *z6wO4`msVJh7Yj9 zsm|6*Jzg(XhBx&fv&7)7%S`LJXf+AOH_^R85v?V*{=ptqK+w<{{<*P|LXJM5@{jgP z0FA?7yOP)Pcf04!o5!?ISXel@bbMKe3qN~MjTT=X^rWX(7B9$_bQ?eF12I9sq-2Aa zv!tXXNd0X+*KcpH#^rv`nwe6tYc2Q)?lbgerGiP$zIoA~3q_VlLPKO9#msl7gxm=~M-n~DZ z$!Y3X*EmaLqZ0#Vhr6qr2Z1iI^3593Q*#pU-(S;Ho2=lo3iP$(Yo=EOp|FB{zqRK3 z`Z_u>*cBp)qGh|~b>ESeyj?`()93kA%V%BX$G4;UOiy*5{lS6H%L`m9>LIs!ZCNkjx4JlZGBms=wmIv}(Gb^oowuF61<-@l2D$+!YtLKv)O6`lbG zQDyNJVjKnJpdY!(PVT@uYbqQPfmj_^exy=pqE)?^#7Lc8O_I4^gWHI1#<$$HV2O9_ zPwT(Az3E72@XTby&pg^;a{fZ5P1)zL&=ZF>y#K#+^*t6}=22ASRW8f9Hjsd%<>f2u zof>tuu8D`r2a0tx-;}CMv27M`{(kDvkt1DC{Pap8LH+y2-e;Wz_2>WC*Qvc~s-|e> zDiJp_r-qM#9;EF4S%U{fIK;)a)tQ>JSYex8YmZOZeI<_0l=vCvpl?;Pzl_9wOq9zW zsdoy-DAng_uS0bS2rCt02o|y3Z=L?_a^~cBnIxUUuT;^(cMWmFJ# z#2ab#>COo`;>^r9^R$;ysxtE$B*$yBSxLJK$7}T6hSUss7E=}x3#&BZBcRLrC z^5Mgmu?xDtn-4O2z8v7){&6L}mFCl6RdtP8m>tE%Llp&x4tSSsg_s7AOzw!HxZZb6 zP}9vEGm(j16fPU`C4ikOD=l5e`)WRZT@{VzPp)_L&oMCYW?~nQ?8}^xZ+>?(KBfY{ z4LOewy?pVai1_f}peg_)na3gQ{O9WUD9^>wiIj+S1Fp0RFAj(-Wjy`>(v6{!k+hF? zqRAtmZ?$1ADx_w6|FF``p5TPuJu;!sWCC7vXPH48i{oW@8i>aOCRRrkgU-lx>GEZ1 zA0Szj0AU`@r+bGwE9fz5411GQIy;^?_`Pr>t@Cl|4MuiJ*Aya)gXOfpzrBeFPNYDf z9$a6u>+v7Aw>;Z0|6WHvUzVz@thZpS^FZrEiL*k`w6mSh_^s+H;_$=xnU|@&)$`5t zVxly;de(+x^GQlZ^d&<#4J|D#{V}nUo8P(^z15+Wdhs+b@}vTyu}1VgriNTbKYI<~ z^EJ!dr^k%Z+obDXI{j(zGRHgVd4+{)p@*-Rr`Q@0=KAsDNBm0#cYM0C79G%U?>w~{ zq08t$v)sc^-#rSP48-f?NtAkhEjosF@8-zZOwy}C0u87;W9Hv97+#e zB|_fp|lo+4=WTMUex4F&3Y?mc(dnJvH7H&!UT!Zhomj zGOca+%Ir!<%OIV~U)Z`5s+fE37y%Fm4<0nED~!9iXr)^V7U|wD~?mThgg!W6vSa#R`eMH90T|@Kn^72}- zVuirjVG9szwb{2~rwzR>_2Z<;Jk*7I)1>PAec4JDmM0jTPONZJIlXt8m4;n0tB6)( zy|#eNLUeiXeJoSu2>I?eODU?Uh^t3C8#f)v_AgMGO%MYxAjZk1{ob=b0PwjL3Vb{n zU)VJK7_zr~nu||4&i^~kReF6xfL6L%u37a@R;U~|5 z-;|Mman-*V5drh(Q+0c9`z7br36!QWQ$=r*mr^O`PV!m1ed1~$GqOmC0XzBAQpiM2 z^V6ke${mVFe8L1j_RMxrBJi#4=ix0)XAZvi<_YWyj=!^&zi3i_*kL)Q$EoaPtDK$9 zCzmm6jxc0D7`?|p2`hHD#3t8{%P!lHcMbOOjwz6b4J_Lw$L;^ThcI zBk*DJt2BqMaBuIWggukCk~#Y2+Oz-#Av18Tk?|}y40uIq-K<`0UVmI3wJ&+4*J+#ouH0Fzu>wj9Q0Nt+t36|`%W^&2Eq|G8>4bsxi~m*KKqQnKNMi;GJd6oGZ& z`nGRb(gsUqfE~Bc2u`G{$F2foOQIb;st9QTGWW@(v4Q6+SFDIJ#O>0UZ$;E@F#EL(eGl*5>L29Vy>f4$$VN{ zTRWdUYmO<04{$F%BZKAVpTl3j-Vb?Tk2sN97Be&ZabEKB3U*Yrscs%ff)gs)+#byN zXGg>LBOb|9TCGKh!Ktr77f!(G`;NsDBkfnY<4OQo;_!TV@7^5&y6db>hAA{L+=Gh; z!=B)z+1lE=U~v$$qat1(F#qJ+#qfaWXfEphs2*Z5#(-*-yNliw7c02CyZcsZP{T^y zEl=9wUcMunP_>2eI2?_N=Y}#~rh;bWih*JdF&zgkFC~~-?WLn7KFxfz|MS&r*UHp1 zbT`vFKZbfe6t<3oqJOL`T#gKMkW1(G&0Dr;GxVNYz+OU+&y0S@epYs*;cUEmoKDn8 zqldoCy&b73>*pU|a!60oWz&mzdOmf0r5Z1F?_I$I{+5o8s7$ws=#Y@T(A`{mlB#0H z^z`(IdP&MAyqKuN*V)(vOm1ya2(m~5e694^EF*w@kumu_A8zxx2G;?(&77Pmk7Y2M zzPTV|I=TlPeBtat@gvG4P;*5Kr6wQxY?#${WCxE@{(+a9Y5qBEwVlc~XCsDBJ1pAk ztMbuXiRTE#5EA{3o454wjIg%PD*ny`>qn9lsJS-Z)KWuhw3$q!v{D zkQyLNSY>BLDmE6Z-*|<7ch@4j~1x53gG^QwHZxDHZtQD@tGLKjJ}+y7l? zm}8(iaVCzl~k|5nm0b}%iu$-hc<)1yBmMED}I>ZBFjiCK{s>z z^!(*NhPldG=ns)rRS!UdT zU0e7=TX_-`q+Qx=)+I@atR#}!-gC+ED<-^uXe+@FuqgjzdZe85e7&+}!qvLREi;*M zvBLjFMZ%Vmf3W)y_NX#8ty-458}X!TldYXh?7qBNgiRO?q667MxiQlX_4^kuAyf;M zc)g5OYt}@L4|R2YJi?&{#rJ?e^pGetE(n@sz)5j1{fwQMJ>`5|T+CVw*C%~{e&YwR zOtpScLD%5kwMz*?O4(HwZg`(z#4wkVYBb~j|G7EU=u^XrO+ECpefPu252Lj{f+ryl z2X^MJ(O9jtJZpNK4lg+#cSJX@ip%@(K`Fy!XjUtSQR>GDuqwA}*DkaWQmmf9MA6AO zJ5tG)HTf7;G_ABQ)ouvdau>R*FAR2ek4ak|t>*+RzNoCo^6)_6Hm`ne*b>bHeIME_ zxqyw0O^XwIWGmZs-7lRhY?gNi*5p@gMltzeJxnIRcBd)#)Ff@Dp*3)U8bQ*kXeHGf z3={NOfTpai{Sbn`;Uy`W^q)GKr~|8^Wgf5yrKskN%+1wpJUkU35Dr5X8gy_4pD;M- zW8>po@Z%p@!cd3e5+f9GIqUhD;XqNv7ThBstH$2F*``CQR;`K*3CVT|@b^FVjEV8- zbu2Pz4+l@qr~{>Y#Cp5>W+9xK%iNuqNdk_~&s(CSdo}FNg2;;(FKQ;E$)u%mQffz! zE1hG!wz#PACyFaFZ+6f2j_fGFeu_nO=qajN$Ai8+%@>&4#xXl<2=P(sYH zYSC!!aH8hJg>gt51rR9c1^WMyT`iOH-W zHKmFaO5$+!l0}P@kT}`Df4{(~_di`{m(WJ|Kpn6Ke_rL{+f%)bU6a2u}b6bcNoOeBOH2+1d2lm$z^LiBU5B?WTtb zM30B!&d)0#N@=50e#9m#N2DbKaLr|)P}uj@EgPkYC0HfZ(RB-tJIfzyhZtIoQ);<`Y)Cm*FPE5a@vl+((@S<3!gDV7r!F*F4bMc^V$V`tWDHmgC3{z zQyD5}Q`6>tLKZ1i4T|j+4u}9Xeb#(l2&upBPn`fN|Pp{i-U)g7M>|Othi9X-kD>1a_$v;TXO3CR? z+eY&^-Z_6{!T!Nrgq!s3_vbgd9>veBku63>NVw@lSa??7Y&ZR1FKWLbCG(F~)O7J? zR-AJ`&%b+CZmyDufg>GdW1(L)kgYn6)E5~?tL(BhNgrxvC-3Ck{}S;7Jjs@uWdEtV za|QKjF6l$ri2gXN`4%?|w=5Xv|0NZey=;_)#pKmXgQTYVXS20_c9t0w4D2hW3ulK5 zvj2a~4vH*ZL)E?EkaUN7;wgtUl)Y@-$dJas#~Ft|_1xd$h#+^ERBqn#fSbX0I3><> zt>o$-cCbA}Mf0PVrZ5zjD(lW<_02Ai-|zLfXhD#~F(?^6Fv(9E7!36!H)d8kogUF3(&yvN9RTUMFOB8+h0AIYdRV=Bf^t@C+nk!YL zp?qe)VdJXJ^E@KUZ8&!-R)`MFcvL^XzRROyY;Lx+fMMP$33#^=a@L=oK3WpZpjdJ7 zZoRW*2zNfENZYS>dRmO=Ot)^`8W5gb$=D=X4nq-8`H$oiT;0jUVU*u8p!e8!$Z`=e zsF)Y}=_w85#}QpHkDFsmaN&ZEbBkc?1LoH>z@hCDgXwj9%*YV!^lt^I$c5 zyIspT_a+a+g`RD@Tn0^+u?ZX{3c9a`UKE@W8l>#BwxF{*x@=@D_fz6G-KWMo(3UG; zwchpheS(oRLnn?{zXs4#QSB!x|CjCCJG}i}>(2!T2XB!NI8zhe*6A%-JLmUHPMx`* zD2OKNoZOw;IIobnz$1VZ>N!Z#N-^6qNgiBDRk~px4(}{!my`kTrc zMxelnB}PfR1;HPI&=CfT8;wcoEQ`2_2w-nXxOi~OcULd{rE{~njzbPWtfHbTJ}Zud zPmo+8VnEQDFN^4MAn-1O?T;cOx4~C1$3j8?%%|Z@^D=rNqy+~k>}Kz8{WAX9i=#Y3 zfr9{A;1kC~$wl<1PZf##1Qz(p?gjoa2%@k_eV%*`5AE!5w-R72Hh{3c8*#Gm0#!}TNc4R(#3zUyFl*$i#TFayhIs5X0+0W5W%}%(BSvQi z8;2|iWH8PqnU);}8zl&YE<+bUM$hJqLe^O8TJr{28iUow!5t))ib*AkYx|2@E4vd( zt9hlQ^pQ^>ib%1D2fRw63y|6})Kx_Xi3R#Afp!!lM1+@nJjV!KC-90)9I|%2>Z^G_ zjg9Z1Nt_+FKhxNFwok%qOf9OIt`V;eL{b@hEP{nwJu))#sD#U)qPu%$MXT$NNf4N? zRtj%Z7N}}yG^t9DAIc!{)hZK_+Ngu&S>Q4`k~LF5*jbT(LKJ>ZEA&?<_g5@CtsBH4 ztDMet_9?(iT`Sup0d>L;UHh4-41B>DN+s8`*LhAaB>8mH%FnpCJOm~+uYnUO3n(?z zpiv>|cjfBU-TcNy;$4%oBMC9NloN8mO&MuvS)?=`U;!MF1DCEu>%Mi~^XE#l&JvcZz|5AOvAS^|d^obAgn_TFU#26EV^ z`ss-a6|DosQo`))?5%CV9+@UdGit`;yzEd2Qd{mFD;f`oZfMK#VqQx#f;5wR|GV40 z{DJlE&vr6u9)cPW2`iwo8Mxd5mH_cl6Oj?NY|PY!dzDo4=H0tJJGR32G&MC{9bvMS zhCE|^nu9}&uSREVa)Xa(OTpoa*kb$^TC8u*mG>Au#DnNq_L`Q_F2NDG!j{{GA|N0>fx;_iYWJ|tZ*CQ`>Wj(#1!`^Ftp&9qsVWZ#R zkcvZ{(YQP17Z@u=SW0;Gp!1twmzyP7gNLHOCO5jP z&}bi68xug_QX`}D5E!C^kpjreE?{OaFH?|;MJn6`6yQfRy3ceXU45u8T*Y^&S*I}z zw@w;Jq;PhNbb}s{o7$It`GV7bm@^)I4dN8TcM#J zi;XC)oci0qEcE0RK$J>iW^$m-;?9Ey9ZwSm!6Rc(4m)q`=`De(7pQyk$pp8Xq~EvZ zcgIF*lguL~Ctc;RUKe}zhCH9pVbOhWz7tK)TpJ6vrK^h$l~NyKqHQgIXLB97jV^cm zhCC#SBB2vjGZ*H1=%OvhPGH=X@yiMrH25oVaGXvUviLzZMKWMdcfNMTPn2%^?IY|T zn_gWgu5LvHBB6H%qHt$rtiPC3uWJocA4plmdQ;V55x{`}%e+?orK@Y&mXPZi2RR=` zSpe(bV;+3r!Uet>si3@WyEA7-O^3!qHGMuR1R7LjL&jN>V><1I6$~*s zFF!xO#lwHK3IZArg)F17#yc*rwNm_z5FVC~R+-5kw}tM22&85i^Lnu`^l@#=Y1a~k zA2Bm!OFgfA-j}k-81wqtIm=~7m&Z-s-W<$Uby+o7h=CeqaY#};1foQBVuNiTb_a6AaV6ffgV2A=+tasWJkKY0-ny z4o%q%Q>fYM3**e(gmO*LxJXJ4Wj?yh@zC~S<5cm+&#My-*w`&#TDKF%S87K?#!FG0 zD-gw2bKt&%N{WqK06*Lk=A^~XJ2a02jTE9|TP4xtMynTx`%bCJ{tKdx-S>Cp-bb@|;4#&uZv@J2 z1bER(8W21XBb@uF+&zuH^8na-pgr{F!-4`?ei&cPEiH!Fb*<4>fn{UkIyyQQ+Al0j zff7#a-o3j3sTWzuItB3RV+>W3B!kVMt{4GRY}vFar@JAe7D>RFT`2AR$N?tS+-*zH ztJtCWG<$Sx>;bAq0z3+Sg!ERjv31&n9akHEzvV!S3oZRMw&xoSSe3eJ2_G)}lRq_G*4bT{>=(7&BIW zBX63F3`U}Rm%jVh6H^pNQ@DGgwqGtLc}JJGA&gN1S%%!T=p@H9T!0v%wuv2nZF?q9 z{z9oUpb+x=6rI^q0X4-WF13}dxvkBZH}&fZeuvKTvU@{VDE{Gc{%mkW${rtDl5pyS z&$Wu{ON2~HLimTrpwREm4G`De3)Cj3sCX+%DfBYrmg^4y+2AtRY8D|sWe753jv!5; zp`Wf1wt~~QJu32TT?xB#yX#+8)tza+zGcIPY~Zgd@$c_9g-1rdKOqXJ5+SXiAdKp< zg_HBm=){CSA}#}UcEzHTIXlBit#1Vb*hPl>+t84phMw~itz@e}x=geHf>Znm#^hF2 zRq34musK{m%iRcZp~^`F0Q|k!UEU+tnm>2`{D;S^Gb%nWuu|(s=??s6?Nk!7FZceP zi1Vyc?&*t`Ea3aS2vG! zYE0ssT@OY)R;tSNX7v(=pI!)_=|2v5{~o?Osvg)&*)rj}kmLpz0-ad&gl06?6JrE$ z_XRs2g4b{biI+kG71(n@wro_$_vO?HbJ8MTU09Ih)Tif$G#2R}>^MF$)LVZH`db`t z-(&6_JM3$oAK1Tt6&stG1e!te#*G`xQ|*jVz4B{oC%ntL!^6VDUQ6v&bX#L!F{V4W zZ4>pn5{M0nrZ{55qv}u zB<+kUqrJ(S3QM+@REB7HB@1 zK=%$ErH2?87|cMwfodM%N$fK3A<+PT^FEyas9z3TZbw9bX+U8;%X;xMvs&VB$9V{i zYfk*Go#B!cm;POvgtazq+;8yihQnxtbG<_qb`~5_+G5C(z>|U`b>Omn@cem3a!$-o z=kyhFS`7z`y4>!NBRFs}J8`dy$ckX?#&G=9b_eHxKmqb`Hcms=!53BBwdOS9dtum} z`FJ`k_%g72`Yoz{ z97cRr5En*9Mk0Hdd$?LIbeAp$!6)jD|g)<$AZbJJ`Z8+Gy&k)*&m><+%*P4&aJGNPctaE23VY@ zNNWbd@u*~rPzbidaS#(6+%OpKtv5%G#(V%9hRwhmD`*#)c~it|w2&&V#)cg0PAD!Y zeIj#xSFS5O{@k*%GT9*UYKz9q`T~F9;<e_zLZ;BT%11E>TiS&(TY=?0z^{RyCap=P+eVZ1Qrm! z@52060MOR{&pE~c>gvsS>f*FsjPQNZ>PrqVKR~M=`1X#&AGlFWysgGOUXVZsuzVqW zDF4h@v4`yO;~~xn5yS|QbV3Id6>>(P8ex|QU`{xu$4_o9cMU%{D&dO=K^09s_F^%T){?$x4Nk%0ydH3I6DgOufr zQvz~eoz-a$bXFY3Ax#4)tn&Nj2Y{gW0p6;>%*wpi?0P+S-aM_~eDYLKD*S7$UKUz> zdishCZaS1gGCGxKSFBrS!G5NR0bIdU63S4So?q7Em`3zKBfv}}oSO^BiPdmulK%INpUUYP#p2)ItW@{MiJV|!Iqk&d_>o@{A_xJUklaE`Z*y=@ zb;7&OGT=6o7gqIVr2eWT7|9LPT4#ywNCp@;!pS|-Zd>8j3*)Gw5#p1xaPHlW^{p^O zcB7=WMg+S*zsJGCVhAExq-q>~(n_&; zq=o#`uGX(3nt&QZHL+R=xfi{>_N4PhN-p$#8f!N%8>-R+-~xMd)b+~8fPR%7o5SolWJw3rxMz&RRM~5j0t9z~k`rQM9F-~Y(Sagf( z-+91q{1DX4t1ZB^hFnL_xy6@1t_u4WidI>S7Qa`O3AQeo4|rN@4CLo$;{m*c0LM~> z*k^zXb3v6RCkKj|@_T{9S%s|vcD0^TymJb@zU67q)2B}pD%FRJU0}80L8-!S_nuq{Gyu0{vJSe8(;z% zg3iR661}XWD!x(FTr~K(yT@3IUu7ar)vTda8b@LY8sSVSQ}N5@8+!`mgzHs%QK(VW zPJl%ik~3+QJs1+WjNZGpPM{!I-U$x2PQqZ^77u1e6VEF>Koh*br!PIB&TTT` zIVIxIscNpRq;%U9NsOTbPc2e|UB7?On$VmTkv!HaDJivb%#3;sr6eYnm&fZjBJ%ML zb&@a-)Mw{PO2YQb!If*)wD)gnhCMBaozoUOPmS5%^Bn)qTmo}^27wT`kXJ&LSHdcC zs7){~HuWzpGDfXRL#mOSLqZdf7%EpmG^SCjmYH2#6U!K%|A6#yh!w{MKrPBc_RWFK zH|SklSWrN?f6vYy5|Azm+=D|yC*`_6JqtbHkF(rL08LD8axiO6O`|Z=c9jkwVq&t_ zKpD-mcn_kC`1tq~iAf=`5As+!u{p5Y7*YrhxE+^!*%?h=SM%oWTdnxFXoJKdz#n3R zXF*w*`fO44*o3VtByRlrYY76A#kF24`qxlj=%{`;Af6Zw%QqT(lBrk}ac~lvg!H=& zA$!zsFz`J+1E)I&2v}%pI!-6~E&^JB1dU^3!Nf3)TpGbhe7E(Fiz9X2);$uHgie0c zC#gHZO;W*4x2P%$Ek`qnVED?@wTY&osFor-k}*9e1}xCJkQFeIxN+^83La>VR?Q{^4PD6bXf{k>DBLf$!hRv4*#A-fR;PP({7R z^N6>#y}}$qlVA$=!ty6@7q^Oj8;u2iAUJP;^+<(5mPE|C)9-OS9`!Elj($3*0Mo<; zU(unE%LAZFWe94MlXBb;=E#|wkLO{h{_^(^3AVhHE@ZZ_&%3+y5r}!6-U{KDtGZ08Zr~lduK$6ZZ6V& zWK;wGfXGD^8c7|w&rCTHK#i!H54;EDuH^S9w#f?mu0N<8RQx_QXvP>S9#h0lQH@)p z1JCovogOVn37}^pex*IZtzOx9fz#aM6*ghJj|{KveYhr0mE+T99K~OYJ^XNZlK8!Y zIO5k15eH_zr&ZRJqn|swV$yQzoD^ZDXa!*yE}9tYYwXRhJw3X;b5??ytv_7NX$mld z8!Ls@Zz?wd=v?$N2I~TPvjP}Z8PHf9QLd0GFsn~XBIhq*!vxr_+KGT3+CEbmPNq~q zIWF5@6osu5kIZ2w?nBDjG9u0#pTH(!6C!8{z}n*Oe(24cH|;ob5uX!jX=WBpz!lMK zdZyDuC(z}~!)PnF4O_u|MI2JQo@OKuUqx2ucOX`l)Zz2D1coF2@&)Kx?WMTSGv|6kS)G zj?EQ?_=r!~moFzV|G{>5U&b(BSioEfftRb})KQ3Tee+^TU!29vP>}{MJE?kDFkX&j zS+!~#j^9YlOePSUfShX}D6|~MWVs8V9dq;Y%9HPiAX!Net61Ha>g1#=&e*@?=S{{e z)}+{mlCMx`bf1vJaS9UZ#b%8_n2{h_;ulDQ^w?DjLvzRgh6N4xC*wxEDD0ud%a&CT zR1fhi8oH7;zJoweIR#%MK`Y&KQ~WUwV@eDZ@Yg&u@MvpcvzOGFk$1|gmFf0x36PAeYn~_zx^LX~Wo2QX@ z-Hy=9)Wk>?P+A1OSfCRNoVV(BbkQ)>B`#4>Epi$N4=0GxbHl0Fwkg-3V$=?Dd<>A) z6c34Z5ij66YEzFx7J!HeH3)4I1F8ma@E)&jH2fJXUmA3JQlE0YSwk^vrtpty0Jq7U z6UqFcl!yDFmvO*xU083GK$j#Bj59Bt4KQum?8tW<0PGz_4;lvuc0e>kxBq<|l!z#t z$0Z&YHs3f7RY%M~9|Me`A9YeZL|8choC;_yN^%(4xX@}!p}-CjQXF~;EKWPkJ}S+z zC;Dt}>Jjh^H4v5|=;NZ32Sh8NO+&8bC&UJ=Mg@CxdN?`Dv^)}Lx}=qXR!LC!I)~JB zzz|-iLgJe!-Ea}NLSfJVN=-Unhy<`cAyZBb&JiT%i700Qn=>G9MuW$U8#xC7r=uJ` z8aaNkj9q**pvwZ4g`5yWWr@UxkONX%Wbfmk_xl9N2z4zAF9=q?mr=c1Zt)EA`^(FvBJj0>uL;w?KXSICqP|FD}g9SfRHU=aTgbG53;`X89N%pd(z(t^N z!P|f1CjLU(%jP0sWK3{-0&A)|ojQV{5h)MrGWBl19-E$sdf{fBgLa E1Ad1No&W#< literal 0 HcmV?d00001 diff --git a/src/lost_cities_jax/gates.py b/src/lost_cities_jax/gates.py new file mode 100644 index 0000000..fa35c84 --- /dev/null +++ b/src/lost_cities_jax/gates.py @@ -0,0 +1,1146 @@ +"""Gate 1/2 audit runner for selectivity and strengthened exploiters.""" + +from __future__ import annotations + +import json +import math +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +import jax +import jax.numpy as jnp +import numpy as np +import yaml + +from lost_cities_jax.engine import current_hand_sorted, legal_action_mask, reset_from_order, step +from lost_cities_jax.obs import observation +from lost_cities_jax.opponents import policy_by_name +from lost_cities_jax.ppo import ( + ActorCritic, + JaxPPOConfig, + batch_score_diff_for_players, + checkpoint_policy, + create_train_state, + evaluate_checkpoint_match, + evaluate_checkpoint_vs_static, + evaluate_policy_match, + load_config, + make_shuffle_bank, + mask_logits, + restore_checkpoint, + train_against_checkpoint, +) +from lost_cities_jax.types import ( + CARDS_PER_COLOR, + MAX_STEPS, + N_ACTIONS, + N_CARDS, + PLAY, + State, +) + +ACTION_IDS = jnp.arange(N_ACTIONS, dtype=jnp.int32) +ACTION_HAND_SLOT = ACTION_IDS // 12 +ACTION_PLACE_TYPE = (ACTION_IDS % 12) // 6 +NEG_INF = -1.0e9 + + +@dataclass +class ExploiterSpec: + name: str + config: str + resume: str | None = None + notes: str = "" + + +@dataclass +class RepairConfig: + max_cycles: int = 3 + league_template: str = "configs/jax_ppo/league-v1.yaml" + experiment_name: str = "jax-ppo-gates-1-2-repair" + artifact_subdir: str = "gate2c" + evaluation_games: int | None = None + evaluation_batch_games: int | None = None + guard_expert_ci_low: float = 0.0 + guard_max_steps_rate: float = 0.02 + + +@dataclass +class GatesConfig: + artifact_root: str = "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2" + report_path: str = "docs/reports/gates-1-2-2026-07-05.md" + summary_path: str = "docs/reports/gates-1-2-2026-07-05-summary.jsonl" + target_config: str = ( + "/mnt/2tbhdd/coolrl-lost-cities-artifacts/league/" + "2026-07-05_052325_jax-ppo-league-v1/main_ppo_config.json" + ) + target_checkpoint: str = ( + "/mnt/2tbhdd/coolrl-lost-cities-artifacts/league/" + "2026-07-05_052325_jax-ppo-league-v1/snapshots/cycle_01_update_000500" + ) + shuffle_bank_seed: int = 20260704 + gate1_games: int = 2_000 + gate1_batch_games: int = 8192 + delta_games: int = 200 + delta_target_events: int = 500 + delta_pairs: int = 64 + delta_batch_events: int = 16 + gate2_eval_games: int = 2_000 + gate2_pass_threshold: float = 0.55 + gpu_budget_hours: float = 8.0 + exploiters: list[ExploiterSpec] = field(default_factory=list) + repair: RepairConfig = field(default_factory=RepairConfig) + + +def load_gates_config(path: str | Path) -> GatesConfig: + defaults = GatesConfig() + data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {} + gate1 = data.get("gate1", {}) + gate2 = data.get("gate2", {}) + repair = data.get("repair", {}) + target = data.get("target", {}) + run = data.get("run", {}) + exploiters = [ExploiterSpec(**item) for item in gate2.get("exploiters", [])] + repair_cfg = RepairConfig(**{**asdict(defaults.repair), **repair}) + return GatesConfig( + artifact_root=run.get( + "artifact_root", "/mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2" + ), + report_path=run.get("report_path", "docs/reports/gates-1-2-2026-07-05.md"), + summary_path=run.get("summary_path", "docs/reports/gates-1-2-2026-07-05-summary.jsonl"), + target_config=target.get("config", defaults.target_config), + target_checkpoint=target.get("checkpoint", defaults.target_checkpoint), + shuffle_bank_seed=run.get("shuffle_bank_seed", 20260704), + gate1_games=gate1.get("games", 2_000), + gate1_batch_games=gate1.get("batch_games", 8192), + delta_games=gate1.get("delta_games", 200), + delta_target_events=gate1.get("delta_target_events", 500), + delta_pairs=gate1.get("delta_pairs", 64), + delta_batch_events=gate1.get("delta_batch_events", 16), + gate2_eval_games=gate2.get("eval_games", 2_000), + gate2_pass_threshold=gate2.get("pass_threshold", 0.55), + gpu_budget_hours=run.get("gpu_budget_hours", 8.0), + exploiters=exploiters, + repair=repair_cfg, + ) + + +def run_gates(config_path: str | Path) -> Path: + cfg = load_gates_config(config_path) + run_dir = _create_run_dir(cfg) + _write_json(run_dir / "config.json", _config_json(cfg)) + summary_path = Path(cfg.summary_path) + summary_path.parent.mkdir(parents=True, exist_ok=True) + summary_path.write_text("", encoding="utf-8") + start = time.perf_counter() + + rows: list[dict[str, Any]] = [] + for row in run_policy_tournament(cfg, run_dir): + rows.append(row) + _append_jsonl(summary_path, row) + _append_jsonl(run_dir / "summary.jsonl", row) + + for row in run_delta_open_audit(cfg, run_dir): + rows.append(row) + _append_jsonl(summary_path, row) + _append_jsonl(run_dir / "summary.jsonl", row) + + for row in run_exploiter_battery(cfg, run_dir, start): + rows.append(row) + _append_jsonl(summary_path, row) + _append_jsonl(run_dir / "summary.jsonl", row) + + for row in run_conditional_repair(cfg, run_dir, rows, start): + rows.append(row) + _append_jsonl(summary_path, row) + _append_jsonl(run_dir / "summary.jsonl", row) + + final_row = { + "event": "gates_complete", + "run_dir": str(run_dir), + "elapsed_seconds": time.perf_counter() - start, + } + rows.append(final_row) + _append_jsonl(summary_path, final_row) + _append_jsonl(run_dir / "summary.jsonl", final_row) + write_report(cfg, run_dir, rows) + return run_dir + + +def repair_gates(config_path: str | Path, run_dir: str | Path) -> Path: + cfg = load_gates_config(config_path) + run_path = Path(run_dir) + rows = _read_jsonl(run_path / "summary.jsonl") + summary_path = Path(cfg.summary_path) + prior_complete = _latest_row(rows, "gates_complete") + prior_elapsed = float(prior_complete.get("elapsed_seconds", 0.0)) if prior_complete else 0.0 + start = time.perf_counter() - prior_elapsed + for row in run_conditional_repair(cfg, run_path, rows, start): + rows.append(row) + _append_jsonl(summary_path, row) + _append_jsonl(run_path / "summary.jsonl", row) + final_row = { + "event": "gates_repair_complete", + "run_dir": str(run_path), + "elapsed_seconds_total": time.perf_counter() - start, + } + rows.append(final_row) + _append_jsonl(summary_path, final_row) + _append_jsonl(run_path / "summary.jsonl", final_row) + write_report(cfg, run_path, rows) + return run_path + + +def run_policy_tournament(cfg: GatesConfig, run_dir: Path) -> list[dict[str, Any]]: + target_cfg = load_config(cfg.target_config) + target_policy = checkpoint_policy(target_cfg, cfg.target_checkpoint) + variants = ["expert_cap2", "expert_cap3", "expert_capN"] + rows: list[dict[str, Any]] = [] + for left, right in [ + ("expert_cap2", "expert_cap3"), + ("expert_cap2", "expert_capN"), + ("expert_cap3", "expert_capN"), + ]: + output = run_dir / "gate1" / f"{left}_vs_{right}.json" + result = evaluate_policy_match( + policy_by_name(left), + policy_by_name(right), + games=cfg.gate1_games, + duplicate=True, + shuffle_bank_seed=cfg.shuffle_bank_seed, + batch_games=cfg.gate1_batch_games, + output=output, + ) + rows.append( + { + "event": "gate1_tournament", + "learner": left, + "opponent": right, + **_summary(result), + "json": str(output), + } + ) + + for variant in variants: + output = run_dir / "gate1" / f"{variant}_vs_league_v1_update_500.json" + result = evaluate_policy_match( + policy_by_name(variant), + target_policy, + games=cfg.gate1_games, + duplicate=True, + shuffle_bank_seed=cfg.shuffle_bank_seed, + batch_games=cfg.gate1_batch_games, + output=output, + ) + rows.append( + { + "event": "gate1_vs_league", + "learner": variant, + "opponent": "league_v1_update_500", + **_summary(result), + "json": str(output), + } + ) + return rows + + +def run_delta_open_audit(cfg: GatesConfig, run_dir: Path) -> list[dict[str, Any]]: + target_cfg = load_config(cfg.target_config) + state = create_train_state(target_cfg, jax.random.PRNGKey(0)) + state = restore_checkpoint(Path(cfg.target_checkpoint), state) + opponent = policy_by_name("heuristic_expert") + events = _collect_open_events(cfg, state.params, target_cfg, opponent) + event_count = int(events["actual"].shape[0]) + if event_count == 0: + row = { + "event": "gate1_delta_open", + "states": 0, + "pairs": cfg.delta_pairs, + "mean_delta": 0.0, + "ci95_low": 0.0, + "ci95_high": 0.0, + "judgment": "no_events", + } + _write_json(run_dir / "gate1" / "delta_open_summary.json", row) + return [row] + + deltas = _delta_rollouts(cfg, state.params, target_cfg, opponent, events) + delta_path = run_dir / "gate1" / "delta_open_deltas.npy" + delta_path.parent.mkdir(parents=True, exist_ok=True) + np.save(delta_path, deltas) + hist_path = Path(cfg.report_path).parent / "gates-1-2-delta-open-hist.png" + _write_delta_histogram(deltas, hist_path) + mean = float(np.mean(deltas)) + std = float(np.std(deltas, ddof=1)) if deltas.size > 1 else 0.0 + se = std / math.sqrt(max(deltas.size, 1)) + row = { + "event": "gate1_delta_open", + "states": event_count, + "pairs": cfg.delta_pairs, + "samples": int(deltas.size), + "mean_delta": mean, + "std_delta": std, + "ci95_low": mean - 1.959963984540054 * se, + "ci95_high": mean + 1.959963984540054 * se, + "quantiles": _quantiles(deltas), + "histogram": _histogram(deltas), + "deltas_npy": str(delta_path), + "histogram_png": str(hist_path), + "judgment": _delta_judgment(mean, se), + } + _write_json(run_dir / "gate1" / "delta_open_summary.json", row) + return [row] + + +def run_exploiter_battery( + cfg: GatesConfig, run_dir: Path, start_time: float +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for spec in cfg.exploiters: + if (time.perf_counter() - start_time) / 3600.0 >= cfg.gpu_budget_hours: + rows.append({"event": "gate2_budget_stop", "next_exploiter": spec.name}) + break + rows.append( + _train_and_eval_exploiter( + cfg, + spec, + target_config=cfg.target_config, + target_checkpoint=cfg.target_checkpoint, + artifact_root=run_dir / "exploiters", + event="gate2_exploiter", + target_name="league_v1_update_500", + ) + ) + if rows: + exploiter_rows = [row for row in rows if row.get("event") == "gate2_exploiter"] + if exploiter_rows: + worst = max(exploiter_rows, key=lambda row: row["win_rate"]) + rows.append( + { + "event": "gate2_judgment", + "worst_exploiter": worst["exploiter"], + "worst_win_rate": worst["win_rate"], + "threshold": cfg.gate2_pass_threshold, + "passed": worst["win_rate"] <= cfg.gate2_pass_threshold, + } + ) + return rows + + +def run_conditional_repair( + cfg: GatesConfig, + run_dir: Path, + existing_rows: list[dict[str, Any]], + start_time: float, +) -> list[dict[str, Any]]: + gate2 = _latest_row(existing_rows, "gate2_judgment") + if gate2 is None: + return [{"event": "gate2c_skipped", "reason": "gate2_incomplete"}] + if gate2.get("passed"): + return [{"event": "gate2c_skipped", "reason": "gate2_passed"}] + + rows: list[dict[str, Any]] = [] + target_config = cfg.target_config + target_checkpoint = cfg.target_checkpoint + worst_name = gate2["worst_exploiter"] + extra_pool_members: list[dict[str, Any]] = [] + for cycle in range(1, cfg.repair.max_cycles + 1): + if (time.perf_counter() - start_time) / 3600.0 >= cfg.gpu_budget_hours: + rows.append({"event": "gate2c_budget_stop", "cycle": cycle}) + break + + worst_spec = _exploiter_spec_by_name(cfg, worst_name) + if worst_spec is None: + rows.append( + { + "event": "gate2c_error", + "cycle": cycle, + "reason": f"unknown worst exploiter: {worst_name}", + } + ) + break + + previous_target_config = target_config + previous_target_checkpoint = target_checkpoint + league_config_path = _write_repair_league_config( + cfg, + run_dir, + cycle, + target_config, + target_checkpoint, + worst_spec, + extra_pool_members, + ) + from lost_cities_jax.league import run_league + + league_run_dir = run_league(league_config_path) + league_rows = _read_jsonl(league_run_dir / "league_summary.jsonl") + completion = _latest_row(league_rows, "league_complete") + if completion is None: + rows.append( + { + "event": "gate2c_error", + "cycle": cycle, + "reason": "league_cycle_missing_completion", + "league_run_dir": str(league_run_dir), + } + ) + break + target_config = str(league_run_dir / "main_ppo_config.json") + target_checkpoint = completion["final_checkpoint"] + rows.append( + { + "event": "gate2c_league_cycle", + "cycle": cycle, + "worst_protocol": worst_spec.name, + "league_config": str(league_config_path), + "league_run_dir": str(league_run_dir), + "target_config": target_config, + "target_checkpoint": target_checkpoint, + } + ) + + guard = _evaluate_repair_guard(cfg, run_dir, cycle, target_config, target_checkpoint) + rows.append(guard) + if not guard["passed"]: + rows.append( + { + "event": "gate2c_guard_rollback", + "cycle": cycle, + "rolled_back_to_config": previous_target_config, + "rolled_back_to_checkpoint": previous_target_checkpoint, + "failed_checkpoint": target_checkpoint, + } + ) + target_config = previous_target_config + target_checkpoint = previous_target_checkpoint + continue + + exploiter_member = _repair_exploiter_member(league_rows, cycle) + if exploiter_member is not None: + extra_pool_members.append(exploiter_member) + + battery_rows = run_repair_battery( + cfg, + run_dir, + cycle, + target_config, + target_checkpoint, + start_time, + ) + rows.extend(battery_rows) + judgment = _repair_judgment(cfg, battery_rows, cycle) + rows.append(judgment) + if judgment.get("passed"): + break + worst_name = judgment.get("worst_exploiter", worst_name) + return rows + + +def run_repair_battery( + cfg: GatesConfig, + run_dir: Path, + cycle: int, + target_config: str, + target_checkpoint: str, + start_time: float, +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for spec in cfg.exploiters: + if (time.perf_counter() - start_time) / 3600.0 >= cfg.gpu_budget_hours: + rows.append( + {"event": "gate2c_budget_stop", "cycle": cycle, "next_exploiter": spec.name} + ) + break + rows.append( + _train_and_eval_exploiter( + cfg, + spec, + target_config=target_config, + target_checkpoint=target_checkpoint, + artifact_root=run_dir + / cfg.repair.artifact_subdir + / f"cycle_{cycle:02d}" + / "battery", + event="gate2c_exploiter", + target_name=f"repair_cycle_{cycle:02d}", + repair_cycle=cycle, + ) + ) + return rows + + +def _train_and_eval_exploiter( + cfg: GatesConfig, + spec: ExploiterSpec, + *, + target_config: str, + target_checkpoint: str, + artifact_root: Path, + event: str, + target_name: str, + repair_cycle: int | None = None, +) -> dict[str, Any]: + target_cfg = load_config(target_config) + exploiter_cfg = load_config(spec.config) + exploiter_cfg.run.artifact_root = str(artifact_root) + train_dir = train_against_checkpoint( + exploiter_cfg, + target_cfg, + target_checkpoint, + resume=spec.resume, + ) + output = train_dir / f"eval_vs_{target_name}_duplicate.json" + result = evaluate_checkpoint_match( + exploiter_cfg, + train_dir / "latest", + target_cfg, + target_checkpoint, + games=cfg.gate2_eval_games, + duplicate=True, + output=output, + ) + row = { + "event": event, + "exploiter": spec.name, + "target_name": target_name, + "target_config": target_config, + "target_checkpoint": target_checkpoint, + "train_dir": str(train_dir), + "checkpoint": str(train_dir / "latest"), + "resume": spec.resume, + "notes": spec.notes, + "json": str(output), + **_summary(result), + } + if repair_cycle is not None: + row["cycle"] = repair_cycle + return row + + +def _write_repair_league_config( + cfg: GatesConfig, + run_dir: Path, + cycle: int, + target_config: str, + target_checkpoint: str, + worst_spec: ExploiterSpec, + extra_pool_members: list[dict[str, Any]], +) -> Path: + template_path = Path(cfg.repair.league_template) + data = yaml.safe_load(template_path.read_text(encoding="utf-8")) or {} + data["base_config"] = target_config + data["warm_start_checkpoint"] = target_checkpoint + + run = data.setdefault("run", {}) + run["experiment_name"] = f"{cfg.repair.experiment_name}-c{cycle:02d}" + run["artifact_root"] = str(run_dir / cfg.repair.artifact_subdir / "league") + run["seed"] = int(run.get("seed", 20260705)) + cycle + + league = data.setdefault("league", {}) + league["cycles"] = 1 + league["success_exploiter_win_rate"] = cfg.gate2_pass_threshold + + evaluation = data.setdefault("evaluation", {}) + if cfg.repair.evaluation_games is not None: + evaluation["games"] = cfg.repair.evaluation_games + if cfg.repair.evaluation_batch_games is not None: + evaluation["batch_games"] = cfg.repair.evaluation_batch_games + + exploiter = data.setdefault("exploiter", {}) + exploiter["config"] = worst_spec.config + exploiter_cfg = load_config(worst_spec.config) + exploiter["updates"] = exploiter_cfg.run.total_updates + if worst_spec.resume: + exploiter["resume"] = worst_spec.resume + else: + exploiter.pop("resume", None) + + guards = data.setdefault("guards", {}) + guards["expert_ci_low"] = cfg.repair.guard_expert_ci_low + guards["max_steps_rate"] = cfg.repair.guard_max_steps_rate + + cycle_dir = run_dir / cfg.repair.artifact_subdir / f"cycle_{cycle:02d}" + tracking = data.setdefault("tracking", {}) + tracking["tracked_summary_path"] = str(cycle_dir / "league_summary.jsonl") + tracking["report_path"] = str(cycle_dir / "league_report.md") + + anchors = list(data.get("anchors", [])) + existing_names = {item.get("name") for item in anchors} + for member in extra_pool_members: + if member["name"] not in existing_names: + anchors.append(member) + existing_names.add(member["name"]) + data["anchors"] = anchors + + config_path = cycle_dir / "league_config.yaml" + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + return config_path + + +def _evaluate_repair_guard( + cfg: GatesConfig, + run_dir: Path, + cycle: int, + target_config: str, + target_checkpoint: str, +) -> dict[str, Any]: + target_cfg = load_config(target_config) + output = ( + run_dir + / cfg.repair.artifact_subdir + / f"cycle_{cycle:02d}" + / "guard_vs_heuristic_expert.json" + ) + result = evaluate_checkpoint_vs_static( + target_cfg, + target_checkpoint, + "heuristic_expert", + games=cfg.gate2_eval_games, + duplicate=True, + output=output, + ) + passed = ( + result["score_diff_ci95_low"] > cfg.repair.guard_expert_ci_low + and result["max_steps_rate"] <= cfg.repair.guard_max_steps_rate + ) + return { + "event": "gate2c_guard_expert", + "cycle": cycle, + "target_config": target_config, + "target_checkpoint": target_checkpoint, + "json": str(output), + "passed": passed, + **_summary(result), + } + + +def _repair_exploiter_member( + league_rows: list[dict[str, Any]], cycle: int +) -> dict[str, Any] | None: + exploiter = _latest_row(league_rows, "exploiter_eval") + if exploiter is None: + return None + checkpoint = Path(exploiter["exploiter_checkpoint"]) + return { + "name": f"repair_exploiter_c{cycle:02d}", + "kind": "checkpoint", + "anchor": False, + "stalling": False, + "exploiter": True, + "config": str(checkpoint.parent / "config.json"), + "checkpoint": str(checkpoint), + "recent_win_rate": 1.0 - float(exploiter["win_rate"]), + "created_cycle": cycle, + "created_update": 0, + } + + +def _repair_judgment( + cfg: GatesConfig, battery_rows: list[dict[str, Any]], cycle: int +) -> dict[str, Any]: + exploiters = [row for row in battery_rows if row.get("event") == "gate2c_exploiter"] + if not exploiters: + return {"event": "gate2c_judgment", "cycle": cycle, "passed": False, "reason": "no_battery"} + worst = max(exploiters, key=lambda row: row["win_rate"]) + return { + "event": "gate2c_judgment", + "cycle": cycle, + "worst_exploiter": worst["exploiter"], + "worst_win_rate": worst["win_rate"], + "threshold": cfg.gate2_pass_threshold, + "passed": worst["win_rate"] <= cfg.gate2_pass_threshold, + } + + +def _exploiter_spec_by_name(cfg: GatesConfig, name: str) -> ExploiterSpec | None: + return next((spec for spec in cfg.exploiters if spec.name == name), None) + + +def write_report(cfg: GatesConfig, run_dir: Path, rows: list[dict[str, Any]]) -> None: + report_path = Path(cfg.report_path) + report_path.parent.mkdir(parents=True, exist_ok=True) + tournament = [row for row in rows if row.get("event") == "gate1_tournament"] + vs_league = [row for row in rows if row.get("event") == "gate1_vs_league"] + delta = next((row for row in rows if row.get("event") == "gate1_delta_open"), None) + exploiters = [row for row in rows if row.get("event") == "gate2_exploiter"] + gate2 = next((row for row in rows if row.get("event") == "gate2_judgment"), None) + repair_cycles = [row for row in rows if row.get("event") == "gate2c_league_cycle"] + repair_guards = [row for row in rows if row.get("event") == "gate2c_guard_expert"] + repair_exploiters = [row for row in rows if row.get("event") == "gate2c_exploiter"] + repair_judgments = [row for row in rows if row.get("event") == "gate2c_judgment"] + repair_skips = [row for row in rows if row.get("event") == "gate2c_skipped"] + final_robustness = repair_judgments[-1] if repair_judgments else gate2 + lines = [ + "# Gates 1-2 Report - 2026-07-05", + "", + f"**Run dir:** `{run_dir}`.", + f"**Target:** `{cfg.target_checkpoint}`.", + "", + "## Gate 1A - Policy Class Tournament", + "", + "| Learner | Opponent | Win rate | Mean diff | CI low | Opened colors | Max-step |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: |", + ] + for row in tournament: + lines.append(_match_row(row)) + lines.extend( + [ + "", + "## Gate 1A - Variants vs League v1", + "", + "| Learner | Opponent | Win rate | Mean diff | CI low | Opened colors | Max-step |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for row in vs_league: + lines.append(_match_row(row)) + lines.extend(["", "## Gate 1B - Delta Open Audit", ""]) + if delta: + lines.extend( + [ + f"- States: {delta['states']}", + f"- Paired samples: {delta.get('samples', 0)}", + f"- Mean delta: {delta['mean_delta']:+.4f}", + f"- CI95: [{delta['ci95_low']:+.4f}, {delta['ci95_high']:+.4f}]", + f"- Judgment: `{delta['judgment']}`", + f"- Histogram: `{delta.get('histogram_png', '')}`", + ] + ) + lines.extend( + [ + "", + "## Gate 1C - Selectivity Judgment", + "", + _gate1_judgment(tournament, delta), + "", + "## Gate 2A - Strengthened Exploiter Battery", + "", + "| Exploiter | Win rate | Mean diff | CI low | Opened colors | Max-step | Run |", + "| --- | ---: | ---: | ---: | ---: | ---: | --- |", + ] + ) + for row in exploiters: + lines.append( + "| `{exploiter}` | {win_rate:.4f} | {mean_score_diff:+.4f} | " + "{score_diff_ci95_low:+.4f} | {opened_colors_per_game:.4f} | " + "{max_steps_rate:.4f} | `{train_dir}` |".format(**row) + ) + lines.extend(["", "## Gate 2B - Robustness Judgment", ""]) + if gate2: + status = "통과" if gate2["passed"] else "보수 필요" + lines.append( + f"{status}: worst exploiter `{gate2['worst_exploiter']}` win rate " + f"{gate2['worst_win_rate']:.4f} vs threshold {gate2['threshold']:.4f}." + ) + else: + lines.append("미결: exploiter battery did not complete.") + lines.extend(["", "## Gate 2C - Conditional Repair League", ""]) + if repair_skips: + lines.append(f"Skipped: `{repair_skips[-1]['reason']}`.") + elif repair_cycles: + lines.extend( + [ + "| Cycle | Worst protocol used | Guard pass | Battery worst | Passed | Checkpoint |", + "| ---: | --- | ---: | ---: | ---: | --- |", + ] + ) + guards_by_cycle = {row["cycle"]: row for row in repair_guards} + judgments_by_cycle = {row["cycle"]: row for row in repair_judgments} + for row in repair_cycles: + cycle = row["cycle"] + guard = guards_by_cycle.get(cycle, {}) + judgment = judgments_by_cycle.get(cycle, {}) + worst = judgment.get("worst_win_rate") + worst_text = "n/a" if worst is None else f"{worst:.4f}" + lines.append( + f"| {cycle} | `{row['worst_protocol']}` | " + f"{str(guard.get('passed', False))} | {worst_text} | " + f"{str(judgment.get('passed', False))} | `{row['target_checkpoint']}` |" + ) + if repair_exploiters: + lines.extend( + [ + "", + "| Cycle | Exploiter | Win rate | Mean diff | CI low | Opened colors | Max-step |", + "| ---: | --- | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for row in repair_exploiters: + lines.append( + "| {cycle} | `{exploiter}` | {win_rate:.4f} | {mean_score_diff:+.4f} | " + "{score_diff_ci95_low:+.4f} | {opened_colors_per_game:.4f} | " + "{max_steps_rate:.4f} |".format(**row) + ) + else: + lines.append("Not run yet.") + lines.extend( + [ + "", + "## Human Play Recommendation", + "", + _human_play_recommendation(tournament, delta, final_robustness), + "", + "## Decisions", + "", + "- Existing `heuristic_expert` remains unchanged. The cap variants add only a " + "hard `max_open_colors` gate around new-color openings.", + "- `expert_capN` means no hard cap; EV thresholds and the existing soft " + "concentration penalties are retained.", + "- Warm-started exploiters use shaping coefficient 0 to measure target-specific " + "exploitation without reintroducing early shaping rewards.", + ] + ) + report_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def _collect_open_events(cfg: GatesConfig, params: Any, target_cfg: JaxPPOConfig, opponent) -> dict: + collect_batch = _make_collect_open_events_fn(target_cfg, opponent) + collected_states: list[State] = [] + collected_actual: list[np.ndarray] = [] + collected_alt: list[np.ndarray] = [] + collected_seats: list[np.ndarray] = [] + for seat in [0, 1]: + for start in range(0, cfg.delta_games, cfg.gate1_batch_games): + count = min(cfg.gate1_batch_games, cfg.delta_games - start) + orders = make_shuffle_bank(cfg.shuffle_bank_seed + start, count) + states, actual, alt, event_mask = collect_batch( + params, jnp.asarray(orders, dtype=jnp.int8), jnp.asarray(seat, dtype=jnp.int32) + ) + event_mask_np = np.asarray(jax.device_get(event_mask)).reshape(-1) + event_idx = np.flatnonzero(event_mask_np) + remaining = cfg.delta_target_events - sum(x.shape[0] for x in collected_actual) + if remaining <= 0: + break + event_idx = event_idx[:remaining] + if event_idx.size == 0: + continue + collected_states.append(_take_scan_states(states, event_idx)) + collected_actual.append(np.asarray(jax.device_get(actual)).reshape(-1)[event_idx]) + collected_alt.append(np.asarray(jax.device_get(alt)).reshape(-1)[event_idx]) + collected_seats.append(np.full((event_idx.size,), seat, dtype=np.int32)) + if sum(x.shape[0] for x in collected_actual) >= cfg.delta_target_events: + break + if not collected_actual: + empty_state = jax.tree_util.tree_map( + lambda x: np.asarray(x)[:0], reset_from_order(jnp.arange(N_CARDS, dtype=jnp.int8)) + ) + return { + "states": empty_state, + "actual": np.asarray([], dtype=np.int32), + "alt": np.asarray([], dtype=np.int32), + "seats": np.asarray([], dtype=np.int32), + } + return { + "states": _concat_states(collected_states), + "actual": np.concatenate(collected_actual).astype(np.int32), + "alt": np.concatenate(collected_alt).astype(np.int32), + "seats": np.concatenate(collected_seats).astype(np.int32), + } + + +def _make_collect_open_events_fn(target_cfg: JaxPPOConfig, opponent_policy): + model = ActorCritic(target_cfg.network.hidden_size, target_cfg.network.num_layers) + + @jax.jit + def collect(params, deck_orders: jax.Array, learner_seat: jax.Array): + learner = learner_seat.astype(jnp.int32) + opponent = 1 - learner + env_state = jax.vmap(reset_from_order)(deck_orders) + keys = jax.random.split(jax.random.PRNGKey(0), deck_orders.shape[0]) + + def body(carry, _): + env, key = carry + key, opponent_key = jax.random.split(key) + legal = jax.vmap(legal_action_mask)(env) + obs = jax.vmap(observation, in_axes=(0, None))(env, learner) + logits, _ = model.apply(params, obs) + open_mask = _new_open_action_mask(env, learner) + learner_actions = jnp.argmax(mask_logits(logits, legal), axis=-1).astype(jnp.int32) + alt_mask = legal & ~open_mask + alt_actions = jnp.argmax(jnp.where(alt_mask, logits, NEG_INF), axis=-1).astype( + jnp.int32 + ) + opponent_keys = jax.random.split(opponent_key, deck_orders.shape[0]) + opponent_actions = jax.vmap(opponent_policy, in_axes=(0, None, 0))( + env, opponent, opponent_keys + ) + learner_turn = env.to_move.astype(jnp.int32) == learner + active = ~env.done + actions = jnp.where(learner_turn, learner_actions, opponent_actions) + active_colors = jnp.sum(env.col_len[:, learner, :] > 0, axis=-1) + event = ( + active + & learner_turn + & (active_colors >= 3) + & jnp.take_along_axis(open_mask, learner_actions[:, None], axis=-1)[:, 0] + ) + next_env, _, _ = jax.vmap(step, in_axes=(0, 0))(env, actions) + return (next_env, key), (env, learner_actions, alt_actions, event) + + (_, _), outputs = jax.lax.scan(body, (env_state, keys[0]), xs=None, length=MAX_STEPS) + return outputs + + return collect + + +def _delta_rollouts( + cfg: GatesConfig, + params: Any, + target_cfg: JaxPPOConfig, + opponent, + events: dict, +) -> np.ndarray: + delta_batch = _make_delta_rollout_fn(target_cfg, opponent) + n_events = int(events["actual"].shape[0]) + deltas: list[np.ndarray] = [] + for start in range(0, n_events, cfg.delta_batch_events): + end = min(start + cfg.delta_batch_events, n_events) + states = _state_slice(events["states"], start, end) + repeated_states = jax.tree_util.tree_map( + lambda x: jnp.repeat(jnp.asarray(x), cfg.delta_pairs, axis=0), states + ) + actual = np.repeat(events["actual"][start:end], cfg.delta_pairs) + alt = np.repeat(events["alt"][start:end], cfg.delta_pairs) + seats = np.repeat(events["seats"][start:end], cfg.delta_pairs) + batch = delta_batch( + params, + repeated_states, + jnp.asarray(actual, dtype=jnp.int32), + jnp.asarray(alt, dtype=jnp.int32), + jnp.asarray(seats, dtype=jnp.int32), + jax.random.PRNGKey(cfg.shuffle_bank_seed + 100_000 + start), + ) + deltas.append(np.asarray(jax.device_get(batch), dtype=np.float64)) + return np.concatenate(deltas) + + +def _make_delta_rollout_fn(target_cfg: JaxPPOConfig, opponent_policy): + model = ActorCritic(target_cfg.network.hidden_size, target_cfg.network.num_layers) + + @jax.jit + def delta_batch( + params: Any, + states: State, + actual_actions: jax.Array, + alt_actions: jax.Array, + learners: jax.Array, + rng: jax.Array, + ) -> jax.Array: + actual_env, _, _ = jax.vmap(step, in_axes=(0, 0))(states, actual_actions) + alt_env, _, _ = jax.vmap(step, in_axes=(0, 0))(states, alt_actions) + + def branch_action(env: State, key: jax.Array) -> tuple[jax.Array, jax.Array]: + key, learner_key, opponent_key = jax.random.split(key, 3) + legal = jax.vmap(legal_action_mask)(env) + obs = jax.vmap(observation, in_axes=(0, 0))(env, learners) + logits, _ = model.apply(params, obs) + learner_actions = jax.random.categorical( + learner_key, mask_logits(logits, legal), axis=-1 + ).astype(jnp.int32) + opponent_keys = jax.random.split(opponent_key, learners.shape[0]) + opponent_actions = jax.vmap(opponent_policy, in_axes=(0, 0, 0))( + env, 1 - learners, opponent_keys + ) + learner_turn = env.to_move.astype(jnp.int32) == learners + return jnp.where(learner_turn, learner_actions, opponent_actions), key + + def body(carry, _): + env_a, env_b, key = carry + key, step_key = jax.random.split(key) + actions_a, _ = branch_action(env_a, step_key) + actions_b, _ = branch_action(env_b, step_key) + next_a, _, _ = jax.vmap(step, in_axes=(0, 0))(env_a, actions_a) + next_b, _, _ = jax.vmap(step, in_axes=(0, 0))(env_b, actions_b) + return (next_a, next_b, key), None + + (final_a, final_b, _), _ = jax.lax.scan( + body, (actual_env, alt_env, rng), xs=None, length=MAX_STEPS + ) + return batch_score_diff_for_players(final_a, learners) - batch_score_diff_for_players( + final_b, learners + ) + + return delta_batch + + +def _new_open_action_mask(states: State, learner: jax.Array) -> jax.Array: + hand = jax.vmap(current_hand_sorted, in_axes=(0, None))(states, learner).astype(jnp.int32) + action_cards = hand[:, ACTION_HAND_SLOT] + valid = action_cards < N_CARDS + safe = jnp.clip(action_cards, 0, N_CARDS - 1) + colors = safe // CARDS_PER_COLOR + col_len = states.col_len[:, learner, :] + batch = jnp.arange(colors.shape[0])[:, None] + open_color = col_len[batch, colors] == 0 + return (ACTION_PLACE_TYPE[None, :] == PLAY) & valid & open_color + + +def _take_scan_states(states: State, event_idx: np.ndarray) -> State: + return jax.tree_util.tree_map( + lambda x: np.asarray(jax.device_get(x)).reshape((-1, *x.shape[2:]))[event_idx], states + ) + + +def _concat_states(states: list[State]) -> State: + return jax.tree_util.tree_map(lambda *xs: np.concatenate(xs, axis=0), *states) + + +def _state_slice(state: State, start: int, end: int) -> State: + return jax.tree_util.tree_map(lambda x: x[start:end], state) + + +def _summary(result: dict[str, Any]) -> dict[str, Any]: + keys = [ + "games", + "wins", + "losses", + "ties", + "win_rate", + "wilson_low", + "wilson_high", + "mean_score_diff", + "score_diff_ci95_low", + "score_diff_ci95_high", + "score_diff_std", + "mean_game_length", + "max_steps_rate", + "opened_colors_per_game", + "play_action_rate", + "positive_expeditions_per_game", + ] + return {key: result[key] for key in keys if key in result} + + +def _match_row(row: dict[str, Any]) -> str: + return ( + "| `{learner}` | `{opponent}` | {win_rate:.4f} | {mean_score_diff:+.4f} | " + "{score_diff_ci95_low:+.4f} | {opened_colors_per_game:.4f} | " + "{max_steps_rate:.4f} |" + ).format(**row) + + +def _gate1_judgment(tournament: list[dict[str, Any]], delta: dict[str, Any] | None) -> str: + capn_rows = [row for row in tournament if row["opponent"] == "expert_capN"] + focused_beats_capn = capn_rows and all(row["score_diff_ci95_low"] > 0.0 for row in capn_rows) + capn_beats_any_focus = any(row["score_diff_ci95_high"] < 0.0 for row in capn_rows) + capn_not_significantly_worse = capn_rows and all( + row["score_diff_ci95_low"] <= 0.0 for row in capn_rows + ) + delta_judgment = delta.get("judgment") if delta else "missing" + if capn_beats_any_focus and delta_judgment == "positive": + return "통념 기각: capN이 집중 변형에 밀리지 않고 4번째+ 오픈 delta도 양수다." + if focused_beats_capn and delta_judgment == "negative": + return "에이전트 결함 확정: 집중 변형이 capN을 이기고 4번째+ 오픈 delta도 음수다." + if capn_not_significantly_worse and delta_judgment == "near_zero": + return ( + "통념 기각/미결: capN은 집중 변형보다 유의하게 나쁘지 않고, " + "리그 정책의 4번째+ 오픈 delta는 0 근처다. selectivity는 현재 " + "주요 성능 병목으로 보이지 않는다." + ) + return "미결: 정책 클래스 토너먼트와 delta_open 판정이 같은 방향으로 충분히 정렬되지 않았다." + + +def _human_play_recommendation( + tournament: list[dict[str, Any]], delta: dict[str, Any] | None, gate2: dict[str, Any] | None +) -> str: + del tournament, delta + if gate2 and gate2["passed"]: + return ( + "조건부 예: 강화 exploiter 관문은 통과했다. 다만 selectivity 관문이 " + "미결/부분 지지이면 인간 대전은 실력 인증이 아니라 행동 양식 진단으로 시작해야 한다." + ) + return ( + "아니오: 강화 exploiter 관문이 통과되지 않았거나 완료되지 않았다. 인간 대전 전 " + "최악 exploiter에 대한 보수 리그가 우선이다." + ) + + +def _delta_judgment(mean: float, se: float) -> str: + margin = 1.959963984540054 * se + if mean - margin > 0.0: + return "positive" + if mean + margin < 0.0: + return "negative" + return "near_zero" + + +def _quantiles(values: np.ndarray) -> dict[str, float]: + q = np.quantile(values, [0.0, 0.05, 0.25, 0.5, 0.75, 0.95, 1.0]) + return { + "min": float(q[0]), + "p05": float(q[1]), + "p25": float(q[2]), + "p50": float(q[3]), + "p75": float(q[4]), + "p95": float(q[5]), + "max": float(q[6]), + } + + +def _histogram(values: np.ndarray) -> dict[str, list[float] | list[int]]: + counts, edges = np.histogram(values, bins=[-200, -100, -50, -20, 0, 20, 50, 100, 200]) + return {"edges": edges.astype(float).tolist(), "counts": counts.astype(int).tolist()} + + +def _write_delta_histogram(values: np.ndarray, path: Path) -> None: + try: + import matplotlib.pyplot as plt + except Exception: + return + path.parent.mkdir(parents=True, exist_ok=True) + plt.figure() + plt.hist(values, bins=[-200, -100, -50, -20, 0, 20, 50, 100, 200]) + plt.axvline(float(np.mean(values)), color="red", linestyle="--") + plt.xlabel("Delta score diff: open branch - non-open branch") + plt.ylabel("Count") + plt.tight_layout() + plt.savefig(path) + plt.close() + + +def _config_json(cfg: GatesConfig) -> dict[str, Any]: + data = asdict(cfg) + data["exploiters"] = [asdict(spec) for spec in cfg.exploiters] + return data + + +def _write_json(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _append_jsonl(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(data, sort_keys=True) + "\n") + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + if not path.exists(): + return [] + return [ + json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip() + ] + + +def _latest_row(rows: list[dict[str, Any]], event: str) -> dict[str, Any] | None: + return next((row for row in reversed(rows) if row.get("event") == event), None) + + +def _create_run_dir(cfg: GatesConfig) -> Path: + root = Path(cfg.artifact_root).resolve() + run_dir = root / f"{time.strftime('%Y-%m-%d_%H%M%S')}_gates-1-2" + run_dir.mkdir(parents=True, exist_ok=False) + return run_dir + + +__all__ = [ + "GatesConfig", + "load_gates_config", + "repair_gates", + "run_gates", +] diff --git a/src/lost_cities_jax/league.py b/src/lost_cities_jax/league.py index b748a29..13705f7 100644 --- a/src/lost_cities_jax/league.py +++ b/src/lost_cities_jax/league.py @@ -72,6 +72,7 @@ class LeagueConfig: evaluation_shuffle_bank_seed: int = 20260704 recent_snapshot_evals: int = 3 exploiter_config: str = "configs/jax_ppo/ladder-v2-exploiter.yaml" + exploiter_resume: str | None = None exploiter_updates: int = 250 expert_guard_ci_low: float = 0.0 max_steps_guard: float = 0.02 @@ -115,6 +116,7 @@ def load_league_config(path: str | Path) -> LeagueConfig: evaluation_shuffle_bank_seed=evaluation.get("shuffle_bank_seed", 20260704), recent_snapshot_evals=evaluation.get("recent_snapshot_evals", 3), exploiter_config=exploiter.get("config", "configs/jax_ppo/ladder-v2-exploiter.yaml"), + exploiter_resume=exploiter.get("resume"), exploiter_updates=exploiter.get("updates", 250), expert_guard_ci_low=guards.get("expert_ci_low", 0.0), max_steps_guard=guards.get("max_steps_rate", 0.02), @@ -392,6 +394,7 @@ def run_exploiter_cycle( exploiter_cfg, ppo_cfg, target.checkpoint or cfg.warm_start_checkpoint, + resume=cfg.exploiter_resume, ) result = evaluate_checkpoint_match( exploiter_cfg, diff --git a/src/lost_cities_jax/opponents.py b/src/lost_cities_jax/opponents.py index cc757f3..26708b3 100644 --- a/src/lost_cities_jax/opponents.py +++ b/src/lost_cities_jax/opponents.py @@ -34,6 +34,7 @@ ACTION_DRAW_SOURCE = ACTION_IDS % 6 @dataclass(frozen=True) class HeuristicExpertConfig: + max_open_colors: int | None = None target_open_colors: int = 3 fourth_color_margin: float = 38.0 extra_color_margin: float = 58.0 @@ -64,7 +65,7 @@ class HeuristicExpertConfig: def policy_by_name(name: str) -> ActionPolicy: - normalized = name.replace("-", "_") + normalized = name.replace("-", "_").lower() if normalized == "discard_only": return discard_only_action if normalized == "heuristic_balanced": @@ -73,6 +74,12 @@ def policy_by_name(name: str) -> ActionPolicy: return heuristic_cautious_action if normalized == "heuristic_expert": return heuristic_expert_action + if normalized in {"expert_cap2", "heuristic_expert_cap2"}: + return make_heuristic_expert_policy(HeuristicExpertConfig(max_open_colors=2)) + if normalized in {"expert_cap3", "heuristic_expert_cap3"}: + return make_heuristic_expert_policy(HeuristicExpertConfig(max_open_colors=3)) + if normalized in {"expert_capn", "expert_cap_n", "heuristic_expert_capn"}: + return make_heuristic_expert_policy(HeuristicExpertConfig(max_open_colors=None)) msg = f"unknown opponent policy: {name}" raise ValueError(msg) @@ -197,6 +204,11 @@ def _heuristic_expert_action( progress = (state.draw_ptr - 16).astype(jnp.float32) / DECK_DRAWS active_colors = jnp.sum(state.col_len[player] > 0).astype(jnp.float32) + hard_cap_allows_new_color = ( + jnp.asarray(True) + if cfg.max_open_colors is None + else active_colors < jnp.asarray(cfg.max_open_colors, dtype=jnp.float32) + ) open_threshold = ( cfg.base_open_threshold + progress * cfg.late_open_threshold @@ -259,11 +271,14 @@ def _heuristic_expert_action( defensive_sacrifice = ( (discard_danger > 22.0) & ~own_open + & hard_cap_allows_new_color & (active_colors < cfg.target_open_colors) & (rank_i <= 5) & (color_open_value > -18.0) ) - new_color_allowed = jnp.where(is_handshake, hs_open_allowed, open_allowed) + new_color_allowed = ( + jnp.where(is_handshake, hs_open_allowed, open_allowed) & hard_cap_allows_new_color + ) play_score = jnp.where( own_open, jnp.where( diff --git a/src/lost_cities_jax/ppo.py b/src/lost_cities_jax/ppo.py index b0845f2..aafa2d3 100644 --- a/src/lost_cities_jax/ppo.py +++ b/src/lost_cities_jax/ppo.py @@ -1290,10 +1290,18 @@ def cli_main(argv: list[str] | None = None) -> None: league_run_parser = league_sub.add_parser("run") league_run_parser.add_argument("--config", required=True) + gates_parser = sub.add_parser("gates") + gates_sub = gates_parser.add_subparsers(dest="gates_command", required=True) + gates_run_parser = gates_sub.add_parser("run") + gates_run_parser.add_argument("--config", required=True) + gates_repair_parser = gates_sub.add_parser("repair") + gates_repair_parser.add_argument("--config", required=True) + gates_repair_parser.add_argument("--run-dir", required=True) + args = parser.parse_args(argv) cfg = ( load_config(args.config, overrides=parse_overrides(args.set)) - if hasattr(args, "config") and args.command != "league" + if hasattr(args, "config") and args.command not in {"league", "gates"} else None ) if args.command == "rollout-smoke": @@ -1378,6 +1386,17 @@ def cli_main(argv: list[str] | None = None) -> None: run_dir = run_league(args.config) print(run_dir) + elif args.command == "gates": + if args.gates_command == "run": + from lost_cities_jax.gates import run_gates + + run_dir = run_gates(args.config) + print(run_dir) + elif args.gates_command == "repair": + from lost_cities_jax.gates import repair_gates + + run_dir = repair_gates(args.config, args.run_dir) + print(run_dir) def _create_run_dir(cfg: JaxPPOConfig) -> Path: diff --git a/tests/lost_cities_jax/test_gates.py b/tests/lost_cities_jax/test_gates.py new file mode 100644 index 0000000..dfac693 --- /dev/null +++ b/tests/lost_cities_jax/test_gates.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from lost_cities_jax.gates import load_gates_config + + +def test_gates_config_loads_repair_settings(): + cfg = load_gates_config("configs/jax_ppo/gates-1-2.yaml") + assert cfg.gate2_pass_threshold == 0.55 + assert len(cfg.exploiters) == 3 + assert cfg.repair.max_cycles == 3 + assert cfg.repair.league_template == "configs/jax_ppo/league-v1.yaml" + assert cfg.repair.evaluation_games == 2000 + + +def test_gates_smoke_config_uses_smoke_repair_template(): + cfg = load_gates_config("configs/jax_ppo/gates-1-2-smoke.yaml") + assert cfg.repair.max_cycles == 1 + assert cfg.repair.league_template == "configs/jax_ppo/league-smoke.yaml" + assert cfg.repair.guard_max_steps_rate == 1.0 diff --git a/tests/lost_cities_jax/test_ppo_stack.py b/tests/lost_cities_jax/test_ppo_stack.py index ea79f87..3aef0e7 100644 --- a/tests/lost_cities_jax/test_ppo_stack.py +++ b/tests/lost_cities_jax/test_ppo_stack.py @@ -10,10 +10,12 @@ import pytest from lost_cities_jax import legal_action_mask, reset from lost_cities_jax.engine import current_hand_sorted, decode_action, reset_from_order from lost_cities_jax.opponents import ( + HeuristicExpertConfig, discard_only_action, heuristic_balanced_action, heuristic_cautious_action, heuristic_expert_action, + make_heuristic_expert_policy, ) from lost_cities_jax.ppo import ( JaxPPOConfig, @@ -178,6 +180,76 @@ def test_expert_avoids_discarding_immediately_useful_opponent_card(): assert hand[hand_slot] != dangerous +def test_expert_cap2_blocks_third_open_color(): + state = _manual_state( + p0_hand=[ + _hs_card(2), + _rank_card(2, 7), + _rank_card(2, 8), + _rank_card(2, 9), + _rank_card(2, 10), + _rank_card(3, 2), + _rank_card(3, 3), + _rank_card(4, 2), + ], + p0_board=[_rank_card(0, 2), _rank_card(1, 2)], + to_move=0, + ) + policy = make_heuristic_expert_policy(HeuristicExpertConfig(max_open_colors=2)) + action = int(policy(state, jnp.int32(0), jax.random.PRNGKey(1))) + hand_slot, place_type, _ = [int(x) for x in decode_action(jnp.asarray(action))] + hand = [int(x) for x in current_hand_sorted(state, 0)] + opened_new_color = ( + place_type == PLAY and state.col_len[0, hand[hand_slot] // CARDS_PER_COLOR] == 0 + ) + assert not bool(opened_new_color) + + +def test_expert_cap3_allows_third_but_blocks_fourth_open_color(): + cap3 = make_heuristic_expert_policy(HeuristicExpertConfig(max_open_colors=3)) + third_state = _manual_state( + p0_hand=[ + _hs_card(2), + _rank_card(2, 7), + _rank_card(2, 8), + _rank_card(2, 9), + _rank_card(2, 10), + _rank_card(3, 2), + _rank_card(3, 3), + _rank_card(4, 2), + ], + p0_board=[_rank_card(0, 2), _rank_card(1, 2)], + to_move=0, + ) + action = int(cap3(third_state, jnp.int32(0), jax.random.PRNGKey(1))) + hand_slot, place_type, _ = [int(x) for x in decode_action(jnp.asarray(action))] + hand = [int(x) for x in current_hand_sorted(third_state, 0)] + assert place_type == PLAY + assert third_state.col_len[0, hand[hand_slot] // CARDS_PER_COLOR] == 0 + + fourth_state = _manual_state( + p0_hand=[ + _hs_card(3), + _rank_card(3, 7), + _rank_card(3, 8), + _rank_card(3, 9), + _rank_card(3, 10), + _rank_card(4, 2), + _rank_card(4, 3), + _rank_card(4, 4), + ], + p0_board=[_rank_card(0, 2), _rank_card(1, 2), _rank_card(2, 2)], + to_move=0, + ) + action = int(cap3(fourth_state, jnp.int32(0), jax.random.PRNGKey(1))) + hand_slot, place_type, _ = [int(x) for x in decode_action(jnp.asarray(action))] + hand = [int(x) for x in current_hand_sorted(fourth_state, 0)] + opened_new_color = ( + place_type == PLAY and fourth_state.col_len[0, hand[hand_slot] // CARDS_PER_COLOR] == 0 + ) + assert not bool(opened_new_color) + + @pytest.mark.parametrize( "policy_name", ["discard_only", "heuristic_balanced", "heuristic_cautious"] )