Add gates 1-2 audit and repair workflow

2026-07-05 17:55:22 +09:00
parent 94e9ac1854
commit fa9a1c5286
15 changed files with 1602 additions and 3 deletions
@@ -0,0 +1,40 @@
+run:
+  experiment_name: gates-1-2-long-random-exploiter
+  seed: 20260705
+  learner_seat: 0
+  total_updates: 1200
+  log_every: 10
+  checkpoint_every: 100
+  artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/exploiters
+
+opponent:
+  name: league_v1_update_500_frozen
+
+network:
+  hidden_size: 512
+  num_layers: 3
+
+ppo:
+  batch_games: 8192
+  rollout_steps: 400
+  gamma: 1.0
+  gae_lambda: 0.95
+  clip_epsilon: 0.2
+  entropy_coef: 0.01
+  value_coef: 0.5
+  max_grad_norm: 0.5
+  learning_rate: 0.0003
+  epochs: 4
+  minibatches: 128
+
+reward:
+  terminal_scale: 50.0
+  potential_shaping_initial: 1.0
+  potential_shaping_final: 0.0
+  potential_shaping_anneal_steps: 5000000
+
+evaluation:
+  games: 2000
+  duplicate: true
+  shuffle_bank_seed: 20260704
+  batch_games: 8192
@@ -0,0 +1,40 @@
+run:
+  experiment_name: gates-1-2-replay-exploiter
+  seed: 20260705
+  learner_seat: 0
+  total_updates: 900
+  log_every: 10
+  checkpoint_every: 100
+  artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/exploiters
+
+opponent:
+  name: league_v1_update_500_frozen
+
+network:
+  hidden_size: 512
+  num_layers: 3
+
+ppo:
+  batch_games: 8192
+  rollout_steps: 400
+  gamma: 1.0
+  gae_lambda: 0.95
+  clip_epsilon: 0.2
+  entropy_coef: 0.01
+  value_coef: 0.5
+  max_grad_norm: 0.5
+  learning_rate: 0.0003
+  epochs: 4
+  minibatches: 128
+
+reward:
+  terminal_scale: 50.0
+  potential_shaping_initial: 0.0
+  potential_shaping_final: 0.0
+  potential_shaping_anneal_steps: 0
+
+evaluation:
+  games: 2000
+  duplicate: true
+  shuffle_bank_seed: 20260704
+  batch_games: 8192
@@ -0,0 +1,40 @@
+run:
+  experiment_name: gates-1-2-smoke-exploiter
+  seed: 20260705
+  learner_seat: 0
+  total_updates: 1
+  log_every: 1
+  checkpoint_every: 1
+  artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2-smoke/exploiters
+
+opponent:
+  name: league_v1_update_500_frozen
+
+network:
+  hidden_size: 512
+  num_layers: 3
+
+ppo:
+  batch_games: 8
+  rollout_steps: 16
+  gamma: 1.0
+  gae_lambda: 0.95
+  clip_epsilon: 0.2
+  entropy_coef: 0.01
+  value_coef: 0.5
+  max_grad_norm: 0.5
+  learning_rate: 0.0003
+  epochs: 1
+  minibatches: 2
+
+reward:
+  terminal_scale: 50.0
+  potential_shaping_initial: 1.0
+  potential_shaping_final: 0.0
+  potential_shaping_anneal_steps: 5000000
+
+evaluation:
+  games: 2
+  duplicate: true
+  shuffle_bank_seed: 20260704
+  batch_games: 2
@@ -0,0 +1,40 @@
+run:
+  experiment_name: gates-1-2-warmstart-gate3-exploiter
+  seed: 20260705
+  learner_seat: 0
+  total_updates: 900
+  log_every: 10
+  checkpoint_every: 100
+  artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2/exploiters
+
+opponent:
+  name: league_v1_update_500_frozen
+
+network:
+  hidden_size: 512
+  num_layers: 3
+
+ppo:
+  batch_games: 8192
+  rollout_steps: 400
+  gamma: 1.0
+  gae_lambda: 0.95
+  clip_epsilon: 0.2
+  entropy_coef: 0.01
+  value_coef: 0.5
+  max_grad_norm: 0.5
+  learning_rate: 0.0003
+  epochs: 4
+  minibatches: 128
+
+reward:
+  terminal_scale: 50.0
+  potential_shaping_initial: 0.0
+  potential_shaping_final: 0.0
+  potential_shaping_anneal_steps: 0
+
+evaluation:
+  games: 2000
+  duplicate: true
+  shuffle_bank_seed: 20260704
+  batch_games: 8192
@@ -0,0 +1,35 @@
+run:
+  artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2-smoke
+  report_path: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2-smoke/report.md
+  summary_path: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2-smoke/summary.jsonl
+  shuffle_bank_seed: 20260704
+  gpu_budget_hours: 1.0
+
+target:
+  config: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/main_ppo_config.json
+  checkpoint: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/snapshots/cycle_01_update_000500
+
+gate1:
+  games: 2
+  batch_games: 2
+  delta_games: 2
+  delta_target_events: 2
+  delta_pairs: 2
+  delta_batch_events: 1
+
+gate2:
+  eval_games: 2
+  pass_threshold: 0.55
+  exploiters:
+    - name: long_random_smoke
+      config: configs/jax_ppo/gates-1-2-exploiter-smoke.yaml
+
+repair:
+  max_cycles: 1
+  league_template: configs/jax_ppo/league-smoke.yaml
+  experiment_name: jax-ppo-gates-1-2-repair-smoke
+  artifact_subdir: gate2c
+  evaluation_games: 2
+  evaluation_batch_games: 2
+  guard_expert_ci_low: -1000.0
+  guard_max_steps_rate: 1.0
@@ -0,0 +1,44 @@
+run:
+  artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/gates-1-2
+  report_path: docs/reports/gates-1-2-2026-07-05.md
+  summary_path: docs/reports/gates-1-2-2026-07-05-summary.jsonl
+  shuffle_bank_seed: 20260704
+  gpu_budget_hours: 8.0
+
+target:
+  config: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/main_ppo_config.json
+  checkpoint: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/snapshots/cycle_01_update_000500
+
+gate1:
+  games: 2000
+  batch_games: 8192
+  delta_games: 200
+  delta_target_events: 500
+  delta_pairs: 64
+  delta_batch_events: 16
+
+gate2:
+  eval_games: 2000
+  pass_threshold: 0.55
+  exploiters:
+    - name: long_random
+      config: configs/jax_ppo/gates-1-2-exploiter-long-random.yaml
+      notes: random init + shaping anneal, extended budget
+    - name: warmstart_gate3
+      config: configs/jax_ppo/gates-1-2-exploiter-warmstart.yaml
+      resume: /mnt/2tbhdd/coolrl-lost-cities-artifacts/ladder-v2/2026-07-05_013223_jax-ppo-ladder-v2-expert/latest
+      notes: ladder v2 gate-3 warm start, shaping disabled
+    - name: replay_exploiter
+      config: configs/jax_ppo/gates-1-2-exploiter-replay.yaml
+      resume: /mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/exploiters/2026-07-05_060727_jax-ppo-league-v1-cycle-1-exploiter/latest
+      notes: league v1 exploiter warm start, shaping disabled
+
+repair:
+  max_cycles: 3
+  league_template: configs/jax_ppo/league-v1.yaml
+  experiment_name: jax-ppo-gates-1-2-repair
+  artifact_subdir: gate2c
+  evaluation_games: 2000
+  evaluation_batch_games: 8192
+  guard_expert_ci_low: 0.0
+  guard_max_steps_rate: 0.02