Add JAX PPO static-opponent trainer

2026-07-04 22:30:16 +09:00
parent 768f74693d
commit 9e27f42f27
12 changed files with 1682 additions and 0 deletions
@@ -0,0 +1,40 @@
+run:
+  experiment_name: jax-ppo-balanced
+  seed: 20260704
+  learner_seat: 0
+  total_updates: 250
+  log_every: 1
+  checkpoint_every: 10
+  artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/jax-ppo-static-opponents
+
+opponent:
+  name: heuristic_balanced
+
+network:
+  hidden_size: 512
+  num_layers: 3
+
+ppo:
+  batch_games: 8192
+  rollout_steps: 400
+  gamma: 1.0
+  gae_lambda: 0.95
+  clip_epsilon: 0.2
+  entropy_coef: 0.01
+  value_coef: 0.5
+  max_grad_norm: 0.5
+  learning_rate: 0.0003
+  epochs: 4
+  minibatches: 128
+
+reward:
+  terminal_scale: 50.0
+  potential_shaping_initial: 1.0
+  potential_shaping_final: 0.0
+  potential_shaping_anneal_steps: 5000000
+
+evaluation:
+  games: 10000
+  duplicate: true
+  shuffle_bank_seed: 20260704
+  batch_games: 8192
@@ -0,0 +1,40 @@
+run:
+  experiment_name: jax-ppo-cautious
+  seed: 20260704
+  learner_seat: 0
+  total_updates: 250
+  log_every: 1
+  checkpoint_every: 10
+  artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/jax-ppo-static-opponents
+
+opponent:
+  name: heuristic_cautious
+
+network:
+  hidden_size: 512
+  num_layers: 3
+
+ppo:
+  batch_games: 8192
+  rollout_steps: 400
+  gamma: 1.0
+  gae_lambda: 0.95
+  clip_epsilon: 0.2
+  entropy_coef: 0.01
+  value_coef: 0.5
+  max_grad_norm: 0.5
+  learning_rate: 0.0003
+  epochs: 4
+  minibatches: 128
+
+reward:
+  terminal_scale: 50.0
+  potential_shaping_initial: 1.0
+  potential_shaping_final: 0.0
+  potential_shaping_anneal_steps: 5000000
+
+evaluation:
+  games: 10000
+  duplicate: true
+  shuffle_bank_seed: 20260704
+  batch_games: 8192
@@ -0,0 +1,40 @@
+run:
+  experiment_name: jax-ppo-discard-only
+  seed: 20260704
+  learner_seat: 0
+  total_updates: 250
+  log_every: 1
+  checkpoint_every: 10
+  artifact_root: /mnt/2tbhdd/coolrl-lost-cities-artifacts/jax-ppo-static-opponents
+
+opponent:
+  name: discard_only
+
+network:
+  hidden_size: 512
+  num_layers: 3
+
+ppo:
+  batch_games: 8192
+  rollout_steps: 400
+  gamma: 1.0
+  gae_lambda: 0.95
+  clip_epsilon: 0.2
+  entropy_coef: 0.01
+  value_coef: 0.5
+  max_grad_norm: 0.5
+  learning_rate: 0.0003
+  epochs: 4
+  minibatches: 128
+
+reward:
+  terminal_scale: 50.0
+  potential_shaping_initial: 1.0
+  potential_shaping_final: 0.0
+  potential_shaping_anneal_steps: 5000000
+
+evaluation:
+  games: 10000
+  duplicate: true
+  shuffle_bank_seed: 20260704
+  batch_games: 8192
@@ -0,0 +1,40 @@
+run:
+  experiment_name: jax-ppo-smoke
+  seed: 20260704
+  learner_seat: 0
+  total_updates: 1
+  log_every: 1
+  checkpoint_every: 1
+  artifact_root: runs/tmp/jax-ppo-artifacts
+
+opponent:
+  name: discard_only
+
+network:
+  hidden_size: 64
+  num_layers: 2
+
+ppo:
+  batch_games: 64
+  rollout_steps: 64
+  gamma: 1.0
+  gae_lambda: 0.95
+  clip_epsilon: 0.2
+  entropy_coef: 0.01
+  value_coef: 0.5
+  max_grad_norm: 0.5
+  learning_rate: 0.0003
+  epochs: 1
+  minibatches: 4
+
+reward:
+  terminal_scale: 50.0
+  potential_shaping_initial: 1.0
+  potential_shaping_final: 0.0
+  potential_shaping_anneal_steps: 100000
+
+evaluation:
+  games: 128
+  duplicate: true
+  shuffle_bank_seed: 20260704
+  batch_games: 64