Focus project on JAX PPO
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
# Deep CFR archive
|
||||
|
||||
These YAML files are retained only to reproduce historical Deep CFR work. The
|
||||
supported training stack is JAX PPO; start with `configs/jax_ppo/` instead.
|
||||
@@ -0,0 +1,114 @@
|
||||
run:
|
||||
experiment_name: deep-cfr-default
|
||||
seed: 79
|
||||
max_iterations: 1000
|
||||
max_minutes: null
|
||||
device: cuda
|
||||
use_amp: false
|
||||
deterministic: false
|
||||
|
||||
rules:
|
||||
n_colors: 5
|
||||
n_ranks: 9
|
||||
min_rank: 2
|
||||
n_handshakes: 3
|
||||
hand_size: 8
|
||||
expedition_penalty: -20
|
||||
bonus_threshold: 8
|
||||
bonus_amount: 20
|
||||
|
||||
encoding:
|
||||
derived_playability: true
|
||||
slot_aware_playability: true
|
||||
|
||||
network:
|
||||
hidden_size: 512
|
||||
num_layers: 3
|
||||
activation: relu
|
||||
|
||||
traversal:
|
||||
traversals_per_player: 280
|
||||
max_depth: null
|
||||
max_nodes_per_traversal: 1000
|
||||
regret_matching_epsilon: 0.0001
|
||||
outcome_sampling_epsilon: 0.05
|
||||
outcome_sampling_value_clip: 500.0
|
||||
outcome_unsampled_regret: zero
|
||||
cutoff_value_mode: score_diff
|
||||
cutoff_rollouts: 0
|
||||
cutoff_rollout_policy: random
|
||||
cutoff_rollout_max_steps: 300
|
||||
opponent_policy: average_strategy
|
||||
strategy_sample_interval: 1
|
||||
store_strategy_on_traverser_nodes: true
|
||||
store_strategy_on_opponent_nodes: false
|
||||
num_workers: 8
|
||||
worker_chunk_size: 64
|
||||
progress_every_traversals: 0
|
||||
endpoint_depth_bucket_width: 100
|
||||
endpoint_depth_bucket_max: 1000
|
||||
inference_backend: local
|
||||
scheduler: interleaved
|
||||
interleave_width: 64
|
||||
interleave_max_batch: 128
|
||||
|
||||
regret_matching:
|
||||
all_negative_fallback: argmax_tiebreak
|
||||
|
||||
training_weighting:
|
||||
mode: lcfr
|
||||
|
||||
self_play:
|
||||
snapshot_every: 1
|
||||
max_snapshots: 0
|
||||
anchor_probability: 0.0
|
||||
current_weight: 1.0
|
||||
recent_weight: 0.0
|
||||
older_weight: 0.0
|
||||
anchor_weight: 0.0
|
||||
recent_window: 5
|
||||
|
||||
optimization:
|
||||
advantage_updates_per_iteration: 512
|
||||
strategy_updates_per_iteration: 512
|
||||
advantage_batch_size: 1024
|
||||
strategy_batch_size: 1024
|
||||
learning_rate: 1.0e-4
|
||||
weight_decay: 0.0001
|
||||
grad_clip: 1.0
|
||||
|
||||
memory:
|
||||
advantage_capacity: 2000000
|
||||
strategy_capacity: 2000000
|
||||
|
||||
checkpoint:
|
||||
save_latest: true
|
||||
save_every: 50
|
||||
progress_interval_seconds: 20.0
|
||||
exact_resume: false
|
||||
|
||||
evaluation:
|
||||
eval_every: 5
|
||||
games: 100
|
||||
opponents:
|
||||
- random
|
||||
- discard_only
|
||||
- heuristic_cautious
|
||||
extended_eval_every: 50
|
||||
extended_opponents:
|
||||
- heuristic_balanced
|
||||
- heuristic_aggressive
|
||||
- heuristic_noisy
|
||||
max_steps: 10000
|
||||
on_max_steps: score_diff
|
||||
batch_size: 64
|
||||
device: trainer
|
||||
num_workers: 4
|
||||
|
||||
inference_server:
|
||||
device: cuda
|
||||
num_slots: null
|
||||
max_batch: 256
|
||||
batch_window_us: 200
|
||||
weight_sync_every: 1
|
||||
use_amp: false
|
||||
@@ -0,0 +1,109 @@
|
||||
run:
|
||||
experiment_name: deep-cfr-default-server
|
||||
seed: 79
|
||||
max_iterations: 1000
|
||||
max_minutes: null
|
||||
device: cuda
|
||||
use_amp: false
|
||||
deterministic: false
|
||||
|
||||
rules:
|
||||
n_colors: 5
|
||||
n_ranks: 9
|
||||
min_rank: 2
|
||||
n_handshakes: 3
|
||||
hand_size: 8
|
||||
expedition_penalty: -20
|
||||
bonus_threshold: 8
|
||||
bonus_amount: 20
|
||||
|
||||
encoding:
|
||||
derived_playability: true
|
||||
slot_aware_playability: true
|
||||
|
||||
network:
|
||||
hidden_size: 512
|
||||
num_layers: 3
|
||||
activation: relu
|
||||
|
||||
traversal:
|
||||
traversals_per_player: 280
|
||||
max_depth: null
|
||||
max_nodes_per_traversal: 1000
|
||||
regret_matching_epsilon: 0.0001
|
||||
outcome_sampling_epsilon: 0.2
|
||||
outcome_sampling_value_clip: 500.0
|
||||
outcome_unsampled_regret: zero
|
||||
cutoff_value_mode: score_diff
|
||||
cutoff_rollouts: 0
|
||||
cutoff_rollout_policy: random
|
||||
cutoff_rollout_max_steps: 300
|
||||
opponent_policy: average_strategy
|
||||
strategy_sample_interval: 1
|
||||
store_strategy_on_traverser_nodes: true
|
||||
store_strategy_on_opponent_nodes: false
|
||||
num_workers: 8
|
||||
worker_chunk_size: 8
|
||||
progress_every_traversals: 10
|
||||
endpoint_depth_bucket_width: 100
|
||||
endpoint_depth_bucket_max: 1000
|
||||
inference_backend: server
|
||||
|
||||
regret_matching:
|
||||
all_negative_fallback: argmax_tiebreak
|
||||
|
||||
training_weighting:
|
||||
mode: lcfr
|
||||
|
||||
self_play:
|
||||
snapshot_every: 1
|
||||
max_snapshots: 0
|
||||
anchor_probability: 0.0
|
||||
current_weight: 1.0
|
||||
recent_weight: 0.0
|
||||
older_weight: 0.0
|
||||
anchor_weight: 0.0
|
||||
recent_window: 5
|
||||
|
||||
optimization:
|
||||
advantage_updates_per_iteration: 512
|
||||
strategy_updates_per_iteration: 512
|
||||
advantage_batch_size: 1024
|
||||
strategy_batch_size: 1024
|
||||
learning_rate: 1.0e-4
|
||||
weight_decay: 0.0001
|
||||
grad_clip: 1.0
|
||||
|
||||
memory:
|
||||
advantage_capacity: 2000000
|
||||
strategy_capacity: 2000000
|
||||
|
||||
checkpoint:
|
||||
save_latest: true
|
||||
save_every: 50
|
||||
progress_interval_seconds: 20.0
|
||||
exact_resume: false
|
||||
|
||||
evaluation:
|
||||
eval_every: 5
|
||||
games: 100
|
||||
opponents:
|
||||
- random
|
||||
- discard_only
|
||||
- heuristic_balanced
|
||||
- heuristic_aggressive
|
||||
- heuristic_cautious
|
||||
- heuristic_noisy
|
||||
max_steps: 10000
|
||||
on_max_steps: score_diff
|
||||
batch_size: 64
|
||||
device: trainer
|
||||
num_workers: 4
|
||||
|
||||
inference_server:
|
||||
device: cuda
|
||||
num_slots: null
|
||||
max_batch: 256
|
||||
batch_window_us: 200
|
||||
weight_sync_every: 1
|
||||
use_amp: false
|
||||
@@ -0,0 +1,37 @@
|
||||
run:
|
||||
experiment_name: smoke
|
||||
max_iterations: 1
|
||||
seed: 1
|
||||
device: cpu
|
||||
|
||||
network:
|
||||
hidden_size: 16
|
||||
|
||||
traversal:
|
||||
traversals_per_player: 1
|
||||
max_depth: 2
|
||||
max_nodes_per_traversal: 64
|
||||
num_workers: 0
|
||||
worker_chunk_size: 1
|
||||
|
||||
optimization:
|
||||
advantage_batch_size: 2
|
||||
strategy_batch_size: 2
|
||||
advantage_updates_per_iteration: 1
|
||||
strategy_updates_per_iteration: 1
|
||||
learning_rate: 0.001
|
||||
|
||||
memory:
|
||||
advantage_capacity: 1000
|
||||
strategy_capacity: 1000
|
||||
|
||||
checkpoint:
|
||||
save_every: 0
|
||||
save_latest: false
|
||||
|
||||
evaluation:
|
||||
eval_every: 0
|
||||
games: 2
|
||||
opponents:
|
||||
- random
|
||||
max_steps: 10000
|
||||
@@ -0,0 +1,4 @@
|
||||
# ISMCTS archive
|
||||
|
||||
These YAML files are retained only to reproduce historical ISMCTS work. The
|
||||
supported training stack is JAX PPO; start with `configs/jax_ppo/` instead.
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
# Autonomous cycle eval helper.
|
||||
# Usage: ./autonomous_cycle_eval.sh <run-prefix> [extra eval args...]
|
||||
# Finds latest run matching prefix, runs eval --ckpt latest.pt with 30 games,
|
||||
# and reports: timeouts, natural-end wins per opponent.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PREFIX="${1:-}"
|
||||
shift || true
|
||||
|
||||
if [ -z "$PREFIX" ]; then
|
||||
echo "usage: $0 <run-prefix> [extra eval args...]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RUN=$(ls -td runs/*${PREFIX}* 2>/dev/null | head -1)
|
||||
if [ -z "$RUN" ]; then
|
||||
echo "no run matching ${PREFIX}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CKPT="$RUN/latest.pt"
|
||||
if [ ! -f "$CKPT" ]; then
|
||||
echo "no checkpoint at $CKPT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== eval $CKPT ==="
|
||||
uv run python -m coolrl_lost_cities.games.classic.ismcts.cli \
|
||||
eval --ckpt "$CKPT" --games 30 --verbose "$@" 2>&1
|
||||
@@ -0,0 +1,57 @@
|
||||
run:
|
||||
experiment_name: ismcts-default
|
||||
max_iterations: 500
|
||||
seed: 1
|
||||
device: cuda
|
||||
rules:
|
||||
n_colors: 5
|
||||
n_ranks: 9
|
||||
min_rank: 2
|
||||
n_handshakes: 3
|
||||
hand_size: 8
|
||||
expedition_penalty: -20
|
||||
bonus_threshold: 8
|
||||
bonus_amount: 20
|
||||
encoding:
|
||||
derived_playability: true
|
||||
slot_aware_playability: true
|
||||
network:
|
||||
kind: mlp
|
||||
hidden_size: 768
|
||||
num_layers: 4
|
||||
activation: relu
|
||||
mcts:
|
||||
n_simulations: 50
|
||||
c_puct: 5.0
|
||||
max_depth: 200
|
||||
parallel_simulations: 64
|
||||
virtual_loss_value: 5.0
|
||||
eval_n_simulations: 16
|
||||
rollout_policy: heuristic_balanced
|
||||
use_rollout_value: false
|
||||
root_dirichlet_alpha: 0.3
|
||||
root_dirichlet_epsilon: 0.4
|
||||
temperature:
|
||||
training: 1.0
|
||||
eval: 0.0
|
||||
training:
|
||||
games_per_iter: 10
|
||||
gradient_steps_per_iter: 10
|
||||
batch_size: 128
|
||||
replay_capacity: 100000
|
||||
interleave_games: 8
|
||||
interleave_max_batch: 64
|
||||
num_workers: 8
|
||||
worker_device: cuda
|
||||
optimization:
|
||||
learning_rate: 0.0003
|
||||
grad_clip: 5.0
|
||||
checkpoint:
|
||||
save_every: 20
|
||||
save_latest: true
|
||||
evaluation:
|
||||
eval_every: 5
|
||||
games: 5
|
||||
opponents: [random, discard-only, heuristic-cautious]
|
||||
max_steps: 500
|
||||
num_workers: 8
|
||||
@@ -0,0 +1,50 @@
|
||||
run:
|
||||
experiment_name: ismcts-mini
|
||||
max_iterations: 50
|
||||
seed: 1
|
||||
device: cpu
|
||||
rules:
|
||||
n_colors: 3
|
||||
n_ranks: 5
|
||||
min_rank: 2
|
||||
n_handshakes: 1
|
||||
hand_size: 4
|
||||
expedition_penalty: -20
|
||||
bonus_threshold: 4
|
||||
bonus_amount: 20
|
||||
encoding:
|
||||
derived_playability: true
|
||||
slot_aware_playability: true
|
||||
network:
|
||||
kind: mlp
|
||||
hidden_size: 128
|
||||
num_layers: 2
|
||||
activation: relu
|
||||
mcts:
|
||||
n_simulations: 50
|
||||
c_puct: 1.5
|
||||
max_depth: 100
|
||||
parallel_simulations: 8
|
||||
virtual_loss_value: 1.0
|
||||
temperature:
|
||||
training: 1.0
|
||||
eval: 0.0
|
||||
training:
|
||||
games_per_iter: 10
|
||||
gradient_steps_per_iter: 10
|
||||
batch_size: 128
|
||||
replay_capacity: 50000
|
||||
interleave_games: 8
|
||||
interleave_max_batch: 64
|
||||
optimization:
|
||||
learning_rate: 0.001
|
||||
grad_clip: 5.0
|
||||
checkpoint:
|
||||
save_every: 0
|
||||
save_latest: true
|
||||
evaluation:
|
||||
eval_every: 5
|
||||
games: 20
|
||||
opponents: [random, discard-only, heuristic-cautious]
|
||||
max_steps: 300
|
||||
num_workers: 1
|
||||
Reference in New Issue
Block a user