diff --git a/.github/workflows/deploy-web.yml b/.github/workflows/deploy-web.yml new file mode 100644 index 0000000..f7c0b05 --- /dev/null +++ b/.github/workflows/deploy-web.yml @@ -0,0 +1,46 @@ +name: Deploy web client + +on: + push: + branches: [main] + paths: + - "web/**" + - ".github/workflows/deploy-web.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: github-pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: web/package-lock.json + - run: npm ci + working-directory: web + - run: npm run build -- --base=/${{ github.event.repository.name }}/ + working-directory: web + - uses: actions/upload-pages-artifact@v3 + with: + path: web/dist + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index d1fe164..ce83ce4 100644 --- a/.gitignore +++ b/.gitignore @@ -28,8 +28,11 @@ tools/julia/ .pytest_cache .ruff_cache -# Web dependencies and locally exported deployment models +# Web dependencies and locally exported models. The verified public browser +# policy below is the one exception: it is deliberately served as a static asset. web/node_modules/ web/*.tsbuildinfo web/public/models/*.onnx web/public/models/*.json +!web/public/models/jax-ppo.onnx +!web/public/models/jax-ppo.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..b40991c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,18 @@ +stages: + - deploy + +deploy-pages: + stage: deploy + image: node:22 + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + changes: + - web/**/* + - .gitlab-ci.yml + script: + - cd web + - npm ci + - npm run build -- --base="${CI_PAGES_URL}/" + - mv dist ../public + pages: + publish: public diff --git a/AGENTS.md b/AGENTS.md index f69b4fb..c1d8a80 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,444 +1,98 @@ # AGENTS.md -This repository is managed with `uv`. Use `uv run ...` for commands so the -project environment and Cython extensions are built/loaded consistently. +This repository is managed with `uv`. Use `uv run ...` so the project +environment and Cython extensions are built and loaded consistently. -## Project Layout +## Active project path -- `src/coolrl_lost_cities/games/classic/game.pyx`: Cython Lost Cities engine. -- `src/coolrl_lost_cities/games/classic/deep_cfr/`: Deep CFR training, - traversal, evaluation, analysis, and CLI code. -- `configs/deep_cfr/`: active Deep CFR YAML configs (kebab-case filenames). - Currently holds two: - - `default.yaml`: the canonical "best-known" baseline. Start here, then - override fields via `--set` for experiments/ablations. - - `smoke.yaml`: 1-iter sanity check for the training loop. -- `configs/archive/`: retired/historical configs. Don't modify; reference - if you need to reproduce an old run. -- `runs/`: generated training runs. Gitignored, may be a symlink to larger - storage. Layout: - - `runs/archive/`: past runs. **Do not modify or delete.** - - `runs/tmp/`: smoke, tests, throwaway. Free to `rm -rf` anytime. - - `runs/_/`: real experiments (flat). - Promote a `runs/<...>` directory to `runs/archive/` with a manual `mv` - once analysis is complete. -- `docs/`: profiling notes, migration notes, and experiment documentation. +The active product and research path is **JAX + PPO**: -## Core Commands +- `src/lost_cities_jax/`: pure JAX rules, observations, PPO, evaluation, + league, and human-play tools. +- `configs/jax_ppo/`: active YAML configurations. +- `web/`: static TypeScript client with the final JAX PPO ONNX policy. +- `web/public/models/jax-ppo.onnx`: verified browser policy. Keep its manifest + in sync and do not replace it without running the export validation. -Run lint: +Historical Deep CFR and ISMCTS implementations live under +`src/coolrl_lost_cities/games/classic/`. Their configs are in `legacy/`; they +are reproduction-only and must not be used as the default for new work. See +`docs/legacy.md`. + +## Core commands ```bash uv run ruff check . -``` - -Run all tests: - -```bash +uv run pytest -q tests/lost_cities_jax uv run pytest -q +uv run lost-cities-jax-ppo --help ``` -Run focused Deep CFR tests: +Web checks: ```bash -uv run pytest -q tests/games/classic/test_deep_cfr_trainer.py +cd web +npm test +npm run build ``` -Run the CLI through the console script: +## JAX PPO workflow + +CPU smoke runs write disposable artifacts under `runs/tmp/`: ```bash -uv run lost-cities-deep-cfr --help +uv run lost-cities-jax-ppo rollout-smoke --config configs/jax_ppo/smoke.yaml +uv run lost-cities-jax-ppo train --config configs/jax_ppo/smoke.yaml ``` -Equivalent module form: +GPU training must hold the shared compute lock. Use a real `tmux` session for +long user-observable jobs: ```bash -uv run python -m coolrl_lost_cities.games.classic.deep_cfr.cli --help +flock -n .compute.lock uv run --with 'jax[cuda12]' lost-cities-jax-ppo train \ + --config configs/jax_ppo/balanced.yaml \ + --set run.artifact_root=runs/jax-ppo ``` -## Deep CFR Training - -The CLI auto-derives the run directory from `run.experiment_name` plus a -timestamp. By default runs land under `runs/tmp/`; pass `--keep` for a real -experiment that should live under `runs/`. - -Smoke / throwaway run (lands in `runs/tmp/`): +Evaluation is deterministic and does not need the lock: ```bash -uv run lost-cities-deep-cfr train --config configs/deep_cfr/smoke.yaml -# → runs/tmp/_smoke/ +uv run lost-cities-jax-ppo eval \ + --config configs/jax_ppo/balanced.yaml \ + --checkpoint runs/jax-ppo//latest \ + --opponent heuristic_balanced --games 10000 --duplicate ``` -Real experiment (lands in `runs/`): +Generated artifacts under `runs/` are gitignored. Do not modify or delete +`runs/archive/`. + +## Static browser policy + +The checked-in ONNX policy is small enough for static distribution. To replace +it, export a validated checkpoint and rebuild the web app: ```bash -uv run lost-cities-deep-cfr train \ - --config configs/deep_cfr/default.yaml \ - --keep -# → runs/_deep-cfr-default/ +uv run --with onnx scripts/export_jax_ppo_onnx.py \ + --checkpoint /path/to/checkpoint \ + --output web/public/models/jax-ppo.onnx +cd web && npm test && npm run build ``` -Variant / ablation (override one field; keep slug informative): +Commit the ONNX file and its JSON manifest together. The app must continue to +use a base-relative model URL so static subpath hosts work. -```bash -uv run lost-cities-deep-cfr train \ - --config configs/deep_cfr/default.yaml \ - --keep \ - --set training_weighting.mode=none \ - --set run.experiment_name=ablation-no-lcfr -# → runs/_ablation-no-lcfr/ -``` +## Documentation -Short fixed-iteration run: +- Active plans: `docs/plans/`. +- Retired plans and historical evidence: `docs/plans/archive/`, + `docs/archive/`, and `docs/research/`. +- Do not edit archival documents; add a new dated note or active plan instead. -```bash -uv run lost-cities-deep-cfr train \ - --config configs/deep_cfr/default.yaml \ - --set run.max_iterations=100 \ - --set checkpoint.save_every=0 -``` +When changing docs, run `scripts/librarian.sh` to validate Markdown links and +`file:line` citations. -Resume (path required, no shortcut): +## Git policy -```bash -uv run lost-cities-deep-cfr train \ - --config configs/deep_cfr/default.yaml \ - --resume runs/_/latest.pt -``` - -When `--resume` is given, the trainer reuses the resumed checkpoint's parent -directory; no new timestamped folder is created. - -Useful train controls: - -- `--keep`: real experiment, write under `runs/` (default is `runs/tmp/`). -- `--resume PATH`: resume from a specific checkpoint. `PATH` is required. -- `--set PATH=VALUE`: override config fields. Repeatable, parses values as - YAML (e.g. `--set traversal.num_workers=4`, `--set run.max_minutes=null`). - -Common `--set` overrides: - -- `--set run.device=cuda`: set the trainer device. -- `--set run.experiment_name=foo-v2`: change the slug used in the run dir - name (kebab-case). -- `--set checkpoint.exact_resume=true`: require checkpoint config compatibility. -- `--set checkpoint.save_latest=false --set checkpoint.save_every=0`: - disable checkpoint writes. -- `--set checkpoint.save_every=0`: keep only `latest.pt` (no archives). -- `--set checkpoint.save_every=N`: archive every N iterations. - -## Naming Conventions - -- **Directory names, run dirs, config filenames, `experiment_name` values**: - kebab-case (`deep-cfr-color-shared-512x3.yaml`, - `runs/2026-05-08_103045_color-attn-v2/`). -- **YAML keys, Python identifiers, config field names**: snake_case - (unchanged: `hidden_size`, `traversals_per_player`, `experiment_name`). -- The CLI converts `run.experiment_name` to a kebab slug when building the - run directory, so values may contain spaces or mixed case. - -## Long Runs - -Run long jobs in a real `tmux` session so the user can attach and stop them. -Do not rely on Codex command sessions for long user-observable training runs. - -Start a long unbounded run: - -```bash -tmux new-session -s coolrl-deepcfr-unbounded \ - -c /home/coolguy/dev/coolrl-lost-cities \ - 'uv run lost-cities-deep-cfr train \ - --config configs/deep_cfr/default.yaml \ - --set run.max_iterations=null \ - --set run.max_minutes=null \ - --keep' -``` - -Attach later: - -```bash -tmux attach -t coolrl-deepcfr-unbounded -``` - -Detach without stopping: - -```text -Ctrl+B, D -``` - -Stop training: - -```text -Ctrl+C -``` - -Follow logs from another terminal: - -```bash -tail -f runs/_/train.log -``` - -The unbounded config intentionally has: - -```yaml -run: - max_iterations: null - max_minutes: null -checkpoint: - save_every: 100 -``` - -`latest.pt` is updated continuously; archive checkpoints are written every 100 -iterations. If disk is tight, set `--set checkpoint.save_every=0` (keep only -`latest.pt`) or increase `save_every`. - -## Compute Lock - -여러 에이전트가 한 머신을 공유하므로, train과 속도 벤치마크는 -repo 루트 `.compute.lock`을 잡고 실행한다: - -```bash -flock -n .compute.lock uv run lost-cities-deep-cfr train ... -``` - -승률/점수만 뽑는 eval과 `analyze`는 결정적이라 락 불필요. - -## Evaluation And Analysis - -Evaluate a checkpoint: - -```bash -uv run lost-cities-deep-cfr eval \ - --checkpoint runs//latest.pt \ - --opponent random \ - --games 100 \ - --device cpu -``` - -Save evaluation game records: - -```bash -uv run lost-cities-deep-cfr eval \ - --checkpoint runs//latest.pt \ - --opponent random \ - --games 100 \ - --device cpu \ - --save-games runs//eval_random_games.json -``` - -Generate analysis plots from `metrics.jsonl`: - -```bash -uv run lost-cities-deep-cfr analyze \ - --run runs/ -``` - -Write plots to a separate directory: - -```bash -uv run lost-cities-deep-cfr analyze \ - --run runs/ \ - --output-dir runs//analysis -``` - -The analyzer reads `metrics.jsonl` and writes PNG files grouped by diagnostic -section. Opponents are compared within each plot using fixed colors. The -`lost-cities-deep-cfr analyze` subcommand uses the analyzer default smoothing -window, currently 1 iteration (no smoothing), and supports `--max-iteration`. - -For smoothing controls, run the analyzer module directly: - -```bash -uv run python -m coolrl_lost_cities.games.classic.deep_cfr.analyze \ - --run runs/ \ - --smoothing-window 5 -``` - -Use `--no-smoothing` to force no moving average. - -Current output files: - -- `analysis_01_loss.png` -- `analysis_02_match.png` -- `analysis_03_action.png` -- `analysis_04_gameflow.png` -- `analysis_05_open_quality.png` -- `analysis_06_expedition_outcomes.png` -- `analysis_07_calibration.png` -- `analysis_08_traversal.png` -- `analysis_09_selectivity.png` -- `analysis_final_eval_summary.png` - -## Runtime Artifacts - -Each training run writes: - -- `metrics.jsonl`: structured metrics, one completed iteration per line. -- `train.log`: human-readable timestamped logs. -- `runtime_progress.json`: latest progress snapshot. -- `latest.pt`: latest checkpoint. -- `iteration_*.pt`: archive checkpoints when enabled. -- `config.json`: resolved config for the run. - -If a run is stopped mid-iteration, the in-progress iteration may not appear in -`metrics.jsonl`. Analyze the latest completed metric row. - -## Weights & Biases (Optional) - -Metrics can be mirrored to W&B. `wandb` is an optional extra; default -installs and runs do not require it. - -Install: - -```bash -uv sync --extra wandb -``` - -Run with W&B: - -```bash -# Offline: no login, writes to /wandb/offline-run-*/ -uv run lost-cities-deep-cfr train --config <...> --wandb --wandb-mode offline - -# Online: requires `uv run wandb login` once, then real-time upload -uv run lost-cities-deep-cfr train --config <...> --wandb -``` - -W&B data is stored **per run** at `/wandb/`, not at a global -`runs/wandb/`. Each training run gets its own subfolder, so moving or -deleting a run directory carries its W&B data along with it. - -Sync offline runs to wandb.ai later: - -```bash -wandb sync runs//wandb/offline-run-* -``` - -Flags: - -- `--wandb`: enable W&B mirroring. -- `--wandb-project `: defaults to `coolrl-lost-cities`. -- `--wandb-name `: W&B run name; defaults to `run.experiment_name`. -- `--wandb-mode {online,offline,disabled}`: default `online`. -- `--wandb-group `: group related runs from one experiment/hypothesis. -- `--wandb-job-type `: role of this run, e.g. `train`, `eval`, `sweep`, or `smoke`. -- `--wandb-tag `: tag the run; repeatable. - -W&B is purely additive — `metrics.jsonl` remains the source of truth, and -`analyze` reads `metrics.jsonl`, not W&B. Disabling W&B never breaks -training, resume, or analysis. - -### Groups, notes, and tags - -Use `--wandb-group` for the experiment or hypothesis family, e.g. -`model-size-grid-2026-05-08` or `strict-curriculum-v1`. Use `--wandb-name` -for the individual run, e.g. `512x3-seed79`, and `--wandb-job-type` for the -run role (`train`, `eval`, `sweep`, `smoke`). - -Use `--wandb-notes` for the run's *purpose* (free-form prose) and `--wandb-tag` -for *categories you might filter on later* (short kebab-case keywords, -repeatable). Otherwise use them however you like. Just avoid: - -- Tags that duplicate `config` (`lr-1e-4`, `traversal-280`) — W&B already - indexes config fields. -- Tags that are unique per run (`test-1`, `2026-05-07`) — that's the run - name and timestamp's job. -- Tag-as-sentence (`tested-bigger-traversal-with-lcfr`) — that belongs in - `--wandb-notes`. - -**Notes length**: 3–5 lines, commit-message-body length. Should answer -*why* (hypothesis), *what* (key config delta), and *baseline* (run/iter -to compare against). Long analyses go in `docs/` and are linked from -notes; don't paste them in. - -### Comparing two runs - -Default is **sequential, single seed**. Run baseline first, then the -treatment with exactly one config change, both with the same `run.seed`. -Put both in the same W&B group (e.g. `--wandb-group lr-bump-v1`) and optionally -tag both with a shared hypothesis tag (e.g. `--wandb-tag lr-bump`) so they show -up together in W&B's Compare Runs view. - -Do **not** run multiple seeds per condition unless explicitly asked — -that doubles or quintuples wall-clock and isn't the default protocol. -Single-seed comparison is enough to surface a signal; multi-seed is a -follow-up to confirm it. - -Do **not** run two trainings in parallel on the same GPU — VRAM/SM -contention slows both unevenly and breaks the comparison. - -## Docs & Experiment Workflow - -### Where to write what - -| 기록 내용 | 쓸 곳 | -| --- | --- | -| 진행 중인 계획/가설 | `docs/plans/.md` (1 주제 1 파일) | -| 끝난 계획 | `docs/plans/archive/` (수동 `mv`) | -| 날짜 박힌 실험 기록 | `docs/archive/-YYYY-MM-DD.md` (immutable) | -| 항구적 알고리즘 노트 | `docs/research/.md` (`Last verified:` 헤더) | -| 비용/프로파일 | `docs/reports/-YYYY-MM-DD.md` (dated) | -| 스크래치 / 연구 스레드 인덱스 | `ideas.md` | - -Catalog of current research notes: [docs/research/README.md](docs/research/README.md). - -### Rules - -- `docs/archive/`, `runs/archive/`는 read-only. -- archive 본문 복붙 금지 — 대신 `Source:` 링크 + distill. -- `file.py:NN` 인용은 작성 시점에 `rg`로 검증. -- 한 주제 한 파일 — `foo-v2.md` 만들지 말 것. -- 파일당 ~500줄 soft cap — 강제 분할이 아니라 "다른 위치(archive/research)로 - 가야 할 내용이 누적됐는지" routing 점검 트리거. 초과 시 dated 실험은 - `docs/archive/`로, 항구적 분석은 `docs/research/`로 보내고 본 파일은 - 현재 상태 reference만 남긴다. - -## Notes For Future Agents - -- Prefer `rg`/`rg --files` for search. -- Use `apply_patch` for manual edits. -- Do not commit generated run artifacts from `runs/`. -- Cython-generated `.c` files are gitignored; edit `.pyx`/`.pxd` sources. -- Before committing, run `uv run ruff check .` and at least the relevant pytest - subset. For Deep CFR changes, run - `uv run pytest -q tests/games/classic/test_deep_cfr_trainer.py`. -- When changing docs, run `scripts/librarian.sh` to lint markdown link - integrity and `file:line` code citations across `docs/**`. See - `docs/plans/librarian.md` for the full design. - -## Git Branching Policy (READ THIS) - -**DO NOT create new git branches unless the user explicitly asks for one in -the current task. Work on whatever branch is currently checked out (default -`main`). This is a hard rule — no exceptions for "safety", "isolation", -"experiments", "work-in-progress", or any other self-justified reason.** - -Why this rule exists: -- This project intentionally develops on `main` with frequent small commits. -- Auto-created branches like `experiments/foo`, `feature/bar`, `wip/baz` - fragment review, hide work from the user, and require manual cleanup. -- The user has not authorized branch creation as a default behavior. If - they want a branch, they will say so explicitly ("make a branch", "PR - this", "isolate this in a branch", etc.). - -What you MUST do instead: -- Make commits directly on the currently checked-out branch. -- If you think a branch is justified, **stop and ask the user first** — - do not preemptively create one. -- If a tool or subagent invocation auto-suggests creating a branch - (e.g. PR-style workflows), refuse the branch creation step and commit - to the current branch. -- If you find yourself already on a non-default branch you didn't expect, - stop and ask the user — do not switch, do not create a new one, do not - reset. - -Forbidden without explicit user instruction: -- `git checkout -b ` -- `git switch -c ` -- `git branch ` -- `gh pr create` from an auto-created branch -- Any worktree creation that implicitly creates a new branch - -This rule applies to the main agent and to every subagent or tool the main -agent invokes. Pass it through in subagent prompts when delegating -git-touching work. +Do not create git branches unless the user explicitly asks in the current +task. Work on the checked-out branch. Before committing, run `uv run ruff +check .` and the relevant tests. Never commit generated run artifacts. diff --git a/README.md b/README.md index b506d19..467ec64 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,68 @@ # coolrl-lost-cities -Focused Lost Cities extraction from the legacy `coolrl` repository. +JAX PPO training and on-device browser play for the two-player card game Lost +Cities. The project provides a pure JAX rules engine, PPO training and +evaluation tools, and a static web client that runs the shipped final policy +with WebGPU when available and WebAssembly otherwise. -The current implementation starts with the classic two-player card game: +The active path is **JAX + PPO**. Deep CFR and ISMCTS are retained only as +historical research implementations; see [legacy notes](docs/legacy.md). -- classic 5-expedition rules by default -- Python/Cython game engine -- env wrapper -- random, discard-only, and safe-heuristic bots -- core rule, scoring, mask, env, canonical-state, bot, and GUI smoke tests +## Quick start -Training code, Deep CFR, learned-policy evaluation, desktop GUI, and an -on-device web client now live alongside the original rules port. - -## Development +Install the project, then run a complete CPU-sized sanity check: ```bash -uv run pytest tests/games/classic -uv run lost-cities-classic +uv sync +uv run lost-cities-jax-ppo rollout-smoke --config configs/jax_ppo/smoke.yaml +uv run lost-cities-jax-ppo train --config configs/jax_ppo/smoke.yaml ``` -For future GUI work, install the optional GUI dependencies: +The final command prints a run directory containing `latest`, `config.json`, +and `metrics.jsonl` under `runs/tmp/jax-ppo-artifacts/`. + +## Train and evaluate a PPO policy + +The committed configurations describe the opponent and training budget. For a +GPU run, use CUDA JAX and keep generated artifacts outside git: ```bash -uv sync --extra gui +flock -n .compute.lock uv run --with 'jax[cuda12]' lost-cities-jax-ppo train \ + --config configs/jax_ppo/balanced.yaml \ + --set run.artifact_root=runs/jax-ppo ``` -Run the classic pygame GUI: +Evaluate a saved checkpoint against a fixed opponent. Duplicate evaluation +swaps seats over the same shuffled games: ```bash -uv run lost-cities-classic-gui --mode pvc --bot safe-heuristic +uv run lost-cities-jax-ppo eval \ + --config configs/jax_ppo/balanced.yaml \ + --checkpoint runs/jax-ppo//latest \ + --opponent heuristic_balanced \ + --games 10000 \ + --duplicate ``` -The GUI uses the in-process Cython game engine. +Run `uv run lost-cities-jax-ppo --help` for training against saved opponents, +league runs, gates, human-play logs, and evaluation variants. -## On-device Web Client +## Browser client -The `web/` app runs its TypeScript rules engine and exported JAX PPO policy -entirely in the browser. It prefers WebGPU and falls back to WebAssembly. - -Export a local Orbax checkpoint and start Vite: +The final verified JAX PPO policy is shipped as a 3.1 MB static ONNX asset. +No server or local checkpoint is required to play it: ```bash -uv run --with onnx scripts/export_jax_ppo_onnx.py \ - --checkpoint /path/to/checkpoint \ - --output web/public/models/jax-ppo.onnx cd web -npm install +npm ci npm run dev ``` -See [web/README.md](web/README.md) for tests and model parity tooling. +`npm run build` produces a fully static site. The model path is deployment-base +aware, so the build can be served from GitHub Pages, GitLab Pages, or a normal +web root. Pushes to `main` deploy that build to both configured Pages hosts. +See [web/README.md](web/README.md) for model replacement, tests, and the +cross-runtime parity fixture. ## JAX Rules Engine diff --git a/docs/legacy.md b/docs/legacy.md new file mode 100644 index 0000000..bbf17b2 --- /dev/null +++ b/docs/legacy.md @@ -0,0 +1,25 @@ +# Legacy research stacks + +The supported project path is the JAX rules engine and JAX PPO tooling exposed +by `lost-cities-jax-ppo`. The historical Deep CFR and ISMCTS implementations +are preserved for research reproduction, but they are not current training +recipes and no longer have public console-script entry points. + +## What is preserved + +- Deep CFR sources: `src/coolrl_lost_cities/games/classic/deep_cfr/` +- ISMCTS sources: `src/coolrl_lost_cities/games/classic/ismcts/` +- Their retired configurations: `legacy/deep-cfr/configs/` and + `legacy/ismcts/configs/` +- Earlier plans and dated findings: `docs/plans/archive/`, `docs/archive/`, + and `docs/research/` + +These files remain in the repository so a historical experiment can be read or +reproduced against its original commit. They should not be chosen for new +models, benchmarks, or product work. + +## Current workflow + +Start from `configs/jax_ppo/`, run `lost-cities-jax-ppo`, and use the browser +client in `web/`. The root [README](../README.md) contains the short training, +evaluation, and static-web recipes. diff --git a/docs/performance.md b/docs/performance.md index fff6a9c..167ce13 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -1,7 +1,10 @@ -# Deep CFR Performance Notes +# Legacy Deep CFR Performance Notes -This document tracks current runtime bottlenecks for the active Deep CFR -training path. The numbers below are observational, not a benchmark contract. +> Historical record only. The supported training stack is JAX PPO; see the +> root [README](../README.md) and [legacy notes](legacy.md). + +This document records the Deep CFR runtime bottlenecks observed before the JAX +PPO transition. The numbers below are historical, not a benchmark contract. ## Current Default Runtime @@ -11,7 +14,7 @@ Source run: runs/tmp/2026-05-07_171535_deep-cfr-default/metrics.jsonl ``` -The run used `configs/deep_cfr/default.yaml` with CUDA enabled. At the time of +The run used `legacy/deep-cfr/configs/default.yaml` with CUDA enabled. At the time of inspection, completed metrics covered iterations 70 through 95. The training process was still running, so later rows may differ. @@ -88,7 +91,7 @@ This gives 560 traversals per iteration, split into 70 worker batches. ## Device Use The trainer constructs the advantage and strategy networks on `run.device`. -`configs/deep_cfr/default.yaml` sets: +`legacy/deep-cfr/configs/default.yaml` sets: ```yaml run: diff --git a/docs/plans/deep-cfr-selectivity.md b/docs/plans/archive/deep-cfr-selectivity.md similarity index 100% rename from docs/plans/deep-cfr-selectivity.md rename to docs/plans/archive/deep-cfr-selectivity.md diff --git a/docs/plans/librarian.md b/docs/plans/archive/librarian.md similarity index 99% rename from docs/plans/librarian.md rename to docs/plans/archive/librarian.md index 4aa884a..84af35b 100644 --- a/docs/plans/librarian.md +++ b/docs/plans/archive/librarian.md @@ -117,7 +117,7 @@ operator applies the patch. - ✅ AGENTS.md "Docs & Experiment Workflow" section landed (commit `09d5815`, 2026-05-07). -- ✅ Plan drafted at `docs/plans/librarian.md` (this file). +- ✅ Plan drafted at `docs/plans/archive/librarian.md` (this file). - ✅ Prompt moved: `.claude/agents/librarian.md` → `scripts/librarian-prompt.md`. Claude-specific subagent registration removed. diff --git a/docs/plans/model_size_experiment.md b/docs/plans/archive/model_size_experiment.md similarity index 98% rename from docs/plans/model_size_experiment.md rename to docs/plans/archive/model_size_experiment.md index 5df93d6..88e22a0 100644 --- a/docs/plans/model_size_experiment.md +++ b/docs/plans/archive/model_size_experiment.md @@ -1,11 +1,11 @@ # Plan: Model-Size Experiment (Keystone for Model-Scale Optimizations) > **⚠️ 스택 주의 — 이 문서는 Deep CFR / PyTorch 스택 전용이다.** -> `input_dim=365`, `configs/deep_cfr/`, `DeepCFRMLP` 기준으로 쓰였다. +> `input_dim=365`, `legacy/deep-cfr/configs/`, `DeepCFRMLP` 기준으로 쓰였다. > 현행 JAX PPO 스택(`OBS_DIM=454`, `configs/jax_ppo/`, `ActorCritic`)에는 > **적용되지 않는다.** PPO 쪽 모델 크기 문제는 -> [jax-ppo-model-size-ab.md](jax-ppo-model-size-ab.md)와 -> [../reports/jax-ppo-model-capacity-2026-07-12.md](../reports/jax-ppo-model-capacity-2026-07-12.md)를 볼 것. +> [jax-ppo-model-size-ab.md](../jax-ppo-model-size-ab.md)와 +> [../../reports/jax-ppo-model-capacity-2026-07-12.md](../../reports/jax-ppo-model-capacity-2026-07-12.md)를 볼 것. **Status:** Ready to execute **Owner:** Operator (runs grid on `home`); Codex (adds configs and runner script) diff --git a/docs/plans/option_b_interleaved_traversal.md b/docs/plans/archive/option_b_interleaved_traversal.md similarity index 100% rename from docs/plans/option_b_interleaved_traversal.md rename to docs/plans/archive/option_b_interleaved_traversal.md diff --git a/docs/plans/torch_compile.md b/docs/plans/archive/torch_compile.md similarity index 100% rename from docs/plans/torch_compile.md rename to docs/plans/archive/torch_compile.md diff --git a/docs/plans/jax-ppo-model-size-ab.md b/docs/plans/jax-ppo-model-size-ab.md index 40c2d6a..98019fd 100644 --- a/docs/plans/jax-ppo-model-size-ab.md +++ b/docs/plans/jax-ppo-model-size-ab.md @@ -7,8 +7,8 @@ **Background:** [docs/reports/jax-ppo-model-capacity-2026-07-12.md](../reports/jax-ppo-model-capacity-2026-07-12.md) > 이 계획은 은퇴한 Deep CFR/PyTorch 스택용 -> [model_size_experiment.md](model_size_experiment.md)를 **대체하지 않는다** — -> 그쪽은 다른 스택(`input_dim=365`, `configs/deep_cfr/`) 이야기다. 서로 무관하다. +> [model_size_experiment.md](archive/model_size_experiment.md)를 **대체하지 않는다** — +> 그쪽은 다른 스택(`input_dim=365`, `legacy/deep-cfr/configs/`) 이야기다. 서로 무관하다. ## 가설 diff --git a/docs/reports/final-cycles-and-human-play-2026-07-05.md b/docs/reports/final-cycles-and-human-play-2026-07-05.md index 109b351..5358ee9 100644 --- a/docs/reports/final-cycles-and-human-play-2026-07-05.md +++ b/docs/reports/final-cycles-and-human-play-2026-07-05.md @@ -80,4 +80,4 @@ Validation performed before final report: - `uv run pytest -q` passed: 257 passed, 1 skipped. - `uv run lost-cities-jax-ppo play --help` passed. - `uv run lost-cities-jax-ppo human-play summarize --log-dir /tmp/nonexistent-human-play-log` passed. -- `scripts/librarian.sh` found no link or code-citation errors; it still exits non-zero on the known pre-existing `docs/plans/deep-cfr-selectivity.md` 500-line soft cap. +- `scripts/librarian.sh` found no link or code-citation errors; it still exits non-zero on the known pre-existing `docs/plans/archive/deep-cfr-selectivity.md` 500-line soft cap. diff --git a/docs/reports/jax-ppo-model-capacity-2026-07-12.md b/docs/reports/jax-ppo-model-capacity-2026-07-12.md index c38754d..b33c362 100644 --- a/docs/reports/jax-ppo-model-capacity-2026-07-12.md +++ b/docs/reports/jax-ppo-model-capacity-2026-07-12.md @@ -23,8 +23,8 @@ Metrics: `.../final-cycles/2026-07-05/league/2026-07-05_191933_jax-ppo-final-cyc (`main_ppo_config.json` → `network: {hidden_size: 512, num_layers: 3}`). `hidden_size`를 다룬 기존 문서는 전부 은퇴한 Deep CFR/PyTorch 스택 것이다 -(`input_dim=365`, `configs/deep_cfr/` 기준). 특히 -[docs/plans/model_size_experiment.md](../plans/model_size_experiment.md)는 +(`input_dim=365`, `legacy/deep-cfr/configs/` 기준). 특히 +[docs/plans/model_size_experiment.md](../plans/archive/model_size_experiment.md)는 **현 JAX PPO 스택과 무관하다.** → 512×3은 실험으로 고른 값이 아니라 구 스택에서 복사돼 온 값이다. diff --git a/docs/research/deep-cfr-performance-experiments.md b/docs/research/deep-cfr-performance-experiments.md index c80cd2c..77eb754 100644 --- a/docs/research/deep-cfr-performance-experiments.md +++ b/docs/research/deep-cfr-performance-experiments.md @@ -54,5 +54,5 @@ The 8-worker interleaved path is more effective than a single-process CUDA path - `src/coolrl_lost_cities/games/classic/deep_cfr/trainer.py`: Implementation of AMP and training loops. - `src/coolrl_lost_cities/games/classic/deep_cfr/networks.py`: `DeepCFRMLP` architecture. -- `configs/deep_cfr/default.yaml`: Configuration for interleaved scheduler. -- `scripts/profile_gpu_forward.py`: GPU forward pass micro-benchmarks. \ No newline at end of file +- `legacy/deep-cfr/configs/default.yaml`: Configuration for interleaved scheduler. +- `scripts/profile_gpu_forward.py`: GPU forward pass micro-benchmarks. diff --git a/docs/research/optimization_sequencing.md b/docs/research/optimization_sequencing.md index ee95abf..6f4eaa1 100644 --- a/docs/research/optimization_sequencing.md +++ b/docs/research/optimization_sequencing.md @@ -6,10 +6,10 @@ | Lever | 어디서 nail되는지 | 현재 상태 | | --- | --- | --- | -| **Model size growth** (hidden ≥ 1024 / layers ≥ 6) | `docs/plans/model_size_experiment.md` | 인프라 미설치 | +| **Model size growth** (hidden ≥ 1024 / layers ≥ 6) | `docs/plans/archive/model_size_experiment.md` | 인프라 미설치 | | **Option B** (per-worker interleaved traversal) | plan 미작성 | 미시작 | | **AMP** trainer | `docs/plans/archive/amp_trainer.md` (구현 됨, default off) | 모델 키운 후 재측정 | -| **torch.compile** trainer | `docs/plans/torch_compile.md` | 모델 키운 후 재측정 | +| **torch.compile** trainer | `docs/plans/archive/torch_compile.md` | 모델 키운 후 재측정 | | **TensorRT** inference | plan 미작성 | 모델 키운 + eval dense 시점 | | **Option A re-enable** | 코드 있음 (default off) | 모델 키운 후 또는 Option B 후 | | **Julia port** | `docs/research/julia_port_evaluation.md` | Torch.jl 결과 대기 중. Flux FAIL. | diff --git a/docs/research/option-a-bench-result.md b/docs/research/option-a-bench-result.md index cf6a098..bcd0f1b 100644 --- a/docs/research/option-a-bench-result.md +++ b/docs/research/option-a-bench-result.md @@ -19,7 +19,7 @@ with torch.inference_mode(): advantages = networks[player](x).squeeze(0).detach().cpu().numpy().astype(np.float32) ``` -When `traversal.inference_backend` is set to `server` in `configs/deep_cfr/default.yaml`, the `networks[player]` call is intercepted by a `NetworkProxy` (instantiated in `workers.py`, around line 91). This proxy posts a request to the `InferenceServer` and blocks until a response is received via a per-slot event. +When `traversal.inference_backend` is set to `server` in `legacy/deep-cfr/configs/default.yaml`, the `networks[player]` call is intercepted by a `NetworkProxy` (instantiated in `workers.py`, around line 91). This proxy posts a request to the `InferenceServer` and blocks until a response is received via a per-slot event. The server's batching logic in `src/coolrl_lost_cities/games/classic/deep_cfr/inference_server.py` (around line 221) reports the realized batch size: @@ -46,7 +46,7 @@ The "structural ceiling" is that `batch_window_us` and `max_batch` tuning cannot ## Practical implication -Option A is deferred for the current small MLP models (512x3). The `local` backend remains the default in `configs/deep_cfr/default.yaml`. +Option A is deferred for the current small MLP models (512x3). The `local` backend remains the default in `legacy/deep-cfr/configs/default.yaml`. To unlock the projected GPU gains, the traversal must be restructured to drive batch sizes up. This leads to two primary paths: 1. **Option B (Interleaved Traversal):** Refactor the Cython traversal into a state machine that can advance multiple traversals concurrently per worker. Each worker would suspend at a policy call, batch its own requests, and resume continuations once the results return. diff --git a/docs/research/post-a-optimization-calculus.md b/docs/research/post-a-optimization-calculus.md index 410aa46..5b9f905 100644 --- a/docs/research/post-a-optimization-calculus.md +++ b/docs/research/post-a-optimization-calculus.md @@ -9,7 +9,7 @@ Why do architectural optimizations like `torch.compile` and TensorRT integration ## Code reference -The current baseline configuration is defined in `configs/deep_cfr/default.yaml`: +The historical baseline configuration is defined in `legacy/deep-cfr/configs/default.yaml`: ```yaml network: @@ -51,4 +51,4 @@ Avoid premature optimization with `torch.compile` or TensorRT on the current sma - `docs/archive/deep-cfr-performance-experiments-2026-05-07.md` (Small-model regression data) - `docs/archive/option-a-bench-result-2026-05-07.md` (Batched traversal benchmarks) -- `docs/performance.md` (Current runtime bottleneck profile) \ No newline at end of file +- `docs/performance.md` (Current runtime bottleneck profile) diff --git a/docs/research/strategy-memory-location.md b/docs/research/strategy-memory-location.md index 600ad75..dffa34a 100644 --- a/docs/research/strategy-memory-location.md +++ b/docs/research/strategy-memory-location.md @@ -46,7 +46,7 @@ sampling-mode branch. The `info_state` is computed by `_policy(state, player, ...)` for the *current acting player*, which is the right thing in both conventions. -`configs/deep_cfr/default.yaml` sets: +`legacy/deep-cfr/configs/default.yaml` sets: ```yaml store_strategy_on_traverser_nodes: true diff --git a/experiments/option_b_interleaved_traversal/README.md b/experiments/option_b_interleaved_traversal/README.md index 911e6af..7a70efb 100644 --- a/experiments/option_b_interleaved_traversal/README.md +++ b/experiments/option_b_interleaved_traversal/README.md @@ -1,6 +1,6 @@ # Option B Interleaved Traversal Prototype -Experiment-only prototype for `docs/plans/option_b_interleaved_traversal.md`. +Experiment-only prototype for `docs/plans/archive/option_b_interleaved_traversal.md`. It does not wire into the trainer and does not replace the production Cython recursive traversal path. @@ -37,7 +37,7 @@ uv run python experiments/option_b_interleaved_traversal/prototype_interleaved.p --output experiments/option_b_interleaved_traversal/results_cuda.json ``` -2026-05-07 results, `configs/deep_cfr/default.yaml`, RTX 3090 host: +2026-05-07 results, `legacy/deep-cfr/configs/default.yaml`, RTX 3090 host: | Device | Mode | total s | forward s | scheduler s | batch mean | batch max | speedup | | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | diff --git a/experiments/option_b_interleaved_traversal/prototype_interleaved.py b/experiments/option_b_interleaved_traversal/prototype_interleaved.py index 303fad3..d2e5ff6 100644 --- a/experiments/option_b_interleaved_traversal/prototype_interleaved.py +++ b/experiments/option_b_interleaved_traversal/prototype_interleaved.py @@ -650,7 +650,7 @@ def _build_proto_config(cfg: Any, max_depth: int | None, max_nodes: int | None) def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("--config", default="configs/deep_cfr/default.yaml") + parser.add_argument("--config", default="legacy/deep-cfr/configs/default.yaml") parser.add_argument("--device", default="cpu") parser.add_argument("--traversals", type=int, default=64) parser.add_argument("--interleave-width", type=int, default=32) diff --git a/experiments/traversal_policy_boundary/README.md b/experiments/traversal_policy_boundary/README.md index 960c68c..09d803c 100644 --- a/experiments/traversal_policy_boundary/README.md +++ b/experiments/traversal_policy_boundary/README.md @@ -30,7 +30,7 @@ uv run python experiments/traversal_policy_boundary/bench_policy_boundary.py \ --output experiments/traversal_policy_boundary/results_cuda.json ``` -2026-05-07 results, `configs/deep_cfr/default.yaml`, RTX 3090 host: +2026-05-07 results, `legacy/deep-cfr/configs/default.yaml`, RTX 3090 host: | Device | Component | Median us/call | p95 us/call | | --- | --- | ---: | ---: | diff --git a/experiments/traversal_policy_boundary/bench_policy_boundary.py b/experiments/traversal_policy_boundary/bench_policy_boundary.py index 000212f..4a08a5f 100644 --- a/experiments/traversal_policy_boundary/bench_policy_boundary.py +++ b/experiments/traversal_policy_boundary/bench_policy_boundary.py @@ -321,7 +321,7 @@ def _print_table(result: dict[str, Any]) -> None: def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("--config", default="configs/deep_cfr/default.yaml") + parser.add_argument("--config", default="legacy/deep-cfr/configs/default.yaml") parser.add_argument("--device", default="cpu") parser.add_argument("--traversals", type=int, default=32) parser.add_argument("--runs", type=int, default=5) diff --git a/legacy/deep-cfr/README.md b/legacy/deep-cfr/README.md new file mode 100644 index 0000000..7dd726b --- /dev/null +++ b/legacy/deep-cfr/README.md @@ -0,0 +1,4 @@ +# Deep CFR archive + +These YAML files are retained only to reproduce historical Deep CFR work. The +supported training stack is JAX PPO; start with `configs/jax_ppo/` instead. diff --git a/configs/deep_cfr/default.yaml b/legacy/deep-cfr/configs/default.yaml similarity index 100% rename from configs/deep_cfr/default.yaml rename to legacy/deep-cfr/configs/default.yaml diff --git a/configs/deep_cfr/default_server.yaml b/legacy/deep-cfr/configs/default_server.yaml similarity index 100% rename from configs/deep_cfr/default_server.yaml rename to legacy/deep-cfr/configs/default_server.yaml diff --git a/configs/deep_cfr/smoke.yaml b/legacy/deep-cfr/configs/smoke.yaml similarity index 100% rename from configs/deep_cfr/smoke.yaml rename to legacy/deep-cfr/configs/smoke.yaml diff --git a/legacy/ismcts/README.md b/legacy/ismcts/README.md new file mode 100644 index 0000000..4590727 --- /dev/null +++ b/legacy/ismcts/README.md @@ -0,0 +1,4 @@ +# ISMCTS archive + +These YAML files are retained only to reproduce historical ISMCTS work. The +supported training stack is JAX PPO; start with `configs/jax_ppo/` instead. diff --git a/scripts/autonomous_cycle_eval.sh b/legacy/ismcts/autonomous_cycle_eval.sh similarity index 84% rename from scripts/autonomous_cycle_eval.sh rename to legacy/ismcts/autonomous_cycle_eval.sh index 31c2e9c..d8f8870 100755 --- a/scripts/autonomous_cycle_eval.sh +++ b/legacy/ismcts/autonomous_cycle_eval.sh @@ -27,4 +27,5 @@ if [ ! -f "$CKPT" ]; then fi echo "=== eval $CKPT ===" -uv run lost-cities-ismcts eval --ckpt "$CKPT" --games 30 --verbose "$@" 2>&1 +uv run python -m coolrl_lost_cities.games.classic.ismcts.cli \ + eval --ckpt "$CKPT" --games 30 --verbose "$@" 2>&1 diff --git a/configs/ismcts/default.yaml b/legacy/ismcts/configs/default.yaml similarity index 100% rename from configs/ismcts/default.yaml rename to legacy/ismcts/configs/default.yaml diff --git a/configs/ismcts/mini.yaml b/legacy/ismcts/configs/mini.yaml similarity index 100% rename from configs/ismcts/mini.yaml rename to legacy/ismcts/configs/mini.yaml diff --git a/pyproject.toml b/pyproject.toml index 83639e4..fc07f56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "coolrl-lost-cities" version = "0.1.0" -description = "Focused Lost Cities classic game extraction" +description = "JAX PPO training and browser play for Lost Cities" readme = "README.md" authors = [ { name = "정시원", email = "sebastianrcnt@gmail.com" } @@ -34,8 +34,6 @@ lost-cities-classic = "coolrl_lost_cities.games.classic:main" lost-cities-eval = "coolrl_lost_cities.games.classic.evaluation:main" lost-cities-classic-gui = "coolrl_lost_cities.games.classic.pygame_pvp:main" lost-cities-play = "coolrl_lost_cities.games.classic.pygame_table:main" -lost-cities-deep-cfr = "coolrl_lost_cities.games.classic.deep_cfr.cli:main" -lost-cities-ismcts = "coolrl_lost_cities.games.classic.ismcts.cli:main" lost-cities-jax-ppo = "lost_cities_jax.ppo_cli:main" [dependency-groups] diff --git a/scripts/bench_inference_backend.py b/scripts/bench_inference_backend.py index ef7efd3..d934088 100644 --- a/scripts/bench_inference_backend.py +++ b/scripts/bench_inference_backend.py @@ -37,12 +37,12 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Benchmark Deep CFR traversal inference backends.") parser.add_argument( "--config-local", - default="configs/deep_cfr/default.yaml", + default="legacy/deep-cfr/configs/default.yaml", help="Config for the local traversal inference backend.", ) parser.add_argument( "--config-server", - default="configs/deep_cfr/default_server.yaml", + default="legacy/deep-cfr/configs/default_server.yaml", help="Config for the server traversal inference backend.", ) parser.add_argument("--iterations", type=int, default=10) diff --git a/scripts/export_jax_ppo_onnx.py b/scripts/export_jax_ppo_onnx.py index bd8ac44..d5f249e 100644 --- a/scripts/export_jax_ppo_onnx.py +++ b/scripts/export_jax_ppo_onnx.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import json from pathlib import Path @@ -86,10 +87,14 @@ def export_model(checkpoint: Path, config: Path | None, output: Path) -> None: ) output.parent.mkdir(parents=True, exist_ok=True) onnx.save(model, output) + model_bytes = output.read_bytes() manifest = { "format": "coolrl-lost-cities-jax-ppo-onnx-v1", - "source_checkpoint": str(checkpoint.resolve()), - "source_config": str(config_path.resolve()), + "model_file": output.name, + "model_size_bytes": len(model_bytes), + "model_sha256": hashlib.sha256(model_bytes).hexdigest(), + "source_checkpoint": checkpoint.name, + "source_config": config_path.name, "observation_size": OBS_DIM, "action_size": N_ACTIONS, "hidden_size": cfg.network.hidden_size, diff --git a/scripts/librarian-ignore.txt b/scripts/librarian-ignore.txt index 28c464c..7117e33 100644 --- a/scripts/librarian-ignore.txt +++ b/scripts/librarian-ignore.txt @@ -7,14 +7,7 @@ # # Format examples: # scripts/foo.sh # exact path -# configs/deep_cfr/model-*.yaml # glob +# configs/jax_ppo/model-*.yaml # glob # docs/research/*.md # whole directory # # Comments after `#` are stripped per line. Blank lines ignored. - -# Future config + script described in docs/plans/model_size_experiment.md -configs/deep_cfr/model-size-*.yaml -scripts/run_model_size_experiment.sh - -# Future config described in docs/plans/torch_compile.md -configs/deep_cfr/default_compile.yaml diff --git a/scripts/profile_gpu_forward.py b/scripts/profile_gpu_forward.py index 83ecd9b..adc29ef 100644 --- a/scripts/profile_gpu_forward.py +++ b/scripts/profile_gpu_forward.py @@ -1,7 +1,7 @@ """Profile GPU forward-pass throughput for the Deep CFR trainer network. Builds the same DeepCFRMLP that ``DeepCFRTrainer.__init__`` constructs from -``configs/deep_cfr/default.yaml``, then measures average forward-pass time on +``legacy/deep-cfr/configs/default.yaml``, then measures average forward-pass time on CUDA across a sweep of batch sizes. The goal is to decide whether batched traversal inference (Optimization Priorities #5) is worth implementing. """ @@ -19,7 +19,7 @@ from coolrl_lost_cities.games.classic.deep_cfr.config import load_config from coolrl_lost_cities.games.classic.deep_cfr.networks import DeepCFRMLP REPO_ROOT = Path(__file__).resolve().parent.parent -CONFIG_PATH = REPO_ROOT / "configs" / "deep_cfr" / "default.yaml" +CONFIG_PATH = REPO_ROOT / "legacy" / "deep-cfr" / "configs" / "default.yaml" BATCH_SIZES = [1, 4, 16, 64, 256, 1024] WARMUP_ITERS = 10 diff --git a/src/coolrl_lost_cities/games/classic/ismcts/eval_checkpoint.py b/src/coolrl_lost_cities/games/classic/ismcts/eval_checkpoint.py index feea2ab..da372e0 100644 --- a/src/coolrl_lost_cities/games/classic/ismcts/eval_checkpoint.py +++ b/src/coolrl_lost_cities/games/classic/ismcts/eval_checkpoint.py @@ -6,7 +6,7 @@ against opponents that are not in `evaluation.opponents` (e.g. heuristic-balance the rollout policy) and for running many more games than per-iter eval typically allows. -Invoked via ``lost-cities-ismcts eval`` (see ``cli.py``). +Available through the archived ISMCTS CLI module (see ``cli.py``). """ from __future__ import annotations diff --git a/src/coolrl_lost_cities/games/classic/ismcts/pretrain.py b/src/coolrl_lost_cities/games/classic/ismcts/pretrain.py index 2fccedf..fd82eef 100644 --- a/src/coolrl_lost_cities/games/classic/ismcts/pretrain.py +++ b/src/coolrl_lost_cities/games/classic/ismcts/pretrain.py @@ -8,14 +8,12 @@ AlphaZero-style network's policy + value heads in supervised fashion: - value loss: MSE on the final game score diff from each decision-maker's perspective, normalized by value_scale (same convention as the trainer). -The resulting checkpoint can be passed to `lost-cities-ismcts train ---resume-from` to start self-play with a heuristic-level prior instead of a -random init. This addresses the self-play "weak-equilibrium" problem: starting -from random, MCTS visit distributions converge to a mutually mediocre policy -that has near-zero win rate against the heuristic. Warm-starting at heuristic -level gives self-play a meaningful baseline to improve from. - -Invoked via ``lost-cities-ismcts pretrain``. +The resulting checkpoint can be passed to the archived ISMCTS trainer to start +self-play with a heuristic-level prior instead of a random init. This addresses +the self-play "weak-equilibrium" problem: starting from random, MCTS visit +distributions converge to a mutually mediocre policy that has near-zero win +rate against the heuristic. Warm-starting at heuristic level gives self-play a +meaningful baseline to improve from. """ from __future__ import annotations diff --git a/tests/games/classic/test_deep_cfr_trainer.py b/tests/games/classic/test_deep_cfr_trainer.py index e7450d1..82b77ac 100644 --- a/tests/games/classic/test_deep_cfr_trainer.py +++ b/tests/games/classic/test_deep_cfr_trainer.py @@ -44,7 +44,7 @@ def _deep_cfr_config(data: dict) -> DeepCFRConfig: def test_deep_cfr_loads_smoke_yaml_config() -> None: - config = load_config("configs/deep_cfr/smoke.yaml") + config = load_config("legacy/deep-cfr/configs/smoke.yaml") assert config.run.max_iterations == 1 assert config.network.hidden_size == 16 diff --git a/web/README.md b/web/README.md index 2ba598f..8751838 100644 --- a/web/README.md +++ b/web/README.md @@ -1,11 +1,38 @@ # COOLRL Lost Cities Web Browser-only Lost Cities client. The rules engine, observation builder, and PPO -inference all run on the device. There is no application server. +inference all run on the device; there is no application server. + +The verified final JAX PPO policy is committed at +`public/models/jax-ppo.onnx` (3.1 MB). Vite copies it to the static build, and +the app resolves the asset relative to the deployed site so it works on GitHub +Pages, GitLab Pages, or a normal web root. Its size and SHA-256 are recorded in +[`public/models/jax-ppo.json`](public/models/jax-ppo.json). + +Pushes to `main` build and publish the client through the repository's GitHub +Pages and GitLab Pages workflows. Each workflow supplies the correct base URL +for its host. ## Setup -From the repository root, export an Orbax checkpoint to the browser model: +Install and run the checked-in final policy: + +```bash +cd web +npm ci +npm run dev +``` + +Build the same static bundle used by a host: + +```bash +npm run build +npm run preview +``` + +To replace the shipped policy, export a verified Orbax checkpoint from the +repository root. The exporter writes both the ONNX file and its public +metadata manifest: ```bash uv run --with onnx scripts/export_jax_ppo_onnx.py \ @@ -13,14 +40,6 @@ uv run --with onnx scripts/export_jax_ppo_onnx.py \ --output web/public/models/jax-ppo.onnx ``` -Then install and run the web app: - -```bash -cd web -npm install -npm run dev -``` - The policy tries WebGPU first and falls back to ONNX Runtime WebAssembly. If the model asset is absent, the UI remains playable using a simple local heuristic and reports that fallback in the header. diff --git a/web/public/models/jax-ppo.json b/web/public/models/jax-ppo.json new file mode 100644 index 0000000..9ccec5b --- /dev/null +++ b/web/public/models/jax-ppo.json @@ -0,0 +1,14 @@ +{ + "format": "coolrl-lost-cities-jax-ppo-onnx-v1", + "model_file": "jax-ppo.onnx", + "model_size_bytes": 3230780, + "model_sha256": "e8241e305c01ea450e92a6178002a22db96710fa94e238bba57953743eb285b2", + "source_checkpoint": "final_candidate", + "source_config": "main_ppo_config.json", + "observation_size": 454, + "action_size": 96, + "hidden_size": 512, + "num_layers": 3, + "dtype": "float32", + "validation_max_abs_error": 9.918212890625e-05 +} diff --git a/web/public/models/jax-ppo.onnx b/web/public/models/jax-ppo.onnx new file mode 100644 index 0000000..630b5a1 Binary files /dev/null and b/web/public/models/jax-ppo.onnx differ diff --git a/web/src/model/policy.ts b/web/src/model/policy.ts index 85bbfdc..291d3c2 100644 --- a/web/src/model/policy.ts +++ b/web/src/model/policy.ts @@ -7,6 +7,8 @@ import { cardColor, cardRank, isHandshake } from "../game/cards"; export type ExecutionProvider = "webgpu" | "wasm" | "heuristic"; +const MODEL_URL = `${import.meta.env.BASE_URL}models/jax-ppo.onnx`; + export interface Policy { readonly provider: ExecutionProvider; action(state: GameState): Promise; @@ -66,7 +68,7 @@ class HeuristicPolicy implements Policy { } async function createSession(provider: "webgpu" | "wasm"): Promise { - return ort.InferenceSession.create("/models/jax-ppo.onnx", { + return ort.InferenceSession.create(MODEL_URL, { executionProviders: [provider], graphOptimizationLevel: "all", });