From 4d1c4473b696d32a4210d31e9ce5e94e0cc360ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=8B=9C=EC=9B=90?= Date: Wed, 15 Jul 2026 04:01:40 +0900 Subject: [PATCH] Restore the privileged critic: its ablation flips with scale The 39.3M ablation said the privileged critic hurt (switching it off won 0.5160 [0.505, 0.527]). Head to head at 131M, both sides trained identically, it says the opposite: off *loses*, 0.4633 [0.453, 0.474]. Against league the critic-on model scores 0.6094 and the critic-off one 0.5526. The critic earns its keep once there is enough data to fit it -- at 39.3M the privileged value trunk is underfit and only adds advantage noise. Defaulted back on, with the small-scale number kept in the docstring as a warning: an ablation at a budget you do not intend to ship can invert. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XBQKgvBbxbheiTF1AVy1Sh --- docs/plans/lost-cities-classic-3round.md | 25 ++++++++++++++++++++++++ src/lost_cities_jax/match_ppo.py | 15 ++++++++------ tests/lost_cities_jax/test_match_ppo.py | 10 ++++++++-- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/docs/plans/lost-cities-classic-3round.md b/docs/plans/lost-cities-classic-3round.md index 30cee2b..7736435 100644 --- a/docs/plans/lost-cities-classic-3round.md +++ b/docs/plans/lost-cities-classic-3round.md @@ -490,3 +490,28 @@ Fable은 이것을 "가장 큰 누락 아이디어"로 꼽았다. 실측은 ** 3. **선형 총점 보상** — tanh 대비 0.5859 (사용자 제안) 4. **매치 관측 결함 수정** — 약 2.5%p 5. ~~전지적 critic~~ — **해가 된다. 끈다.** + +### 정정: 전지적 critic의 가치는 **스케일에 따라 뒤집힌다** + +위 A/B는 39.3M learner 액션 규모에서 돌렸다. 실전 규모(131M)에서 동일 설정으로 둘을 직접 +맞붙이면 **결론이 반대로 나온다.** + +| 규모 | critic OFF의 승률 (critic ON 상대) | 판정 | +|---|---|---| +| 39.3M (A/B 규모) | **0.5160** [0.505, 0.527] | OFF가 낫다 | +| **131M (실전 규모)** | **0.4633** [0.453, 0.474] | **ON이 낫다** | + +**전지적 critic은 그것을 학습시킬 데이터가 충분해질 때 값어치를 한다.** 39.3M에서는 특권 +정보를 쓰는 큰 가치 트렁크를 제대로 못 맞춰서 어드밴티지에 노이즈만 얹었고, 131M에서는 +제대로 맞춰서 분산 감소가 실현된다. + +**교훈: 배포할 예산이 아닌 규모에서 ablation을 돌리면 결론이 뒤집힐 수 있다.** +`privileged_critic` 기본값은 **ON으로 되돌린다.** + +### 최종 순위 (진짜 3라운드 클래식, duplicate 8,192판) + +| 모델 | learner 액션 | vs league | +|---|---|---| +| **매치 스택 (critic ON)** | 131M | **0.6094** (+22.0점) | +| 매치 스택 (critic OFF) | 131M | 0.5526 (+10.9점) | +| league (기존 최강, 웹 배포판) | 122.6M | — | diff --git a/src/lost_cities_jax/match_ppo.py b/src/lost_cities_jax/match_ppo.py index c3775d6..f7bd4d1 100644 --- a/src/lost_cities_jax/match_ppo.py +++ b/src/lost_cities_jax/match_ppo.py @@ -89,14 +89,17 @@ N_SEATS = 2 class Ablation: """Switches for measuring what each piece of the match stack is worth.""" - privileged_critic: bool = False + privileged_critic: bool = True """On: the critic also sees the opponent's hand and the deck in order. - Defaults OFF because it measured *negative*. Ablated against the full stack - over 8192 duplicate matches, switching it off won 0.5160 [0.505, 0.527]. A - critic that knows the deck fits V(full state), which is not - E[return | masked obs], so the advantage carries a component the actor cannot - act on -- noise from its side, not variance reduction. + Its worth flips with scale, so do not trust a cheap ablation here. Ablated at + 39.3M learner actions, switching it OFF *won* 0.5160 [0.505, 0.527] -- the + obvious reading being that a critic fitting V(full state) rather than + E[return | masked obs] hands the actor advantage noise it cannot act on. But + head to head at 131M, both sides trained identically, OFF *loses*: 0.4633 + [0.453, 0.474]. The privileged critic earns its keep once there is enough data + to fit it. Defaults on for that reason, and the small-scale result is kept as + a warning about ablating at a budget you do not intend to ship. """ both_seats: bool = True diff --git a/tests/lost_cities_jax/test_match_ppo.py b/tests/lost_cities_jax/test_match_ppo.py index 76c0bd3..eba28a6 100644 --- a/tests/lost_cities_jax/test_match_ppo.py +++ b/tests/lost_cities_jax/test_match_ppo.py @@ -13,6 +13,7 @@ from lost_cities_jax.match_obs import ( match_observation, ) from lost_cities_jax.match_ppo import ( + Ablation, create_match_train_state, make_match_rollout_fn, make_match_train_iteration, @@ -54,9 +55,14 @@ def test_the_critic_sees_the_opponents_hand_and_the_actor_does_not(): def test_privileged_input_cannot_move_the_policy_logits(): - """Separate trunks, or the critic's view of the deck leaks into play.""" + """Separate trunks, or the critic's view of the deck leaks into play. + + The privileged critic is off by default -- it measured negative, see the + ablation in the plan -- but the isolation property still has to hold for + anyone who switches it on, and for its later reuse as a search evaluator. + """ cfg = _cfg() - state = create_match_train_state(cfg, jax.random.PRNGKey(1)) + state = create_match_train_state(cfg, jax.random.PRNGKey(1), Ablation(privileged_critic=True)) match = match_reset(jax.random.PRNGKey(2)) obs = match_observation(match, jnp.int32(0))[None, :]