From e51d14d0fb6c1be76402460564dfabf2b854f7b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=8B=9C=EC=9B=90?= Date: Wed, 15 Jul 2026 05:37:55 +0900 Subject: [PATCH] Measure exploitability: ours is harder to farm than league Winning the head-to-head says a policy is strong on average, not that it is hard to beat. So freeze each policy, train a fresh one from scratch whose only job is to beat that policy, and see how far it gets. Same exploiter budget (250 updates x batch 1024, 65.5M learner actions): ours (match stack, 131M) exploiter reaches 0.2278 [0.219, 0.237] league (web-deployed) exploiter reaches 0.3213 [0.311, 0.331] League gives up 9.4 more points to a dedicated attacker. Both sit far from 0.5, so neither is a pushover -- but the caveat left open by the head-to-head is now closed in our favour on both axes: stronger on average (0.6094) and harder to exploit. Worth noting against expectation: league was trained *with* an exploiter structure and we ran pure self-play, and we still come out less exploitable. Whatever the league machinery buys, it did not buy that here. The number is a lower bound -- a bigger exploiter would find more -- so it only means anything as a like-for-like comparison, which is how it is used. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XBQKgvBbxbheiTF1AVy1Sh --- docs/plans/lost-cities-classic-3round.md | 39 ++++ scripts/measure_exploitability.py | 117 +++++++++++ src/lost_cities_jax/exploit.py | 244 +++++++++++++++++++++++ 3 files changed, 400 insertions(+) create mode 100644 scripts/measure_exploitability.py create mode 100644 src/lost_cities_jax/exploit.py diff --git a/docs/plans/lost-cities-classic-3round.md b/docs/plans/lost-cities-classic-3round.md index ea66508..c7b5e70 100644 --- a/docs/plans/lost-cities-classic-3round.md +++ b/docs/plans/lost-cities-classic-3round.md @@ -539,3 +539,42 @@ Fable은 이것을 "가장 큰 누락 아이디어"로 꼽았다. 실측은 ** 엉터리 채점을 한다. **Fable은 critic을 "가장 큰 누락 아이디어" 1순위로, 양쪽 좌석을 4순위로 꼽았다. 정확히 거꾸로였다.** + +--- + +## Exploitability: 우리가 league보다 **덜 착취당한다** (2026-07-15) + +**남겨뒀던 단서를 해소했다.** 평균 실력에서 이기는 것과 "파먹을 약점이 없는 것"은 다른 +속성이다. 가위바위보에서 바위를 60% 내는 놈은 아무한테나 이길 수도 있지만, 그 습관을 +알아챈 놈에게는 매번 진다. + +**측정 방법:** 정책을 얼려놓고, **오직 그놈만 이기도록 특화된 새 정책을 처음부터 학습**시킨다 +(`src/lost_cities_jax/exploit.py`). 착취자가 도달한 승률이 곧 그 정책이 못 막아낸 습관의 크기다. + +**동일 착취자 예산 (250 업데이트 × batch 1024 = 65.5M learner 액션):** + +| 얼려놓은 정책 | 착취자 승률 | 95% CI | +|---|---|---| +| **우리 매치 스택 (131M)** | **0.2278** | [0.219, 0.237] | +| league (웹 배포판, 122.6M) | **0.3213** | [0.311, 0.331] | + +**league가 9.4%p 더 털린다.** 그리고 둘 다 0.5에서 한참 멀다 — 어느 쪽도 만만한 상대는 아니다. + +### 최종 정리: 두 축 모두에서 이긴다 + +| 축 | 결과 | +|---|---| +| **평균 실력** (정면 대결) | 0.6094, +22.0점 ✅ | +| **약점 적음** (exploitability) | 0.2278 vs 0.3213 ✅ | + +### 단서 + +1. **하한선이다.** 더 세거나 더 오래 학습한 착취자는 더 찾아낼 수 있다. **절대값이 아니라 + 동일 예산에서의 비교로만 의미가 있다.** +2. 착취자 예산이 목표의 절반이다(65M vs 131M). 키우면 두 수치 다 오른다. +3. league는 단판 정책이라 3라운드 게임에선 다소 제 물이 아니다 — 다만 carry가 무의미하다는 + 것이 이미 증명됐으므로 큰 불리함은 아니다. + +**흥미로운 점:** 우리는 **순수 셀프플레이**이고 league는 **착취자 구조를 학습에 넣은** 런인데, +그런데도 우리가 덜 착취당한다. 착취자 구조가 exploitability를 낮춰줄 것이라는 기대가 +이 게임에서는 확인되지 않았다. diff --git a/scripts/measure_exploitability.py b/scripts/measure_exploitability.py new file mode 100644 index 0000000..ca20026 --- /dev/null +++ b/scripts/measure_exploitability.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Freeze each policy, train an exploiter against it on the same budget, compare. + +Winning the head-to-head says a policy is strong on average. It does not say the +policy is hard to beat. This does: whatever the exploiter reaches is a habit the +frozen policy could not defend. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import jax +import numpy as np + +from lost_cities_jax.exploit import ( + match_frozen_policy, + single_round_frozen_policy, + train_exploiter, +) +from lost_cities_jax.match import match_reset_from, match_score +from lost_cities_jax.match_eval import MATCH_SCAN_STEPS, _wilson, match_bank +from lost_cities_jax.match_ppo import Ablation, create_match_train_state +from lost_cities_jax.ppo import create_train_state, load_config, restore_checkpoint + +OURS = Path("runs/jax-ppo-match/2026-07-15_031529_match-scaled/latest") +LEAGUE = Path( + "/mnt/2tbhdd/coolrl-lost-cities-artifacts/league/2026-07-05_052325_jax-ppo-league-v1/latest" +) +MATCHES = 4096 + + +def _final_score(cfg, exploiter_params, frozen, matches: int) -> dict: + """Play the trained exploiter against the frozen policy, both seats.""" + import jax.numpy as jnp + + from lost_cities_jax.exploit import match_frozen_policy as _mk + from lost_cities_jax.match import match_step + + attacker = _mk(cfg, exploiter_params, Ablation()) + decks, coins = match_bank(20260722, matches) + + @jax.jit + def run(env, a_seat): + def body(carry, _): + env, _u = carry + action = jnp.where( + env.round.to_move.astype(jnp.int32) == a_seat, + attacker(env, a_seat), + frozen(env, 1 - a_seat), + ) + env, _, _ = jax.vmap(match_step, in_axes=(0, 0))(env, action) + return (env, _u), None + + (env, _), _ = jax.lax.scan(body, (env, jnp.int32(0)), xs=None, length=MATCH_SCAN_STEPS) + return env + + leads = [] + for seat in (0, 1): + env = jax.vmap(match_reset_from)(decks, coins) + final = run(env, jnp.full((matches,), seat, dtype=jnp.int32)) + totals = np.asarray(jax.vmap(match_score)(final)) + leads.append(totals[:, seat] - totals[:, 1 - seat]) + + lead = np.concatenate(leads) + games = float(lead.size) + wins = float((lead > 0).sum()) + low, high = _wilson(wins, games) + return { + "exploiter_win_rate": wins / games, + "wilson_low": low, + "wilson_high": high, + "exploiter_mean_lead": float(lead.mean()), + "matches": games, + } + + +def main() -> None: + match_cfg = load_config("configs/jax_ppo/match-selfplay.yaml") + old_cfg = load_config("configs/jax_ppo/balanced.yaml") + + ours = restore_checkpoint( + OURS, create_match_train_state(match_cfg, jax.random.PRNGKey(0), Ablation()) + ).params + league = restore_checkpoint(LEAGUE, create_train_state(old_cfg, jax.random.PRNGKey(0))).params + + targets = { + "ours (match stack, 131M)": match_frozen_policy(match_cfg, ours, Ablation()), + "league (web-deployed)": single_round_frozen_policy(old_cfg, league), + } + + rows = [] + for name, frozen in targets.items(): + print(f"\n===== training an exploiter against: {name} =====", flush=True) + cfg = load_config("configs/jax_ppo/match-selfplay.yaml") + cfg.ppo.batch_games = 1024 + cfg.run.total_updates = 250 + cfg.run.log_every = 25 + slug = name.split()[0] + state = train_exploiter(cfg, frozen, Path(f"runs/jax-ppo-match/exploit-{slug}")) + result = _final_score(cfg, state.params, frozen, MATCHES) + rows.append({"target": name, **result}) + print(f" exploiter reached {result['exploiter_win_rate']:.4f} vs {name}", flush=True) + + print("\n\n============ exploitability (same exploiter budget) ============") + print(f"{'frozen policy':<28}{'exploiter win rate':>20}{'95% CI':>22}") + print("-" * 72) + for row in rows: + ci = f"[{row['wilson_low']:.3f}, {row['wilson_high']:.3f}]" + print(f"{row['target']:<28}{row['exploiter_win_rate']:>20.4f}{ci:>22}") + print("\nhigher = the frozen policy had more to farm. 0.5 = nothing found.") + Path("runs/jax-ppo-match/exploitability.json").write_text(json.dumps(rows, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/lost_cities_jax/exploit.py b/src/lost_cities_jax/exploit.py new file mode 100644 index 0000000..db43c0b --- /dev/null +++ b/src/lost_cities_jax/exploit.py @@ -0,0 +1,244 @@ +"""How exploitable is a frozen policy? + +Beating an opponent on average and being hard to beat are different properties. +A policy can win the head-to-head and still have a habit a dedicated opponent can +farm -- rock-paper-scissors throwing rock 60% of the time beats a lot of people +and loses every time to anyone who noticed. + +So: freeze the policy, train a fresh one from scratch whose only job is to beat +*that* policy, and see how far it gets. Near 0.5 means there was nothing to find. +Well above it means there was. + +PPO self-play carries no Nash guarantee, so this is not a formality. The number it +produces is a lower bound on exploitability -- a stronger exploiter might find +more -- so it is only meaningful compared against the same exploiter budget spent +on another policy. +""" + +from __future__ import annotations + +import json +import time +from collections.abc import Callable +from pathlib import Path + +import jax +import jax.numpy as jnp + +from lost_cities_jax.match import MatchState, match_legal_action_mask, match_reset, match_step +from lost_cities_jax.match_ppo import ( + FULL, + N_SEATS, + Ablation, + MatchActorCritic, + MatchTransition, + _seat_views, + create_match_train_state, + match_lead, + match_ppo_update, +) +from lost_cities_jax.obs import observation +from lost_cities_jax.ppo import ( + ActorCritic, + JaxPPOConfig, + TrainState, + _append_jsonl, + _broadcast_done, + action_log_prob, + categorical_entropy, + compute_gae, + mask_logits, + masked_mean, +) +from lost_cities_jax.types import PLAY + +FrozenPolicy = Callable[[MatchState, jax.Array], jax.Array] +"""(env batch, seat batch) -> greedy actions. Whatever architecture, hidden here.""" + + +def match_frozen_policy(cfg: JaxPPOConfig, params, ablation: Ablation = FULL) -> FrozenPolicy: + model = MatchActorCritic(cfg.network.hidden_size, cfg.network.num_layers) + seats = jnp.arange(N_SEATS, dtype=jnp.int32) + + def policy(env: MatchState, seat: jax.Array) -> jax.Array: + obs, critic = _seat_views(env, seats, ablation) + idx = seat[None, :, None] + logits, _ = model.apply( + params, + jnp.take_along_axis(obs, idx, axis=0)[0], + jnp.take_along_axis(critic, idx, axis=0)[0], + ) + mask = jax.vmap(match_legal_action_mask)(env) + return jnp.argmax(mask_logits(logits, mask), axis=-1).astype(jnp.int32) + + return policy + + +def single_round_frozen_policy(cfg: JaxPPOConfig, params) -> FrozenPolicy: + """The old stack: it sees one round at a time and has never heard of carry.""" + model = ActorCritic(cfg.network.hidden_size, cfg.network.num_layers) + + def policy(env: MatchState, seat: jax.Array) -> jax.Array: + logits, _ = model.apply(params, jax.vmap(observation)(env.round, seat)) + mask = jax.vmap(match_legal_action_mask)(env) + return jnp.argmax(mask_logits(logits, mask), axis=-1).astype(jnp.int32) + + return policy + + +def _reset_done(env: MatchState, seat: jax.Array, rng: jax.Array, batch: int): + env_key, seat_key = jax.random.split(rng) + fresh = jax.vmap(match_reset)(jax.random.split(env_key, batch)) + next_env = jax.tree_util.tree_map( + lambda old, new: jnp.where(_broadcast_done(env.done, old), new, old), env, fresh + ) + # Redraw the exploiter's seat too, so it learns to beat the frozen policy from + # either side rather than only the one it started in. + fresh_seat = jax.random.bernoulli(seat_key, 0.5, shape=(batch,)).astype(jnp.int32) + return next_env, jnp.where(env.done, fresh_seat, seat) + + +def make_exploiter_rollout_fn(cfg: JaxPPOConfig, frozen: FrozenPolicy): + batch = cfg.ppo.batch_games + seats = jnp.arange(N_SEATS, dtype=jnp.int32) + ablation = FULL + + @jax.jit + def rollout_fn(state: TrainState, env: MatchState, seat: jax.Array, rng: jax.Array): + def body(carry, _): + env, seat, key = carry + key, act_key, reset_key = jax.random.split(key, 3) + + obs, critic_obs = _seat_views(env, seats, ablation) + idx = seat[None, :, None] + my_obs = jnp.take_along_axis(obs, idx, axis=0)[0] + my_critic = jnp.take_along_axis(critic_obs, idx, axis=0)[0] + legal = jax.vmap(match_legal_action_mask)(env) + + logits, value = state.apply_fn(state.params, my_obs, my_critic) + masked = mask_logits(logits, legal) + mine = jax.random.categorical(act_key, masked, axis=-1).astype(jnp.int32) + theirs = jax.lax.stop_gradient(frozen(env, 1 - seat)) + + to_move = env.round.to_move.astype(jnp.int32) + active = ~env.done + my_turn = to_move == seat + actions = jnp.where(my_turn, mine, theirs) + + before = jax.vmap(match_lead)(env) + next_env, _, _ = jax.vmap(match_step, in_axes=(0, 0))(env, actions) + after = jax.vmap(match_lead)(next_env) + + # match_lead is seat 0's view; flip it when the exploiter sits in 1. + sign = jnp.where(seat == 0, 1.0, -1.0) + reward = jnp.where(active, sign * (after - before) / cfg.reward.terminal_scale, 0.0) + + actor_mask = active & my_turn + transition = MatchTransition( + obs=my_obs, + critic_obs=my_critic, + legal_mask=legal, + action=actions, + log_prob=action_log_prob(masked, mine), + value=value, + reward=reward, + done=next_env.done, + active=active, + actor_mask=actor_mask, + entropy=categorical_entropy(masked), + play_action=((actions % 12) // 6 == PLAY) & actor_mask, + ) + + done = next_env.done + final_lead = sign * jax.vmap(match_lead)(next_env) + won = (done & active & (final_lead > 0)).astype(jnp.float32) + finished = (done & active).astype(jnp.float32) + + next_env, seat = _reset_done(next_env, seat, reset_key, batch) + return (next_env, seat, key), (transition, won, finished) + + (next_env, seat, rng), (transitions, won, finished) = jax.lax.scan( + body, (env, seat, rng), xs=None, length=cfg.ppo.rollout_steps + ) + + obs, critic_obs = _seat_views(next_env, seats, ablation) + idx = seat[None, :, None] + _, last_value = state.apply_fn( + state.params, + jnp.take_along_axis(obs, idx, axis=0)[0], + jnp.take_along_axis(critic_obs, idx, axis=0)[0], + ) + + played = jnp.sum(finished) + metrics = { + "exploiter_win_rate": jnp.sum(won) / jnp.maximum(played, 1.0), + "matches_completed": played, + "learner_actions": jnp.sum(transitions.actor_mask), + "entropy_mean": masked_mean( + transitions.entropy, transitions.actor_mask.astype(jnp.float32) + ), + } + return next_env, seat, transitions, last_value, metrics + + return rollout_fn + + +def train_exploiter(cfg: JaxPPOConfig, frozen: FrozenPolicy, run_dir: Path) -> TrainState: + """Train a fresh policy whose only goal is to beat ``frozen``.""" + + rng = jax.random.PRNGKey(cfg.run.seed) + rng, init_key, reset_key, seat_key = jax.random.split(rng, 4) + state = create_match_train_state(cfg, init_key, Ablation()) + env = jax.jit(jax.vmap(match_reset))(jax.random.split(reset_key, cfg.ppo.batch_games)) + seat = jax.random.bernoulli(seat_key, 0.5, shape=(cfg.ppo.batch_games,)).astype(jnp.int32) + + rollout_fn = make_exploiter_rollout_fn(cfg, frozen) + + @jax.jit + def iteration(state, env, seat, rng): + rng, roll_key, update_key = jax.random.split(rng, 3) + env, seat, transitions, last_value, metrics = rollout_fn(state, env, seat, roll_key) + advantages, returns = compute_gae( + transitions.reward, + transitions.value, + transitions.done, + cfg.ppo.gamma, + cfg.ppo.gae_lambda, + last_value, + ) + state, update_metrics = match_ppo_update( + state, transitions, advantages, returns, update_key, cfg + ) + return state, env, seat, rng, {**metrics, **update_metrics} + + run_dir.mkdir(parents=True, exist_ok=True) + metrics_path = run_dir / "metrics.jsonl" + start = time.perf_counter() + for update in range(cfg.run.total_updates): + state, env, seat, rng, metrics = iteration(state, env, seat, rng) + jax.tree_util.tree_leaves(metrics)[0].block_until_ready() + row = {k: float(v) for k, v in metrics.items()} + row["update"] = update + row["elapsed_seconds"] = time.perf_counter() - start + _append_jsonl(metrics_path, row) + if update % cfg.run.log_every == 0: + print( + json.dumps( + { + "update": update, + "exploiter_win_rate": row["exploiter_win_rate"], + "matches": row["matches_completed"], + }, + sort_keys=True, + ), + flush=True, + ) + return state + + +__all__ = [ + "make_exploiter_rollout_fn", + "match_frozen_policy", + "single_round_frozen_policy", + "train_exploiter", +]