From 100f498cf1b8d828819343a4a576d251727ef2e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EC=8B=9C=EC=9B=90?= Date: Thu, 7 May 2026 21:08:35 +0900 Subject: [PATCH] add heavy julia cfr thread scaling Co-Authored-By: Codex --- docs/research/julia_port_evaluation.md | 62 +++++++------- experiments/julia_cfr_toy/README.md | 35 +++++--- .../julia_cfr_toy/bench_cfr_threaded.jl | 80 +++++++++++++------ 3 files changed, 115 insertions(+), 62 deletions(-) diff --git a/docs/research/julia_port_evaluation.md b/docs/research/julia_port_evaluation.md index 9d003ef..05d2410 100644 --- a/docs/research/julia_port_evaluation.md +++ b/docs/research/julia_port_evaluation.md @@ -66,26 +66,33 @@ local trees, root regrets reduced at the end. | threads | iter ms | speedup | efficiency | | ---: | ---: | ---: | ---: | -| 1 | 0.219 | 1.00× | 100% | -| 2 | 0.129 | 1.69× | 85% | -| 4 | 0.096 | 2.28× | 57% | -| 8 | 0.090 | 2.44× | 31% | +| 1 | 0.223 | 1.00× | 100% | +| 2 | 0.130 | 1.71× | 85% | +| 4 | 0.097 | 2.29× | 57% | +| 8 | 0.093 | 2.40× | 30% | -**Verdict on criterion 3:** Partial / inconclusive. 8T efficiency is -31% — contention or dispatch overhead dominates at this toy's small -per-thread workload. Julia does improve absolute throughput (2.44× -wall-clock at 8T vs 1T), but not near-linearly. The next scaling test -must increase per-thread work (larger tree or more traversals per -chunk) before a hard conclusion can be drawn. Do not treat this as a -hard ceiling — the toy may simply be too small for 8 threads to amortize -dispatch cost. +The light workload is too small to settle the question: 1T iter time is +only ~0.2 ms, so thread dispatch overhead can dominate. -## Open evidence (criteria 3, 4, 5) +Heavy mode keeps the same tree and algorithm but increases traversals +per iteration from 1000 to 50000 (50× work). This raises 1T iter time +to 8.272 ms. + +| threads | iter ms | speedup | efficiency | +| ---: | ---: | ---: | ---: | +| 1 | 8.272 | 1.00× | 100% | +| 2 | 5.058 | 1.64× | 82% | +| 4 | 2.602 | 3.18× | 79% | +| 8 | 1.739 | 4.76× | 59% | + +**Verdict on criterion 3:** PARTIAL. Heavy 8T efficiency is 59%. +Dispatch overhead was a significant part of the light-mode result, but +the heavier workload still does not reach near-linear 8-thread scaling. +Julia delivers useful throughput scaling (4.76× at 8T), but this is not +the decisive PASS threshold for the threading criterion. + +## Open evidence (criteria 4, 5) -- **Multi-thread scaling with heavier per-thread work.** Increase tree - depth or traversals-per-chunk and re-run the 1/2/4/8 thread sweep. - Current 31% at 8T is likely a toy-size artifact, not a Julia limit. - This remains the decisive test for the porting decision. - **Flux.jl + CUDA.jl MLP forward at bs={1, 64, 256}.** Compare to PyTorch numbers in `docs/performance.md`. Criterion 4. Not started. - **Real-game-state slice port.** Port `play_card` + scoring, run on a @@ -94,15 +101,16 @@ dispatch cost. ## Decision posture -Strong but not yet sufficient. The three completed benchmarks remove the -main risk (GC under recursion) and confirm compute parity. Multi-thread -scaling shows 2.44× wall-clock at 8T but only 31% efficiency — the toy -workload is likely too small per thread to amortize dispatch costs, so -the result is inconclusive rather than negative. A heavier per-thread -workload must be tested before criterion 3 can be marked pass or fail. +Promising but not enough to justify a port yet. The completed benchmarks +remove the main risk (GC under recursion) and confirm compute parity. +Heavy thread scaling upgrades criterion 3 from inconclusive to PARTIAL: +8T is 4.76× faster than 1T, but 59% efficiency is below the near-linear +PASS threshold. -After criterion 3 is settled, ML-stack and game-state evidence (criteria -4, 5) determine whether to start a serious port plan. +This means Julia remains a credible option, but not a slam dunk. ML-stack +and game-state evidence (criteria 4, 5) must be positive before starting +a serious port plan. If those are positive, criterion 3 should be revisited +on a real traversal slice where each thread has substantially more work +than this toy benchmark. -Do not commit to porting until criterion 3 is conclusively settled with -appropriate per-thread workload. +Do not commit to porting on the current evidence alone. diff --git a/experiments/julia_cfr_toy/README.md b/experiments/julia_cfr_toy/README.md index 7dfdec0..7299b78 100644 --- a/experiments/julia_cfr_toy/README.md +++ b/experiments/julia_cfr_toy/README.md @@ -19,7 +19,7 @@ uv run python experiments/julia_cfr_toy/bench_cfr_runner.py Run Julia thread-local scaling: ```bash -tools/julia/current/bin/julia --threads=8 experiments/julia_cfr_toy/bench_cfr_threaded.jl +tools/julia/current/bin/julia --threads=8 experiments/julia_cfr_toy/bench_cfr_threaded.jl --heavy ``` The Python runner builds `bench_cfr.pyx` in place when needed, runs the Julia @@ -66,14 +66,27 @@ Thread-local trees, same 100 iterations × 1000 traversals total work. Each thread processes its own chunk, then root regrets are reduced. Each threaded case is checked against a sequential run with the same chunking. -| threads | iter ms | total s | μs/trav | alloc MB | gc s | gc share | speedup | efficiency | -| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| 1 | 0.219 | 0.0219 | 0.219 | 0.000 | 0.000 | 0.0% | 1.00× | 100% | -| 2 | 0.129 | 0.0129 | 0.129 | 0.004 | 0.000 | 0.0% | 1.69× | 85% | -| 4 | 0.096 | 0.0096 | 0.096 | 0.004 | 0.000 | 0.0% | 2.28× | 57% | -| 8 | 0.090 | 0.0090 | 0.090 | 0.004 | 0.000 | 0.0% | 2.44× | 31% | +Light mode: -**Interpretation:** 8T efficiency 31% — contention/dispatch overhead dominates -at this tiny per-thread workload. Julia still improves absolute throughput, but -this is not near-linear scaling. The next scaling test should increase per-thread -work before treating this as a hard limit. +| threads | iter ms | speedup | efficiency | +| ---: | ---: | ---: | ---: | +| 1 | 0.223 | 1.00× | 100% | +| 2 | 0.130 | 1.71× | 85% | +| 4 | 0.097 | 2.29× | 57% | +| 8 | 0.093 | 2.40× | 30% | + +Heavy mode increases traversal count from 1000 to 50000 per iteration +without changing tree shape or algorithm. This raises 1T iter time from +~0.2 ms to ~8 ms, enough to amortize more thread dispatch overhead. + +| threads | iter ms | speedup | efficiency | +| ---: | ---: | ---: | ---: | +| 1 | 8.272 | 1.00× | 100% | +| 2 | 5.058 | 1.64× | 82% | +| 4 | 2.602 | 3.18× | 79% | +| 8 | 1.739 | 4.76× | 59% | + +**Interpretation:** heavy 8T efficiency 59% — PARTIAL. The 31% light result +was partly a toy-size artifact, but the heavier workload still does not reach +near-linear scaling. Julia delivers useful throughput scaling, not decisive +8-thread linearity. diff --git a/experiments/julia_cfr_toy/bench_cfr_threaded.jl b/experiments/julia_cfr_toy/bench_cfr_threaded.jl index 9601bbc..7d810ad 100644 --- a/experiments/julia_cfr_toy/bench_cfr_threaded.jl +++ b/experiments/julia_cfr_toy/bench_cfr_threaded.jl @@ -7,6 +7,7 @@ include("bench_cfr.jl") const THREAD_GRID = (1, 2, 4, 8) const RUNS_PER_CASE = 5 const ROOT_ACTIONS = 4 +const HEAVY_WORK_MULTIPLIER = 50 function iteration_ranges(chunks::Int)::Vector{UnitRange{Int}} ranges = Vector{UnitRange{Int}}(undef, chunks) @@ -39,10 +40,19 @@ function reduce_root!(out::Vector{Float64}, trees::Vector{CFRTree}, chunks::Int) end end -function run_chunk!(tree::CFRTree, range::UnitRange{Int}) +function run_iteration_with_traversals!(tree::CFRTree, iteration::Int, traversals_per_iter::Int) + base = (iteration - 1) * traversals_per_iter + value = 0.0 + for offset in 1:traversals_per_iter + value += traverse!(tree, 1, 0, base + offset) + end + return value +end + +function run_chunk!(tree::CFRTree, range::UnitRange{Int}, traversals_per_iter::Int) value = 0.0 for iteration in range - value += run_iteration!(tree, iteration) + value += run_iteration_with_traversals!(tree, iteration, traversals_per_iter) end return value end @@ -52,16 +62,17 @@ function run_case!( trees::Vector{CFRTree}, ranges::Vector{UnitRange{Int}}, chunks::Int; + traversals_per_iter::Int, threaded::Bool, ) reset_trees!(trees, chunks) if threaded && chunks > 1 @threads for chunk in 1:chunks - run_chunk!(trees[chunk], ranges[chunk]) + run_chunk!(trees[chunk], ranges[chunk], traversals_per_iter) end else for chunk in 1:chunks - run_chunk!(trees[chunk], ranges[chunk]) + run_chunk!(trees[chunk], ranges[chunk], traversals_per_iter) end end reduce_root!(out, trees, chunks) @@ -74,15 +85,15 @@ function assert_close(label::AbstractString, actual::Vector{Float64}, expected:: end end -function measure_case(chunks::Int) +function measure_case(chunks::Int, traversals_per_iter::Int) ranges = iteration_ranges(chunks) reference_trees = [CFRTree() for _ in 1:chunks] measured_trees = [CFRTree() for _ in 1:chunks] reference = zeros(Float64, ROOT_ACTIONS) actual = zeros(Float64, ROOT_ACTIONS) - run_case!(reference, reference_trees, ranges, chunks; threaded=false) - run_case!(actual, measured_trees, ranges, chunks; threaded=chunks > 1) + run_case!(reference, reference_trees, ranges, chunks; traversals_per_iter=traversals_per_iter, threaded=false) + run_case!(actual, measured_trees, ranges, chunks; traversals_per_iter=traversals_per_iter, threaded=chunks > 1) assert_close("warmup $(chunks)T", actual, reference) times = Vector{Float64}(undef, RUNS_PER_CASE) @@ -94,7 +105,14 @@ function measure_case(chunks::Int) elapsed_ref = Ref(0.0) allocated = @allocated begin elapsed_ref[] = @elapsed begin - run_case!(actual, measured_trees, ranges, chunks; threaded=chunks > 1) + run_case!( + actual, + measured_trees, + ranges, + chunks; + traversals_per_iter=traversals_per_iter, + threaded=chunks > 1, + ) end end after = Base.gc_num() @@ -109,9 +127,10 @@ function measure_case(chunks::Int) gc_time_s = median(gc_times) return Dict( "threads" => chunks, + "traversals_per_iter" => traversals_per_iter, "total_s" => total_s, "iter_ms" => total_s * 1000.0 / MEASURE_ITERATIONS, - "traversal_us" => total_s * 1_000_000.0 / (MEASURE_ITERATIONS * TRAVERSALS_PER_ITER), + "traversal_us" => total_s * 1_000_000.0 / (MEASURE_ITERATIONS * traversals_per_iter), "alloc_mb" => alloc_mb, "gc_time_s" => gc_time_s, "gc_share" => total_s > 0.0 ? gc_time_s / total_s : 0.0, @@ -128,30 +147,20 @@ function scaling_label(efficiency::Float64)::String return "contention" end -function main() - available = Threads.nthreads() - grid = [threads for threads in THREAD_GRID if threads <= available] - if isempty(grid) - error("no thread counts available") - end - - results = [measure_case(threads) for threads in grid] +function run_suite(label::AbstractString, traversals_per_iter::Int, grid::Vector{Int}) + results = [measure_case(threads, traversals_per_iter) for threads in grid] one_thread_s = results[1]["total_s"] - println("threads iter_ms total_s μs/trav alloc_MB gc_s gc_share speedup efficiency") + println("--- $(label) ---") + println("threads iter_ms speedup efficiency") for result in results threads = result["threads"] speedup = one_thread_s / result["total_s"] efficiency = speedup / threads @printf( - "%-7d %8.3f %8.4f %8.3f %9.3f %5.3f %8.1f%% %8.2f× %9.0f%%\n", + "%-7d %8.3f %8.2f× %9.0f%%\n", threads, result["iter_ms"], - result["total_s"], - result["traversal_us"], - result["alloc_mb"], - result["gc_time_s"], - result["gc_share"] * 100.0, speedup, efficiency * 100.0, ) @@ -161,6 +170,29 @@ function main() speedup = one_thread_s / last["total_s"] efficiency = speedup / last["threads"] @printf("%dT efficiency %.0f%% — %s\n", last["threads"], efficiency * 100.0, scaling_label(efficiency)) + return results +end + +function heavy_requested()::Bool + return "--heavy" in ARGS || get(ENV, "HEAVY", "0") == "1" +end + +function main() + available = Threads.nthreads() + grid = [threads for threads in THREAD_GRID if threads <= available] + if isempty(grid) + error("no thread counts available") + end + + run_suite("Light mode (existing)", TRAVERSALS_PER_ITER, grid) + if heavy_requested() + println() + run_suite( + "Heavy mode ($(HEAVY_WORK_MULTIPLIER)× work)", + TRAVERSALS_PER_ITER * HEAVY_WORK_MULTIPLIER, + grid, + ) + end end main()