diff --git a/docs/research/julia_port_evaluation.md b/docs/research/julia_port_evaluation.md index 0c5b17b..bc581a3 100644 --- a/docs/research/julia_port_evaluation.md +++ b/docs/research/julia_port_evaluation.md @@ -91,13 +91,37 @@ the heavier workload still does not reach near-linear 8-thread scaling. Julia delivers useful throughput scaling (4.76× at 8T), but this is not the decisive PASS threshold for the threading criterion. -## Open evidence (criteria 4, 5) +## Open evidence (criterion 5) -- **Flux.jl + CUDA.jl MLP forward at bs={1, 64, 256}.** Compare to - PyTorch numbers in `docs/performance.md`. Criterion 4. Not started. - **Real-game-state slice port.** Port `play_card` + scoring, run on a fixed corpus of game states, compare to current Cython. Criterion 5. - Not started. + Not started. Skipped for now because criterion 4 failed; per the + decision rule, the full Julia port is no longer a GO candidate on the + current evidence. + +### 2026-05-07 — Flux.jl + CUDA.jl MLP forward (criterion 4) + +Path: `experiments/julia_flux_mlp/`. + +Same `DeepCFRMLP` shape as the current Deep CFR default MLP: +365→512→512→512→22, ReLU, identical PyTorch-exported weights loaded +into Flux. Output parity passed with max absolute difference +`5.215e-08`. Timing uses 10 warmup forwards, then 100 timed forwards, +with CUDA synchronized around the timed loop in both runtimes. + +| backend | batch | forward ms | μs/state | ratio vs PyTorch | +| --- | ---: | ---: | ---: | ---: | +| PyTorch | 1 | 0.0829 | 82.8755 | 1.00× | +| Flux | 1 | 0.1669 | 166.8867 | 2.01× | +| PyTorch | 64 | 0.0927 | 1.4477 | 1.00× | +| Flux | 64 | 0.1909 | 2.9836 | 2.06× | +| PyTorch | 256 | 0.0877 | 0.3427 | 1.00× | +| Flux | 256 | 0.1758 | 0.6867 | 2.00× | + +**Verdict on criterion 4:** FAIL. bs=64 is ~2.06× slower than +PyTorch, outside the ±20% PASS band. bs=1 and bs=256 are also ~2× +slower, outside the ±30% bands. Criterion 5 was not run after this +FAIL because the full Julia-port decision rule is already blocked. ## Pass/fail thresholds (decided in advance) @@ -169,16 +193,17 @@ purpose. ## Decision posture -Promising but not enough to justify a port yet. The completed benchmarks -remove the main risk (GC under recursion) and confirm compute parity. -Heavy thread scaling upgrades criterion 3 from inconclusive to PARTIAL: -8T is 4.76× faster than 1T, but 59% efficiency is below the near-linear -PASS threshold. +No full Julia port on the current evidence. The completed benchmarks +remove the main risk (GC under recursion) and confirm compute parity, +but criterion 3 is only PARTIAL and criterion 4 is FAIL. Per the +decision rule, a full port would spend months to replace a PyTorch GPU +path that is already ~2× faster for the exact model shape we use. -This means Julia remains a credible option, but not a slam dunk. ML-stack -and game-state evidence (criteria 4, 5) must be positive before starting -a serious port plan. If those are positive, criterion 3 should be revisited -on a real traversal slice where each thread has substantially more work -than this toy benchmark. +Recommended next path: stay on Python/Cython and pursue Option B +(per-worker interleaved traversal) as the GIL-escape path. A narrower +Julia experiment could still be considered later for traversal-only +logic, but it would need an explicit hybrid plan that keeps PyTorch for +networks and separately proves PythonCall/PyCall overhead is acceptable. -Do not commit to porting on the current evidence alone. +Criterion 5 remains unrun because criterion 4 already blocks the full +port decision. diff --git a/experiments/julia_flux_mlp/Manifest.toml b/experiments/julia_flux_mlp/Manifest.toml new file mode 100644 index 0000000..ef0b9b7 --- /dev/null +++ b/experiments/julia_flux_mlp/Manifest.toml @@ -0,0 +1,1269 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.11.9" +manifest_format = "2.0" +project_hash = "e96cb07619308aef14b191163ae32aa9f26e52d4" + +[[deps.ADTypes]] +git-tree-sha1 = "bbc22a9a08a0ef6460041086d8a7b27940ed4ffd" +uuid = "47edcb42-4c32-4615-8424-f2b9edc5f35b" +version = "1.22.0" +weakdeps = ["ChainRulesCore", "ConstructionBase", "EnzymeCore"] + + [deps.ADTypes.extensions] + ADTypesChainRulesCoreExt = "ChainRulesCore" + ADTypesConstructionBaseExt = "ConstructionBase" + ADTypesEnzymeCoreExt = "EnzymeCore" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.5.0" +weakdeps = ["ChainRulesCore", "Test"] + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + AbstractFFTsTestExt = "Test" + +[[deps.Accessors]] +deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"] +git-tree-sha1 = "2eeb2c9bef11013efc6f8f97f32ee59b146b09fb" +uuid = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697" +version = "0.1.44" + + [deps.Accessors.extensions] + AxisKeysExt = "AxisKeys" + IntervalSetsExt = "IntervalSets" + LinearAlgebraExt = "LinearAlgebra" + StaticArraysExt = "StaticArrays" + StructArraysExt = "StructArrays" + TestExt = "Test" + UnitfulExt = "Unitful" + + [deps.Accessors.weakdeps] + AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "0761717147821d696c9470a7a86364b2fbd22fd8" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "4.5.2" +weakdeps = ["SparseArrays", "StaticArrays"] + + [deps.Adapt.extensions] + AdaptSparseArraysExt = "SparseArrays" + AdaptStaticArraysExt = "StaticArrays" + +[[deps.AliasTables]] +deps = ["PtrArrays", "Random"] +git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff" +uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8" +version = "1.1.3" + +[[deps.ArgCheck]] +git-tree-sha1 = "f9e9a66c9b7be1ad7372bbd9b062d9230c30c5ce" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.5.0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Atomix]] +deps = ["UnsafeAtomics"] +git-tree-sha1 = "b8651b2eb5796a386b0398a20b519a6a6150f75c" +uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" +version = "1.1.3" + + [deps.Atomix.extensions] + AtomixCUDAExt = "CUDA" + AtomixMetalExt = "Metal" + AtomixOpenCLExt = "OpenCL" + AtomixoneAPIExt = "oneAPI" + + [deps.Atomix.weakdeps] + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + OpenCL = "08131aa3-fb12-5dee-8b74-c09406e224a2" + oneAPI = "8f75cd03-7ff8-4ecb-9b8f-daf728133b1b" + +[[deps.BFloat16s]] +deps = ["LinearAlgebra", "Printf", "Random"] +git-tree-sha1 = "e386db8b4753b42caac75ac81d0a4fe161a68a97" +uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" +version = "0.6.1" + +[[deps.BangBang]] +deps = ["Accessors", "ConstructionBase", "InitialValues", "LinearAlgebra"] +git-tree-sha1 = "cceb62468025be98d42a5dc581b163c20896b040" +uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" +version = "0.4.9" + + [deps.BangBang.extensions] + BangBangChainRulesCoreExt = "ChainRulesCore" + BangBangDataFramesExt = "DataFrames" + BangBangStaticArraysExt = "StaticArrays" + BangBangStructArraysExt = "StructArrays" + BangBangTablesExt = "Tables" + BangBangTypedTablesExt = "TypedTables" + + [deps.BangBang.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.Baselet]] +git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" +uuid = "9718e550-a3fa-408a-8086-8db961cd8217" +version = "0.1.1" + +[[deps.CEnum]] +git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.5.0" + +[[deps.CUDA]] +deps = ["CUDACore", "CUDATools", "Reexport", "cuBLAS", "cuFFT", "cuRAND", "cuSOLVER", "cuSPARSE"] +git-tree-sha1 = "6e0ed8fe61c12fced0fcb6912bc73196e323fff2" +uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" +version = "6.1.0" + +[[deps.CUDACore]] +deps = ["Adapt", "BFloat16s", "CEnum", "CUDA_Compiler_jll", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "ExprTools", "GPUArrays", "GPUCompiler", "GPUToolbox", "KernelAbstractions", "LLVM", "LLVMLoopInfo", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "PrecompileTools", "Preferences", "Printf", "Random", "Random123", "RandomNumbers", "StaticArrays"] +git-tree-sha1 = "2d9540161684aa2b816cc711793725f5896b6d93" +uuid = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" +version = "6.1.0" +weakdeps = ["ChainRulesCore", "EnzymeCore", "SpecialFunctions"] + + [deps.CUDACore.extensions] + ChainRulesCoreExt = "ChainRulesCore" + EnzymeCoreExt = "EnzymeCore" + SpecialFunctionsExt = "SpecialFunctions" + +[[deps.CUDATools]] +deps = ["CUDACore", "CUDA_Compiler_jll", "CUPTI", "Crayons", "GPUCompiler", "LLVM", "NVML", "NVTX", "PrecompileTools", "Preferences", "PrettyTables", "Printf", "Statistics", "demumble_jll"] +git-tree-sha1 = "3de36a048e6abb49f9d34405e8cf7d0b27715ae7" +uuid = "9ec180c6-1c07-47c7-9e6e-ebefa4d1f6d0" +version = "6.1.0" + +[[deps.CUDA_Compiler_jll]] +deps = ["Artifacts", "CUDA_Driver_jll", "CUDA_Runtime_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "b977706846cb0a75d3842a1fed810ab2e6ab2f94" +uuid = "d1e2174e-dfdc-576e-b43e-73b79eb1aca8" +version = "0.4.3+0" + +[[deps.CUDA_Driver_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "TOML"] +git-tree-sha1 = "3b759ec65ac87ad192c2925114fa5c126657a5bd" +uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" +version = "13.2.1+0" + +[[deps.CUDA_Runtime_Discovery]] +deps = ["Libdl"] +git-tree-sha1 = "4fd010410c0c91c8b9bd6194a9ebdebecd9a8107" +uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" +version = "2.0.0" + +[[deps.CUDA_Runtime_jll]] +deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "c0314d9fb0ebd00e404feba4c3fbc04c9975abc1" +uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" +version = "0.21.0+1" + +[[deps.CUPTI]] +deps = ["CEnum", "CUDACore", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "GPUToolbox"] +git-tree-sha1 = "d57a233b32dc0f7f8eeb73dd986a01348dc4bef5" +uuid = "9e67e8f6-ba02-4b6c-a7db-3b11ae1e7ab7" +version = "6.1.0" + +[[deps.ChainRules]] +deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "SparseInverseSubset", "Statistics", "StructArrays", "SuiteSparse"] +git-tree-sha1 = "3c190c570fb3108c09f838607386d10c71701789" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "1.73.0" + +[[deps.ChainRulesCore]] +deps = ["Compat", "LinearAlgebra"] +git-tree-sha1 = "12177ad6b3cad7fd50c8b3825ce24a99ad61c18f" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.26.1" +weakdeps = ["SparseArrays"] + + [deps.ChainRulesCore.extensions] + ChainRulesCoreSparseArraysExt = "SparseArrays" + +[[deps.CommonSubexpressions]] +deps = ["MacroTools"] +git-tree-sha1 = "cda2cfaebb4be89c9084adaca7dd7333369715c5" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.1" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.18.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.1+0" + +[[deps.CompositionsBase]] +git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" +uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" +version = "0.1.2" +weakdeps = ["InverseFunctions"] + + [deps.CompositionsBase.extensions] + CompositionsBaseInverseFunctionsExt = "InverseFunctions" + +[[deps.ConstructionBase]] +git-tree-sha1 = "b4b092499347b18a015186eae3042f72267106cb" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.6.0" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseLinearAlgebraExt = "LinearAlgebra" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.ContextVariablesX]] +deps = ["Compat", "Logging", "UUIDs"] +git-tree-sha1 = "25cc3803f1030ab855e383129dcd3dc294e322cc" +uuid = "6add18c4-b38d-439d-96f6-d6bc489c04c5" +version = "0.1.3" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataStructures]] +deps = ["OrderedCollections"] +git-tree-sha1 = "e86f4a2805f7f19bec5129bc9150c38208e5dc23" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.19.4" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DefineSingletons]] +git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" +uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" +version = "0.1.2" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.DiffResults]] +deps = ["StaticArraysCore"] +git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.1.0" + +[[deps.DiffRules]] +deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.15.1" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + +[[deps.DocStringExtensions]] +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.5" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.EnzymeCore]] +git-tree-sha1 = "c6ee69ee502060982d12dbaaf3d8fcb4e835a0d1" +uuid = "f151be2c-9106-41f4-ab19-57ee4f262869" +version = "0.8.20" +weakdeps = ["Adapt", "ChainRulesCore"] + + [deps.EnzymeCore.extensions] + AdaptExt = "Adapt" + EnzymeCoreChainRulesCoreExt = "ChainRulesCore" + +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + +[[deps.FLoops]] +deps = ["BangBang", "Compat", "FLoopsBase", "InitialValues", "JuliaVariables", "MLStyle", "Serialization", "Setfield", "Transducers"] +git-tree-sha1 = "0a2e5873e9a5f54abb06418d57a8df689336a660" +uuid = "cc61a311-1640-44b5-9fba-1b764f453329" +version = "0.2.2" + +[[deps.FLoopsBase]] +deps = ["ContextVariablesX"] +git-tree-sha1 = "656f7a6859be8673bf1f35da5670246b923964f7" +uuid = "b9860ae5-e623-471e-878b-f6a53c775ea6" +version = "0.1.1" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.FillArrays]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.16.0" + + [deps.FillArrays.extensions] + FillArraysPDMatsExt = "PDMats" + FillArraysSparseArraysExt = "SparseArrays" + FillArraysStaticArraysExt = "StaticArrays" + FillArraysStatisticsExt = "Statistics" + + [deps.FillArrays.weakdeps] + PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.Flux]] +deps = ["ADTypes", "Adapt", "ChainRulesCore", "Compat", "EnzymeCore", "Functors", "GPUArrays", "LinearAlgebra", "MLCore", "MLDataDevices", "MLUtils", "MacroTools", "NNlib", "OneHotArrays", "Optimisers", "Preferences", "ProgressLogging", "Random", "Reexport", "Setfield", "SparseArrays", "SpecialFunctions", "Statistics", "Zygote"] +git-tree-sha1 = "cb318a415a089c337d0c15000d1608cee8434ebf" +uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" +version = "0.16.10" + + [deps.Flux.extensions] + FluxAMDGPUExt = "AMDGPU" + FluxCUDAExt = "CUDA" + FluxCUDAcuDNNExt = ["CUDA", "cuDNN"] + FluxEnzymeExt = "Enzyme" + FluxFiniteDifferencesExt = "FiniteDifferences" + FluxMPIExt = "MPI" + FluxMPINCCLExt = ["CUDA", "MPI", "NCCL"] + FluxMooncakeExt = "Mooncake" + + [deps.Flux.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" + MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" + Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6" + NCCL = "3fe64909-d7a1-4096-9b7d-7a0f12cf0f6b" + cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" + +[[deps.ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] +git-tree-sha1 = "cddeab6487248a39dae1a960fff0ac17b2a28888" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "1.3.3" +weakdeps = ["StaticArrays"] + + [deps.ForwardDiff.extensions] + ForwardDiffStaticArraysExt = "StaticArrays" + +[[deps.Functors]] +deps = ["Compat", "ConstructionBase", "LinearAlgebra", "Random"] +git-tree-sha1 = "60a0339f28a233601cb74468032b5c302d5067de" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.5.2" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.GPUArrays]] +deps = ["Adapt", "GPUArraysCore", "KernelAbstractions", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "ScopedValues", "Serialization", "SparseArrays", "Statistics"] +git-tree-sha1 = "34fd745547978beb471f029f447290ef4dbc7bbd" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "11.5.3" + + [deps.GPUArrays.extensions] + JLD2Ext = "JLD2" + + [deps.GPUArrays.weakdeps] + JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" + +[[deps.GPUArraysCore]] +deps = ["Adapt"] +git-tree-sha1 = "83cf05ab16a73219e5f6bd1bdfa9848fa24ac627" +uuid = "46192b85-c4d5-4398-a991-12ede77f4527" +version = "0.2.0" + +[[deps.GPUCompiler]] +deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "PrecompileTools", "Preferences", "Scratch", "Serialization", "TOML", "Tracy", "UUIDs"] +git-tree-sha1 = "31de8ab061cc64fb26fea90f34621b82ad0e9bc8" +uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" +version = "1.10.0" + +[[deps.GPUToolbox]] +deps = ["LLVM"] +git-tree-sha1 = "a589b6c1a0eff953571f5d8b0474f5020831114d" +uuid = "096a3bc2-3ced-46d0-87f4-dd12716f4bfc" +version = "1.1.1" + +[[deps.HashArrayMappedTries]] +git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" +uuid = "076d061b-32b6-4027-95e0-9a2c6f6d7e74" +version = "0.2.0" + +[[deps.IRTools]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "57e9ce6cf68d0abf5cb6b3b4abf9bedf05c939c0" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.15" + +[[deps.InitialValues]] +git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" +uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" +version = "0.3.1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.InverseFunctions]] +git-tree-sha1 = "a779299d77cd080bf77b97535acecd73e1c5e5cb" +uuid = "3587e190-3f89-42d0-90ee-14403ec27112" +version = "0.1.17" +weakdeps = ["Dates", "Test"] + + [deps.InverseFunctions.extensions] + InverseFunctionsDatesExt = "Dates" + InverseFunctionsTestExt = "Test" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.6" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.7.1" + +[[deps.JSON]] +deps = ["Dates", "Logging", "Parsers", "PrecompileTools", "StructUtils", "UUIDs", "Unicode"] +git-tree-sha1 = "fe23330af47b8ab4e135b2ff65f7398c3a2bfc65" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "1.5.2" + + [deps.JSON.extensions] + JSONArrowExt = ["ArrowTypes"] + + [deps.JSON.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + +[[deps.JuliaNVTXCallbacks_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "af433a10f3942e882d3c671aacb203e006a5808f" +uuid = "9c1d0b0a-7046-5b2e-a33f-ea22f176ac7e" +version = "0.2.1+0" + +[[deps.JuliaVariables]] +deps = ["MLStyle", "NameResolution"] +git-tree-sha1 = "49fb3cb53362ddadb4415e9b73926d6b40709e70" +uuid = "b14d175d-62b4-44ba-8fb7-3064adc8c3ec" +version = "0.2.4" + +[[deps.KernelAbstractions]] +deps = ["Adapt", "Atomix", "InteractiveUtils", "MacroTools", "PrecompileTools", "Requires", "StaticArrays", "UUIDs"] +git-tree-sha1 = "f2e76d3ced51a2a9e185abc0b97494c7273f649f" +uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +version = "0.9.41" +weakdeps = ["EnzymeCore", "LinearAlgebra", "SparseArrays"] + + [deps.KernelAbstractions.extensions] + EnzymeExt = "EnzymeCore" + LinearAlgebraExt = "LinearAlgebra" + SparseArraysExt = "SparseArrays" + +[[deps.LLVM]] +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "PrecompileTools", "Preferences", "Printf", "Unicode"] +git-tree-sha1 = "85592339c4363f40863f0b61f9cba80b885070c3" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "9.7.1" +weakdeps = ["BFloat16s"] + + [deps.LLVM.extensions] + BFloat16sExt = "BFloat16s" + +[[deps.LLVMExtra_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "f1d1adfff151fd02b4062d1af82df02052dc4a0c" +uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" +version = "0.0.42+0" + +[[deps.LLVMLoopInfo]] +git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" +uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586" +version = "1.0.0" + +[[deps.LaTeXStrings]] +git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.4.0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.6.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.7.2+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.LibTracyClient_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "d4e20500d210247322901841d4eafc7a0c52642d" +uuid = "ad6e5548-8b26-5c9f-8ef3-ef0ad883f3a5" +version = "0.13.1+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.11.0" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "13ca9e2586b89836fd20cccf56e57e2b9ae7f38f" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.29" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.MLCore]] +deps = ["DataAPI", "SimpleTraits", "Tables"] +git-tree-sha1 = "73907695f35bc7ffd9f11f6c4f2ee8c1302084be" +uuid = "c2834f40-e789-41da-a90e-33b280584a8c" +version = "1.0.0" + +[[deps.MLDataDevices]] +deps = ["Adapt", "Functors", "Preferences", "Random", "SciMLPublic"] +git-tree-sha1 = "29b00f22be6fd821a214760f0224329f21998a05" +uuid = "7e8f7934-dd98-4c1a-8fe8-92b47a384d40" +version = "1.17.10" + + [deps.MLDataDevices.extensions] + AMDGPUExt = "AMDGPU" + CUDAExt = "CUDA" + ChainRulesCoreExt = "ChainRulesCore" + ChainRulesExt = "ChainRules" + ComponentArraysExt = "ComponentArrays" + FillArraysExt = "FillArrays" + GPUArraysSparseArraysExt = ["GPUArrays", "SparseArrays"] + MLUtilsExt = "MLUtils" + MetalExt = ["GPUArrays", "Metal"] + OneHotArraysExt = "OneHotArrays" + OpenCLExt = ["GPUArrays", "OpenCL"] + ReactantExt = "Reactant" + RecursiveArrayToolsExt = "RecursiveArrayTools" + ReverseDiffExt = "ReverseDiff" + SparseArraysExt = "SparseArrays" + TrackerExt = "Tracker" + ZygoteExt = "Zygote" + cuDNNExt = ["CUDA", "cuDNN"] + oneAPIExt = ["GPUArrays", "oneAPI"] + + [deps.MLDataDevices.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2" + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" + FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" + GPUArrays = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" + MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + OneHotArrays = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" + OpenCL = "08131aa3-fb12-5dee-8b74-c09406e224a2" + Reactant = "3c362404-f566-11ee-1572-e11a4b42c853" + RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" + oneAPI = "8f75cd03-7ff8-4ecb-9b8f-daf728133b1b" + +[[deps.MLStyle]] +git-tree-sha1 = "bc38dff0548128765760c79eb7388a4b37fae2c8" +uuid = "d8e11817-5142-5d16-987a-aa16d5891078" +version = "0.4.17" + +[[deps.MLUtils]] +deps = ["ChainRulesCore", "Compat", "DataAPI", "DelimitedFiles", "FLoops", "MLCore", "NNlib", "Random", "ShowCases", "SimpleTraits", "Statistics", "StatsBase", "Tables", "Transducers"] +git-tree-sha1 = "a772d8d1987433538a5c226f79393324b55f7846" +uuid = "f1d291b0-491e-4a28-83b9-f70985020b54" +version = "0.4.8" + +[[deps.MacroTools]] +git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.16" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.6+0" + +[[deps.MicroCollections]] +deps = ["Accessors", "BangBang", "InitialValues"] +git-tree-sha1 = "44d32db644e84c75dab479f1bc15ee76a1a3618f" +uuid = "128add7d-3638-4c79-886c-908ea0c25c34" +version = "0.2.0" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.12.12" + +[[deps.NNlib]] +deps = ["Adapt", "Atomix", "ChainRulesCore", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "Random", "ScopedValues", "Statistics"] +git-tree-sha1 = "78cd28dbd5f03f99ccaba45c987107adcb61c115" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.9.34" + + [deps.NNlib.extensions] + NNlibAMDGPUExt = "AMDGPU" + NNlibCUDACUDNNExt = ["CUDA", "cuDNN"] + NNlibCUDAExt = "CUDA" + NNlibEnzymeCoreExt = "EnzymeCore" + NNlibFFTWExt = "FFTW" + NNlibForwardDiffExt = "ForwardDiff" + NNlibMetalExt = "Metal" + NNlibSpecialFunctionsExt = "SpecialFunctions" + + [deps.NNlib.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" + cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" + +[[deps.NVML]] +deps = ["CEnum", "CUDACore", "GPUToolbox", "Libdl"] +git-tree-sha1 = "269695e0ac0bfd854feba8179433c57e10dc95a7" +uuid = "611af6d1-644e-4c5d-bd58-854d7d1254b9" +version = "6.1.0" + +[[deps.NVTX]] +deps = ["JuliaNVTXCallbacks_jll", "Libdl", "NVTX_jll"] +git-tree-sha1 = "a9083c3e469e63cca454d1fc3b19472d9d92c14a" +uuid = "5da4648a-3479-48b8-97b9-01cb529c0a1f" +version = "1.0.3" + + [deps.NVTX.extensions] + NVTXColorsExt = "Colors" + + [deps.NVTX.weakdeps] + Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + +[[deps.NVTX_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "af2232f69447494514c25742ba1503ec7e9877fe" +uuid = "e98f9f5b-d649-5603-91fd-7774390e6439" +version = "3.2.2+0" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "9b8215b1ee9e78a293f99797cd31375471b2bcae" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.1.3" + +[[deps.NameResolution]] +deps = ["PrettyPrint"] +git-tree-sha1 = "1a0fa0e9613f46c9b8c11eee38ebb4f590013c5e" +uuid = "71a1bf82-56d0-4bbc-8a3c-48b961074391" +version = "0.1.5" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OneHotArrays]] +deps = ["Adapt", "ChainRulesCore", "Compat", "GPUArraysCore", "LinearAlgebra", "NNlib"] +git-tree-sha1 = "9510d7008275fc5b33fc72a73f8fddef0b5430c6" +uuid = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" +version = "0.2.11" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.27+1" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.5+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.6+0" + +[[deps.Optimisers]] +deps = ["ChainRulesCore", "ConstructionBase", "Functors", "LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "36b5d2b9dd06290cd65fcf5bdbc3a551ed133af5" +uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" +version = "0.4.7" + + [deps.Optimisers.extensions] + OptimisersAdaptExt = ["Adapt"] + OptimisersEnzymeCoreExt = "EnzymeCore" + OptimisersReactantExt = "Reactant" + + [deps.Optimisers.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + Reactant = "3c362404-f566-11ee-1572-e11a4b42c853" + +[[deps.OrderedCollections]] +git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.8.1" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "5d5e0a78e971354b1c7bff0655d11fdc1b0e12c8" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.4" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.11.0" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.5.2" + +[[deps.PrettyPrint]] +git-tree-sha1 = "632eb4abab3449ab30c5e1afaa874f0b98b586e4" +uuid = "8162dcfd-2161-5ef2-ae6c-7681170c5f98" +version = "0.2.0" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "624de6279ab7d94fc9f672f0068107eb6619732c" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "3.3.2" + + [deps.PrettyTables.extensions] + PrettyTablesTypstryExt = "Typstry" + + [deps.PrettyTables.weakdeps] + Typstry = "f0ed7684-a786-439e-b1e3-3b82803b501e" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.ProgressLogging]] +deps = ["Logging", "SHA", "UUIDs"] +git-tree-sha1 = "f0803bc1171e455a04124affa9c21bba5ac4db32" +uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" +version = "0.1.6" + +[[deps.PtrArrays]] +git-tree-sha1 = "4fbbafbc6251b883f4d2705356f3641f3652a7fe" +uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d" +version = "1.4.0" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.Random123]] +deps = ["Random", "RandomNumbers"] +git-tree-sha1 = "dbe5fd0b334694e905cb9fda73cd8554333c46e2" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.7.1" + +[[deps.RandomNumbers]] +deps = ["Random"] +git-tree-sha1 = "c6ec94d2aaba1ab2ff983052cf6a606ca5985902" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.6.0" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.1" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.SciMLPublic]] +git-tree-sha1 = "0ba076dbdce87ba230fff48ca9bca62e1f345c9b" +uuid = "431bcebd-1456-4ced-9d72-93c2757fff0b" +version = "1.0.1" + +[[deps.ScopedValues]] +deps = ["HashArrayMappedTries", "Logging"] +git-tree-sha1 = "67a144433c4ce877ee6d1ada69a124d6b1ecf7be" +uuid = "7e506255-f358-4e82-b7e4-beb19740aa63" +version = "1.6.2" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.3.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] +git-tree-sha1 = "c5391c6ace3bc430ca630251d02ea9687169ca68" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "1.1.2" + +[[deps.ShowCases]] +git-tree-sha1 = "7f534ad62ab2bd48591bdeac81994ea8c445e4a5" +uuid = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" +version = "0.1.0" + +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "be8eeac05ec97d379347584fa9fe2f5f76795bcb" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.5" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.2" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.11.0" + +[[deps.SparseInverseSubset]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "52962839426b75b3021296f7df242e40ecfc0852" +uuid = "dc90abb0-5640-4711-901d-7e5b23a2fada" +version = "0.1.2" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "2700b235561b0335d5bef7097a111dc513b8655e" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.7.2" +weakdeps = ["ChainRulesCore"] + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + +[[deps.SplittablesBase]] +deps = ["Setfield", "Test"] +git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" +uuid = "171d559e-b47b-412a-8079-5efa626c420e" +version = "0.1.15" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "246a8bb2e6667f832eea063c3a56aef96429a3db" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.18" +weakdeps = ["ChainRulesCore", "Statistics"] + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "6ab403037779dae8c514bad259f32a447262455a" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.4" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "178ed29fd5b2a2cfc3bd31c13375ae925623ff36" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.8.0" + +[[deps.StatsBase]] +deps = ["AliasTables", "DataAPI", "DataStructures", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "aceda6f4e598d331548e04cc6b2124a6148138e3" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.10" + +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.4.4" + +[[deps.StructArrays]] +deps = ["ConstructionBase", "DataAPI", "Tables"] +git-tree-sha1 = "ad8002667372439f2e3611cfd14097e03fa4bccd" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.7.3" +weakdeps = ["Adapt", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "SparseArrays", "StaticArrays"] + + [deps.StructArrays.extensions] + StructArraysAdaptExt = "Adapt" + StructArraysGPUArraysCoreExt = ["GPUArraysCore", "KernelAbstractions"] + StructArraysLinearAlgebraExt = "LinearAlgebra" + StructArraysSparseArraysExt = "SparseArrays" + StructArraysStaticArraysExt = "StaticArrays" + +[[deps.StructUtils]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "dd974aefe288ef2898733aecf40858dc86742d74" +uuid = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" +version = "2.8.1" + + [deps.StructUtils.extensions] + StructUtilsMeasurementsExt = ["Measurements"] + StructUtilsStaticArraysCoreExt = ["StaticArraysCore"] + StructUtilsTablesExt = ["Tables"] + + [deps.StructUtils.weakdeps] + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" + Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.7.0+0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.12.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.Tracy]] +deps = ["ExprTools", "LibTracyClient_jll", "Libdl"] +git-tree-sha1 = "73e3ff50fd3990874c59fef0f35d10644a1487bc" +uuid = "e689c965-62c8-4b79-b2c5-8359227902fd" +version = "0.1.6" + + [deps.Tracy.extensions] + TracyProfilerExt = "TracyProfiler_jll" + + [deps.Tracy.weakdeps] + TracyProfiler_jll = "0c351ed6-8a68-550e-8b79-de6f926da83c" + +[[deps.Transducers]] +deps = ["Accessors", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "ConstructionBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "SplittablesBase", "Tables"] +git-tree-sha1 = "4aa1fdf6c1da74661f6f5d3edfd96648321dade9" +uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" +version = "0.4.85" + + [deps.Transducers.extensions] + TransducersAdaptExt = "Adapt" + TransducersBlockArraysExt = "BlockArrays" + TransducersDataFramesExt = "DataFrames" + TransducersLazyArraysExt = "LazyArrays" + TransducersOnlineStatsBaseExt = "OnlineStatsBase" + TransducersReferenceablesExt = "Referenceables" + + [deps.Transducers.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" + Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.UnsafeAtomics]] +git-tree-sha1 = "0f30765c32d66d58e41f4cb5624d4fc8a82ec13b" +uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" +version = "0.3.1" +weakdeps = ["LLVM"] + + [deps.UnsafeAtomics.extensions] + UnsafeAtomicsLLVM = ["LLVM"] + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.Zygote]] +deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "GPUArraysCore", "IRTools", "InteractiveUtils", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "PrecompileTools", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "a29cbf3968d36022198bcc6f23fdfd70f7caf737" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.7.10" + + [deps.Zygote.extensions] + ZygoteAtomExt = "Atom" + ZygoteColorsExt = "Colors" + ZygoteDistancesExt = "Distances" + ZygoteTrackerExt = "Tracker" + + [deps.Zygote.weakdeps] + Atom = "c52e3926-4ff0-5f6e-af25-54175e0327b1" + Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.ZygoteRules]] +deps = ["ChainRulesCore", "MacroTools"] +git-tree-sha1 = "434b3de333c75fc446aa0d19fc394edafd07ab08" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.7" + +[[deps.cuBLAS]] +deps = ["Adapt", "BFloat16s", "CEnum", "CUDACore", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "GPUArrays", "GPUToolbox", "LLVM", "LinearAlgebra"] +git-tree-sha1 = "9d6409c7fbdef7105385105b172bb193897ec7a3" +uuid = "182d3088-87b7-4494-8cad-fc6afaa545bc" +version = "6.1.0" +weakdeps = ["EnzymeCore"] + + [deps.cuBLAS.extensions] + EnzymeCoreExt = "EnzymeCore" + +[[deps.cuFFT]] +deps = ["AbstractFFTs", "CEnum", "CUDACore", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "GPUToolbox", "LinearAlgebra", "Reexport"] +git-tree-sha1 = "772beeca0de8e7e58ec013b14fbda981b74108f5" +uuid = "533571aa-0936-420e-b4be-9c66f5f626ca" +version = "6.1.0" + +[[deps.cuRAND]] +deps = ["CEnum", "CUDACore", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "GPUToolbox", "Random", "Random123", "RandomNumbers"] +git-tree-sha1 = "d0772ca43c4500c0cce1ff02a9dfda8faf65c960" +uuid = "20fd9a0b-12d5-4c2f-a8af-7c34e9e60431" +version = "6.1.0" + +[[deps.cuSOLVER]] +deps = ["CEnum", "CUDACore", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "GPUToolbox", "LinearAlgebra", "SparseArrays", "cuBLAS", "cuSPARSE"] +git-tree-sha1 = "dae250870a645b30a534a4176985993e615eb944" +uuid = "887afef0-6a32-4de5-add4-7827692ba8fc" +version = "6.1.0" + +[[deps.cuSPARSE]] +deps = ["Adapt", "CEnum", "CUDACore", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "GPUArrays", "GPUToolbox", "KernelAbstractions", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "5da59442cc13bb1a8647bbe20eba66917dc3a64b" +uuid = "b26da814-b3bc-49ef-b0ee-c816305aa060" +version = "6.1.0" + + [deps.cuSPARSE.extensions] + SparseMatricesCSRExt = "SparseMatricesCSR" + + [deps.cuSPARSE.weakdeps] + SparseMatricesCSR = "a0a7dd2c-ebf4-11e9-1f05-cf50bc540ca1" + +[[deps.demumble_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6498e3581023f8e530f34760d18f75a69e3a4ea8" +uuid = "1e29f10c-031c-5a83-9565-69cddfc27673" +version = "1.3.0+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.11.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.59.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" diff --git a/experiments/julia_flux_mlp/Project.toml b/experiments/julia_flux_mlp/Project.toml new file mode 100644 index 0000000..247ee89 --- /dev/null +++ b/experiments/julia_flux_mlp/Project.toml @@ -0,0 +1,8 @@ +name = "JuliaFluxMLP" +uuid = "9fa932db-d4ab-4cbf-935e-32425028f0bc" +version = "0.1.0" + +[deps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" diff --git a/experiments/julia_flux_mlp/README.md b/experiments/julia_flux_mlp/README.md new file mode 100644 index 0000000..6910e7b --- /dev/null +++ b/experiments/julia_flux_mlp/README.md @@ -0,0 +1,45 @@ +# Julia Flux MLP Forward Benchmark + +Criterion 4 for the Julia port evaluation. + +Compares PyTorch `DeepCFRMLP` against a Flux/CUDA implementation with the +same shape and identical exported weights: + +- input dim: 365 +- hidden size: 512 +- hidden layers: 3 +- output dim: 22 +- activation: ReLU + +Run: + +```bash +uv run python experiments/julia_flux_mlp/bench_pytorch_vs_flux.py +``` + +The runner exports PyTorch weights and inputs to `runs/tmp/`, runs Flux on the +same payload, checks output equivalence, then writes `results.json`. + +## Results (2026-05-07) + +Host GPU: NVIDIA GeForce RTX 3090. Timing uses 10 warmup forwards, then 100 +timed forwards, with CUDA synchronized around the timed loop in both runtimes. + +| Backend | batch | forward ms | μs/state | ratio vs PyTorch | +| --- | ---: | ---: | ---: | ---: | +| PyTorch | 1 | 0.0829 | 82.8755 | 1.00× | +| Flux | 1 | 0.1669 | 166.8867 | 2.01× | +| PyTorch | 64 | 0.0927 | 1.4477 | 1.00× | +| Flux | 64 | 0.1909 | 2.9836 | 2.06× | +| PyTorch | 256 | 0.0877 | 0.3427 | 1.00× | +| Flux | 256 | 0.1758 | 0.6867 | 2.00× | + +Maximum output difference: `5.215e-08`. + +Criterion 4 threshold: + +- bs=64 must be within ±20% of PyTorch. +- bs=1 and bs=256 must be within ±30% of PyTorch. + +**Verdict:** FAIL. Flux/CUDA is ~2.0× slower than PyTorch at all measured +batch sizes for this model shape. diff --git a/experiments/julia_flux_mlp/bench_flux.jl b/experiments/julia_flux_mlp/bench_flux.jl new file mode 100644 index 0000000..e9403f9 --- /dev/null +++ b/experiments/julia_flux_mlp/bench_flux.jl @@ -0,0 +1,120 @@ +using CUDA +using Flux +using JSON +using Printf + +const WARMUP_ITERS = 10 +const TIMED_ITERS = 100 + +function dense_from_payload(layer) + weight = Float32.(layer["weight"]) + bias = Float32.(layer["bias"]) + out_dim = Int(layer["out_dim"]) + in_dim = Int(layer["in_dim"]) + dense = Dense(in_dim => out_dim) + dense.weight .= reshape(weight, in_dim, out_dim)' + dense.bias .= bias + return dense +end + +function build_model(payload) + layers = Any[] + dense_layers = payload["layers"] + for (index, layer) in enumerate(dense_layers) + push!(layers, dense_from_payload(layer)) + if index < length(dense_layers) + push!(layers, relu) + end + end + return Flux.fmap(cu, Chain(layers...)) +end + +function input_matrix(batch_payload) + batch_size = Int(batch_payload["batch_size"]) + input_dim = Int(batch_payload["input_dim"]) + flat = Float32.(batch_payload["input"]) + x = Matrix{Float32}(undef, input_dim, batch_size) + @inbounds for sample in 1:batch_size + source_base = (sample - 1) * input_dim + for feature in 1:input_dim + x[feature, sample] = flat[source_base + feature] + end + end + return cu(x) +end + +function flatten_output(y) + cpu = Array(y) + out_dim, batch_size = size(cpu) + flat = Vector{Float32}(undef, batch_size * out_dim) + @inbounds for sample in 1:batch_size + dest_base = (sample - 1) * out_dim + for output in 1:out_dim + flat[dest_base + output] = cpu[output, sample] + end + end + return flat +end + +function timed_forward_ms(model, x) + CUDA.synchronize() + for _ in 1:WARMUP_ITERS + model(x) + end + CUDA.synchronize() + + elapsed = @elapsed begin + for _ in 1:TIMED_ITERS + model(x) + end + CUDA.synchronize() + end + return elapsed * 1000.0 / TIMED_ITERS +end + +function benchmark(payload) + CUDA.allowscalar(false) + model = build_model(payload) + results = Dict{String,Any}() + max_abs_diff = 0.0 + for batch_payload in payload["batches"] + batch_size = Int(batch_payload["batch_size"]) + x = input_matrix(batch_payload) + y = model(x) + CUDA.synchronize() + actual = flatten_output(y) + expected = Float32.(batch_payload["expected_output"]) + diff = maximum(abs.(actual .- expected)) + max_abs_diff = max(max_abs_diff, Float64(diff)) + ms = timed_forward_ms(model, x) + results[string(batch_size)] = Dict( + "forward_ms" => ms, + "us_per_state" => ms * 1000.0 / batch_size, + "max_abs_diff" => Float64(diff), + ) + end + return Dict( + "lang" => "Julia/Flux", + "device" => string(CUDA.name(CUDA.device())), + "timed_iters" => TIMED_ITERS, + "warmup_iters" => WARMUP_ITERS, + "max_abs_diff" => max_abs_diff, + "batches" => results, + ) +end + +function print_json(result) + println(JSON.json(result)) +end + +function main() + if length(ARGS) != 1 + println(stderr, "usage: julia --project=experiments/julia_flux_mlp experiments/julia_flux_mlp/bench_flux.jl ") + exit(2) + end + payload = JSON.parsefile(ARGS[1]) + result = benchmark(payload) + print_json(result) +end + +main() diff --git a/experiments/julia_flux_mlp/bench_pytorch_vs_flux.py b/experiments/julia_flux_mlp/bench_pytorch_vs_flux.py new file mode 100644 index 0000000..4c74509 --- /dev/null +++ b/experiments/julia_flux_mlp/bench_pytorch_vs_flux.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import json +import subprocess +import time +from pathlib import Path +from typing import Any + +import torch + +from coolrl_lost_cities.games.classic.deep_cfr.networks import DeepCFRMLP + +ROOT = Path(__file__).resolve().parent +REPO_ROOT = ROOT.parents[1] +BATCH_SIZES = (1, 64, 256) +INPUT_DIM = 365 +HIDDEN_SIZE = 512 +NUM_LAYERS = 3 +OUTPUT_DIM = 22 +WARMUP_ITERS = 10 +TIMED_ITERS = 100 +SEED = 20260507 +MAX_OUTPUT_DIFF = 1e-4 + + +def _julia_executable() -> str: + local = REPO_ROOT / "tools" / "julia" / "current" / "bin" / "julia" + if local.exists(): + return str(local) + return "julia" + + +def _linear_layers(model: DeepCFRMLP) -> list[torch.nn.Linear]: + return [layer for layer in model.net if isinstance(layer, torch.nn.Linear)] + + +def _export_layers(model: DeepCFRMLP) -> list[dict[str, Any]]: + layers = [] + for layer in _linear_layers(model): + weight = layer.weight.detach().cpu().contiguous() + bias = layer.bias.detach().cpu().contiguous() + layers.append( + { + "in_dim": int(weight.shape[1]), + "out_dim": int(weight.shape[0]), + "weight": weight.flatten().tolist(), + "bias": bias.flatten().tolist(), + } + ) + return layers + + +def _time_pytorch(model: DeepCFRMLP, x: torch.Tensor) -> float: + with torch.inference_mode(): + for _ in range(WARMUP_ITERS): + model(x) + torch.cuda.synchronize() + start = time.perf_counter() + for _ in range(TIMED_ITERS): + model(x) + torch.cuda.synchronize() + return float((time.perf_counter() - start) * 1000.0 / TIMED_ITERS) + + +def _build_payload_and_pytorch_results(payload_path: Path) -> dict[str, Any]: + if not torch.cuda.is_available(): + raise SystemExit("CUDA is not available to PyTorch; aborting Criterion 4") + torch.manual_seed(SEED) + torch.cuda.manual_seed_all(SEED) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + + model = ( + DeepCFRMLP( + INPUT_DIM, + OUTPUT_DIM, + HIDDEN_SIZE, + num_layers=NUM_LAYERS, + activation="relu", + ) + .eval() + .cuda() + ) + + payload: dict[str, Any] = { + "input_dim": INPUT_DIM, + "hidden_size": HIDDEN_SIZE, + "num_layers": NUM_LAYERS, + "output_dim": OUTPUT_DIM, + "activation": "relu", + "layers": _export_layers(model), + "batches": [], + } + pytorch_batches: dict[str, Any] = {} + generator = torch.Generator(device="cpu").manual_seed(SEED + 1) + with torch.inference_mode(): + for batch_size in BATCH_SIZES: + x_cpu = torch.randn(batch_size, INPUT_DIM, generator=generator, dtype=torch.float32) + x_gpu = x_cpu.cuda() + y_cpu = model(x_gpu).detach().cpu().contiguous() + forward_ms = _time_pytorch(model, x_gpu) + payload["batches"].append( + { + "batch_size": batch_size, + "input_dim": INPUT_DIM, + "input": x_cpu.contiguous().flatten().tolist(), + "expected_output": y_cpu.flatten().tolist(), + } + ) + pytorch_batches[str(batch_size)] = { + "forward_ms": forward_ms, + "us_per_state": forward_ms * 1000.0 / batch_size, + } + + payload_path.parent.mkdir(parents=True, exist_ok=True) + payload_path.write_text(json.dumps(payload), encoding="utf-8") + return { + "lang": "PyTorch", + "device": torch.cuda.get_device_name(0), + "timed_iters": TIMED_ITERS, + "warmup_iters": WARMUP_ITERS, + "batches": pytorch_batches, + } + + +def _run_flux(payload_path: Path) -> dict[str, Any]: + proc = subprocess.run( + [ + _julia_executable(), + "--project=experiments/julia_flux_mlp", + "experiments/julia_flux_mlp/bench_flux.jl", + str(payload_path), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return json.loads(proc.stdout) + + +def _verdict(pytorch: dict[str, Any], flux: dict[str, Any]) -> tuple[str, dict[str, float]]: + ratios = {} + for batch_size in BATCH_SIZES: + key = str(batch_size) + ratio = flux["batches"][key]["us_per_state"] / pytorch["batches"][key]["us_per_state"] + ratios[key] = ratio + within_64 = 0.8 <= ratios["64"] <= 1.2 + within_1 = 0.7 <= ratios["1"] <= 1.3 + within_256 = 0.7 <= ratios["256"] <= 1.3 + return ("PASS" if within_64 and within_1 and within_256 else "FAIL"), ratios + + +def _print_table( + pytorch: dict[str, Any], flux: dict[str, Any], ratios: dict[str, float], verdict: str +) -> None: + print("Backend batch forward_ms us/state ratio vs PyTorch") + for batch_size in BATCH_SIZES: + key = str(batch_size) + p = pytorch["batches"][key] + f = flux["batches"][key] + print( + f"PyTorch {batch_size:5d} {p['forward_ms']:10.4f} {p['us_per_state']:8.4f} 1.00x" + ) + print( + f"Flux {batch_size:5d} {f['forward_ms']:10.4f} " + f"{f['us_per_state']:8.4f} {ratios[key]:.2f}x" + ) + print(f"max output diff: {flux['max_abs_diff']:.3e}") + print(f"Criterion 4 verdict: {verdict}") + + +def main() -> None: + payload_path = REPO_ROOT / "runs" / "tmp" / "julia_flux_mlp_payload.json" + pytorch = _build_payload_and_pytorch_results(payload_path) + flux = _run_flux(payload_path) + if float(flux["max_abs_diff"]) > MAX_OUTPUT_DIFF: + raise SystemExit(f"Flux output mismatch: max_abs_diff={flux['max_abs_diff']:.3e}") + verdict, ratios = _verdict(pytorch, flux) + result = {"pytorch": pytorch, "flux": flux, "ratios": ratios, "verdict": verdict} + result_path = ROOT / "results.json" + result_path.write_text(json.dumps(result, indent=2, sort_keys=True), encoding="utf-8") + _print_table(pytorch, flux, ratios, verdict) + print(f"wrote {result_path}") + + +if __name__ == "__main__": + main() diff --git a/experiments/julia_flux_mlp/results.json b/experiments/julia_flux_mlp/results.json new file mode 100644 index 0000000..85e6e2e --- /dev/null +++ b/experiments/julia_flux_mlp/results.json @@ -0,0 +1,52 @@ +{ + "flux": { + "batches": { + "1": { + "forward_ms": 0.16688673, + "max_abs_diff": 1.4901161193847656e-08, + "us_per_state": 166.88673 + }, + "256": { + "forward_ms": 0.17579791, + "max_abs_diff": 2.9802322387695312e-08, + "us_per_state": 0.6867105859375 + }, + "64": { + "forward_ms": 0.19094904, + "max_abs_diff": 5.21540641784668e-08, + "us_per_state": 2.98357875 + } + }, + "device": "NVIDIA GeForce RTX 3090", + "lang": "Julia/Flux", + "max_abs_diff": 5.21540641784668e-08, + "timed_iters": 100, + "warmup_iters": 10 + }, + "pytorch": { + "batches": { + "1": { + "forward_ms": 0.0828754500253126, + "us_per_state": 82.8754500253126 + }, + "256": { + "forward_ms": 0.0877418098389171, + "us_per_state": 0.34274144468326995 + }, + "64": { + "forward_ms": 0.09265014989068732, + "us_per_state": 1.4476585920419893 + } + }, + "device": "NVIDIA GeForce RTX 3090", + "lang": "PyTorch", + "timed_iters": 100, + "warmup_iters": 10 + }, + "ratios": { + "1": 2.0137052643337428, + "256": 2.0035819904187386, + "64": 2.0609684951971476 + }, + "verdict": "FAIL" +} \ No newline at end of file diff --git a/experiments/julia_flux_mlp/src/JuliaFluxMLP.jl b/experiments/julia_flux_mlp/src/JuliaFluxMLP.jl new file mode 100644 index 0000000..cb39e86 --- /dev/null +++ b/experiments/julia_flux_mlp/src/JuliaFluxMLP.jl @@ -0,0 +1,3 @@ +module JuliaFluxMLP + +end