coolguyandCodex
7c20d53103
Add a microbench that separates game/encoding overhead, single-request policy boundary overhead, and batched PyTorch forward lower bounds. Record CPU/CUDA results and link the finding from the Julia port evaluation. Co-Authored-By: Codex <codex@openai.com>
240 lines
11 KiB
JSON
240 lines
11 KiB
JSON
{
|
|
"config": "configs/deep_cfr/default.yaml",
|
|
"device": "cuda",
|
|
"torch_threads": 1,
|
|
"traversals": 8,
|
|
"runs": 3,
|
|
"warmup": 1,
|
|
"corpus_size": 512,
|
|
"input_dim": 365,
|
|
"action_size": 22,
|
|
"full_traversal": {
|
|
"seconds": [
|
|
0.3639091220102273,
|
|
0.623861116997432,
|
|
0.5022158679785207
|
|
],
|
|
"median_seconds": 0.5022158679785207,
|
|
"median_nodes": 3010.0,
|
|
"median_policy_calls": 1502.0,
|
|
"median_us_per_node": 163.2289683405107,
|
|
"median_us_per_policy_call": 326.97123532360166,
|
|
"stats": [
|
|
{
|
|
"traversal_nodes": 2256,
|
|
"traversal_terminals": 8,
|
|
"traversal_depth_cutoffs": 0,
|
|
"traversal_node_limit_cutoffs": 0,
|
|
"traversal_max_depth_reached": 398,
|
|
"traversal_advantage_samples": 1126,
|
|
"traversal_strategy_samples": 1126,
|
|
"traversal_sampled_actions": 1126,
|
|
"traversal_regret_matching_decisions": 1126,
|
|
"traversal_regret_fallback_count": 35,
|
|
"traversal_regret_fallback_rate": 0.03108348134991119,
|
|
"traversal_regret_fallback_avg_depth": 113.8,
|
|
"traversal_regret_fallback_action_play_existing": 0,
|
|
"traversal_regret_fallback_action_open_new": 0,
|
|
"traversal_regret_fallback_action_discard": 0,
|
|
"traversal_regret_fallback_action_draw_deck": 19,
|
|
"traversal_regret_fallback_action_draw_pile": 16,
|
|
"traversal_regret_fallback_legal_actions_mean": 3.057142857142857,
|
|
"traversal_regret_fallback_legal_play_existing_mean": 0.0,
|
|
"traversal_regret_fallback_legal_open_new_mean": 0.0,
|
|
"traversal_regret_fallback_legal_discard_mean": 0.0,
|
|
"traversal_regret_fallback_legal_draw_deck_mean": 1.0,
|
|
"traversal_regret_fallback_legal_draw_pile_mean": 2.057142857142857,
|
|
"traversal_regret_fallback_open_new_available_rate": 0.0,
|
|
"traversal_regret_fallback_open_new_selected": 0,
|
|
"traversal_regret_fallback_open_new_selected_rate": 0.0,
|
|
"traversal_regret_fallback_open_new_selection_over_availability": 0.0,
|
|
"traversal_regret_fallback_avg_opened_colors_before_action": 4.057142857142857,
|
|
"traversal_regret_fallback_argmax_tie_count": 0,
|
|
"traversal_regret_fallback_argmax_tie_rate": 0.0,
|
|
"traversal_regret_fallback_argmax_tie_size_mean": 0.0,
|
|
"traversal_regret_fallback_argmax_full_tie_count": 0,
|
|
"traversal_regret_fallback_argmax_full_tie_rate": 0.0,
|
|
"traversal_cutoff_rollouts": 0,
|
|
"traversal_cutoff_rollout_steps": 0,
|
|
"traversal_cutoff_rollout_timeouts": 0,
|
|
"traversal_endpoint_depth_sum": 2248,
|
|
"traversal_endpoints": 8,
|
|
"traversal_avg_endpoint_depth": 281.0,
|
|
"traversal_endpoint_depth_bucket_300_399": 2,
|
|
"traversal_endpoint_depth_bucket_200_299": 6,
|
|
"traversal_regret_fallback_depth_bucket_150_199": 7,
|
|
"traversal_regret_fallback_depth_bucket_50_99": 5,
|
|
"traversal_regret_fallback_depth_bucket_0_49": 11,
|
|
"traversal_regret_fallback_depth_bucket_350_399": 1,
|
|
"traversal_regret_fallback_depth_bucket_200_249": 4,
|
|
"traversal_regret_fallback_depth_bucket_100_149": 7,
|
|
"traversal_regret_fallback_opened_colors_count_5": 24,
|
|
"traversal_regret_fallback_opened_colors_count_0": 3,
|
|
"traversal_regret_fallback_opened_colors_count_4": 4,
|
|
"traversal_regret_fallback_opened_colors_count_1": 3,
|
|
"traversal_regret_fallback_opened_colors_count_3": 1,
|
|
"advantage_samples_returned": 1126,
|
|
"strategy_samples_returned": 1126
|
|
},
|
|
{
|
|
"traversal_nodes": 3822,
|
|
"traversal_terminals": 8,
|
|
"traversal_depth_cutoffs": 0,
|
|
"traversal_node_limit_cutoffs": 0,
|
|
"traversal_max_depth_reached": 560,
|
|
"traversal_advantage_samples": 1908,
|
|
"traversal_strategy_samples": 1908,
|
|
"traversal_sampled_actions": 1908,
|
|
"traversal_regret_matching_decisions": 1908,
|
|
"traversal_regret_fallback_count": 96,
|
|
"traversal_regret_fallback_rate": 0.050314465408805034,
|
|
"traversal_regret_fallback_avg_depth": 64.83333333333333,
|
|
"traversal_regret_fallback_action_play_existing": 0,
|
|
"traversal_regret_fallback_action_open_new": 0,
|
|
"traversal_regret_fallback_action_discard": 0,
|
|
"traversal_regret_fallback_action_draw_deck": 47,
|
|
"traversal_regret_fallback_action_draw_pile": 49,
|
|
"traversal_regret_fallback_legal_actions_mean": 1.90625,
|
|
"traversal_regret_fallback_legal_play_existing_mean": 0.0,
|
|
"traversal_regret_fallback_legal_open_new_mean": 0.0,
|
|
"traversal_regret_fallback_legal_discard_mean": 0.0,
|
|
"traversal_regret_fallback_legal_draw_deck_mean": 1.0,
|
|
"traversal_regret_fallback_legal_draw_pile_mean": 0.90625,
|
|
"traversal_regret_fallback_open_new_available_rate": 0.0,
|
|
"traversal_regret_fallback_open_new_selected": 0,
|
|
"traversal_regret_fallback_open_new_selected_rate": 0.0,
|
|
"traversal_regret_fallback_open_new_selection_over_availability": 0.0,
|
|
"traversal_regret_fallback_avg_opened_colors_before_action": 3.25,
|
|
"traversal_regret_fallback_argmax_tie_count": 0,
|
|
"traversal_regret_fallback_argmax_tie_rate": 0.0,
|
|
"traversal_regret_fallback_argmax_tie_size_mean": 0.0,
|
|
"traversal_regret_fallback_argmax_full_tie_count": 0,
|
|
"traversal_regret_fallback_argmax_full_tie_rate": 0.0,
|
|
"traversal_cutoff_rollouts": 0,
|
|
"traversal_cutoff_rollout_steps": 0,
|
|
"traversal_cutoff_rollout_timeouts": 0,
|
|
"traversal_endpoint_depth_sum": 3814,
|
|
"traversal_endpoints": 8,
|
|
"traversal_avg_endpoint_depth": 476.75,
|
|
"traversal_endpoint_depth_bucket_400_499": 4,
|
|
"traversal_endpoint_depth_bucket_300_399": 1,
|
|
"traversal_endpoint_depth_bucket_500_599": 3,
|
|
"traversal_regret_fallback_depth_bucket_300_349": 1,
|
|
"traversal_regret_fallback_depth_bucket_200_249": 4,
|
|
"traversal_regret_fallback_depth_bucket_150_199": 4,
|
|
"traversal_regret_fallback_depth_bucket_50_99": 24,
|
|
"traversal_regret_fallback_depth_bucket_0_49": 50,
|
|
"traversal_regret_fallback_depth_bucket_100_149": 13,
|
|
"traversal_regret_fallback_opened_colors_count_5": 24,
|
|
"traversal_regret_fallback_opened_colors_count_4": 22,
|
|
"traversal_regret_fallback_opened_colors_count_3": 20,
|
|
"traversal_regret_fallback_opened_colors_count_1": 14,
|
|
"traversal_regret_fallback_opened_colors_count_2": 15,
|
|
"traversal_regret_fallback_opened_colors_count_0": 1,
|
|
"advantage_samples_returned": 1908,
|
|
"strategy_samples_returned": 1908
|
|
},
|
|
{
|
|
"traversal_nodes": 3010,
|
|
"traversal_terminals": 8,
|
|
"traversal_depth_cutoffs": 0,
|
|
"traversal_node_limit_cutoffs": 0,
|
|
"traversal_max_depth_reached": 532,
|
|
"traversal_advantage_samples": 1502,
|
|
"traversal_strategy_samples": 1502,
|
|
"traversal_sampled_actions": 1502,
|
|
"traversal_regret_matching_decisions": 1502,
|
|
"traversal_regret_fallback_count": 72,
|
|
"traversal_regret_fallback_rate": 0.047936085219707054,
|
|
"traversal_regret_fallback_avg_depth": 246.29166666666666,
|
|
"traversal_regret_fallback_action_play_existing": 1,
|
|
"traversal_regret_fallback_action_open_new": 0,
|
|
"traversal_regret_fallback_action_discard": 54,
|
|
"traversal_regret_fallback_action_draw_deck": 17,
|
|
"traversal_regret_fallback_action_draw_pile": 0,
|
|
"traversal_regret_fallback_legal_actions_mean": 6.375,
|
|
"traversal_regret_fallback_legal_play_existing_mean": 0.027777777777777776,
|
|
"traversal_regret_fallback_legal_open_new_mean": 0.0,
|
|
"traversal_regret_fallback_legal_discard_mean": 6.111111111111111,
|
|
"traversal_regret_fallback_legal_draw_deck_mean": 0.2361111111111111,
|
|
"traversal_regret_fallback_legal_draw_pile_mean": 0.0,
|
|
"traversal_regret_fallback_open_new_available_rate": 0.0,
|
|
"traversal_regret_fallback_open_new_selected": 0,
|
|
"traversal_regret_fallback_open_new_selected_rate": 0.0,
|
|
"traversal_regret_fallback_open_new_selection_over_availability": 0.0,
|
|
"traversal_regret_fallback_avg_opened_colors_before_action": 4.638888888888889,
|
|
"traversal_regret_fallback_argmax_tie_count": 0,
|
|
"traversal_regret_fallback_argmax_tie_rate": 0.0,
|
|
"traversal_regret_fallback_argmax_tie_size_mean": 0.0,
|
|
"traversal_regret_fallback_argmax_full_tie_count": 0,
|
|
"traversal_regret_fallback_argmax_full_tie_rate": 0.0,
|
|
"traversal_cutoff_rollouts": 0,
|
|
"traversal_cutoff_rollout_steps": 0,
|
|
"traversal_cutoff_rollout_timeouts": 0,
|
|
"traversal_endpoint_depth_sum": 3002,
|
|
"traversal_endpoints": 8,
|
|
"traversal_avg_endpoint_depth": 375.25,
|
|
"traversal_endpoint_depth_bucket_300_399": 7,
|
|
"traversal_endpoint_depth_bucket_500_599": 1,
|
|
"traversal_regret_fallback_depth_bucket_300_349": 11,
|
|
"traversal_regret_fallback_depth_bucket_250_299": 18,
|
|
"traversal_regret_fallback_depth_bucket_150_199": 5,
|
|
"traversal_regret_fallback_depth_bucket_0_49": 17,
|
|
"traversal_regret_fallback_depth_bucket_200_249": 6,
|
|
"traversal_regret_fallback_depth_bucket_400_plus": 11,
|
|
"traversal_regret_fallback_depth_bucket_350_399": 4,
|
|
"traversal_regret_fallback_opened_colors_count_5": 62,
|
|
"traversal_regret_fallback_opened_colors_count_2": 2,
|
|
"traversal_regret_fallback_opened_colors_count_3": 2,
|
|
"traversal_regret_fallback_opened_colors_count_4": 3,
|
|
"traversal_regret_fallback_opened_colors_count_1": 2,
|
|
"traversal_regret_fallback_opened_colors_count_0": 1,
|
|
"advantage_samples_returned": 1502,
|
|
"strategy_samples_returned": 1502
|
|
}
|
|
]
|
|
},
|
|
"encode_legal": {
|
|
"median_us": 3.156019374728203,
|
|
"p95_us": 3.877998096868396,
|
|
"mean_legal_actions": 6.955078125,
|
|
"checksum": 7122.0
|
|
},
|
|
"push_pop": {
|
|
"median_us": 0.16100239008665085,
|
|
"p95_us": 0.2500019036233425,
|
|
"checksum": 13048
|
|
},
|
|
"policy_boundary_bs1": {
|
|
"median_us": 181.29700038116425,
|
|
"p95_us": 194.76699526421726,
|
|
"checksum": 998.0805744677782
|
|
},
|
|
"torch_forward": {
|
|
"1": {
|
|
"median_us_per_call": 141.6669983882457,
|
|
"p95_us_per_call": 156.93597379140556,
|
|
"checksum": 850.3550980091095
|
|
},
|
|
"4": {
|
|
"median_us_per_call": 34.81799649307504,
|
|
"p95_us_per_call": 37.939003959763795,
|
|
"checksum": 850.355094909668
|
|
},
|
|
"8": {
|
|
"median_us_per_call": 19.62381247722078,
|
|
"p95_us_per_call": 21.306375856511295,
|
|
"checksum": 850.3550970554352
|
|
},
|
|
"64": {
|
|
"median_us_per_call": 2.5487893253739458,
|
|
"p95_us_per_call": 2.75081220024731,
|
|
"checksum": 850.3550720214844
|
|
},
|
|
"256": {
|
|
"median_us_per_call": 0.6167675792312366,
|
|
"p95_us_per_call": 0.7250391718116589,
|
|
"checksum": 850.3551635742188
|
|
}
|
|
}
|
|
} |