[Doc] cookbook: Refresh SemiAnalysis AgentX launches with five open InferenceX PRs (#45749)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Po-Han Huang (NVIDIA)
2026-10-22 14:15:19 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent a6a6beffac
commit 12370da115
7 changed files with 1689 additions and 1165 deletions
File diff suppressed because it is too large Load Diff
@@ -1,52 +1,37 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed.
// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed.
export const agentx = {
"model": "Qwen3.8-Flash-Next",
"generatedOn": "2026-10-20",
"generatedOn": "2026-10-21",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
"main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc",
"main": "746150a2786f5cfe6fc474220d4a3335365f14c4",
"prs": {
"4344": {
"4389": {
"run": null,
"sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64",
"status": "in-progress"
},
"4342": {
"run": null,
"sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa",
"status": "in-progress"
},
"4341": {
"run": null,
"sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9",
"status": "in-progress"
},
"4343": {
"run": null,
"sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27",
"status": "in-progress"
},
"4350": {
"run": null,
"sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70",
"sha": "e390de596bf5bae7a561308095ad44cf88b74f51",
"status": "in-progress"
},
"4346": {
"run": null,
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794",
"sha": "566c6e105131c0cbfd4b668b51cbc8a861336397",
"status": "in-progress"
},
"3930": {
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600",
"sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157",
"status": "verified-pr"
},
"4333": {
"run": null,
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
"sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
"status": "verified-pr"
},
"4303": {
"run": null,
"sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9",
"status": "in-progress"
},
"3930": {
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792",
"sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4",
"status": "verified-pr"
}
}
},
@@ -79,7 +64,16 @@ export const agentx = {
"id": "nvidia/Qwen3.8-Flash-Next-NVFP4"
}
],
"envs": [],
"envs": [
{
"NCCL_CUMEM_ENABLE": "1",
"NCCL_NVLS_ENABLE": "1",
"SGLANG_ENABLE_FLASHINFER_GEMM": "true",
"SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
"SGL_ENABLE_JIT_DEEPGEMM": "false",
"TORCH_CUDA_ARCH_LIST": "10.0"
}
],
"kvMooncake": {
"donor": "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg",
"donorHardware": "gb300",
@@ -366,45 +360,45 @@ export const agentx = {
"detail": "MTP 3-1-4",
"gpus": 1,
"configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp",
"image": "lmsysorg/sglang:qwen38flashnext",
"image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469",
"imageStatus": "ok",
"imageCommit": null,
"submitted": "sglang",
"setup": [],
"disclosures": ["synthetic-acceptance"],
"kv": {
"available": ["none"],
"available": ["none", "mooncake"],
"unavailable": {
"hicache": "no verified HiCache settings for this model yet",
"mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker"
"hicache": "no verified HiCache settings for this model yet"
}
},
"prov": {
"sha": "7caed5a556ee1ae97f072328fb2c1befa338caa4",
"recipe": "benchmarks/single_node/agentic/qwen3.8next_fp4_b300_sglang_mtp.sh",
"run": "33039186365/attempts/1",
"pr": "2758",
"sha": "3c0d91ceb2334554aef03a8ddfec7e0377a3f274",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml",
"run": "38635046800/attempts/1",
"pr": "4069",
"status": "verified",
"date": "2026-08-27"
"date": "2026-10-21"
},
"points": [
{
"concs": [1],
"item": 0,
"override": "override_c1",
"goldenAL": "2.32",
"kv": "none",
"job": "33039186365/job/98408899989"
"kv": "none"
},
{
"concs": [4],
"item": 0,
"override": "override_c4",
"goldenAL": "2.32",
"kv": "none",
"patch": {
"sglang": [
{
"args": {
"--cuda-graph-max-bs": ["4"],
"--cuda-graph-max-bs-decode": ["4"],
"--max-running-requests": ["8"]
}
}
@@ -414,24 +408,24 @@ export const agentx = {
null,
{
"args": {
"--cuda-graph-max-bs": ["4"],
"--cuda-graph-max-bs-decode": ["4"],
"--max-running-requests": ["8"]
}
}
]
},
"job": "33039186365/job/98408899982"
}
},
{
"concs": [8],
"item": 0,
"override": "override_c8",
"goldenAL": "2.32",
"kv": "none",
"patch": {
"sglang": [
{
"args": {
"--cuda-graph-max-bs": ["8"],
"--cuda-graph-max-bs-decode": ["8"],
"--max-running-requests": ["16"]
}
}
@@ -441,24 +435,24 @@ export const agentx = {
null,
{
"args": {
"--cuda-graph-max-bs": ["8"],
"--cuda-graph-max-bs-decode": ["8"],
"--max-running-requests": ["16"]
}
}
]
},
"job": "33039186365/job/98408899939"
}
},
{
"concs": [12],
"item": 0,
"override": "override_c12",
"goldenAL": "2.32",
"kv": "none",
"patch": {
"sglang": [
{
"args": {
"--cuda-graph-max-bs": ["12"],
"--cuda-graph-max-bs-decode": ["12"],
"--max-running-requests": ["24"]
}
}
@@ -468,24 +462,24 @@ export const agentx = {
null,
{
"args": {
"--cuda-graph-max-bs": ["12"],
"--cuda-graph-max-bs-decode": ["12"],
"--max-running-requests": ["24"]
}
}
]
},
"job": "33039186365/job/98408899965"
}
},
{
"concs": [16],
"item": 0,
"override": "override_c16",
"goldenAL": "2.32",
"kv": "none",
"patch": {
"sglang": [
{
"args": {
"--cuda-graph-max-bs": ["16"],
"--cuda-graph-max-bs-decode": ["16"],
"--max-running-requests": ["32"]
}
}
@@ -495,13 +489,12 @@ export const agentx = {
null,
{
"args": {
"--cuda-graph-max-bs": ["16"],
"--cuda-graph-max-bs-decode": ["16"],
"--max-running-requests": ["32"]
}
}
]
},
"job": "33039186365/job/98408899999"
}
}
],
"routers": {
@@ -515,26 +508,19 @@ export const agentx = {
"workers": 1,
"nodes_per_worker": 1,
"cmd": "python3 -m sglang.launch_server",
"env": {
"NCCL_NVLS_ENABLE": "1",
"SGLANG_ENABLE_FLASHINFER_GEMM": "true",
"SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
"SGL_ENABLE_JIT_DEEPGEMM": "false",
"TORCH_CUDA_ARCH_LIST": "10.0"
},
"args": [
["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
["--host", "0.0.0.0"],
["--port", "30000"],
["--cuda-graph-max-bs", "1"],
["--dp", "1"],
["--ep-size", "1"],
["--cuda-graph-max-bs-decode", "1"],
["--data-parallel-size", "1"],
["--expert-parallel-size", "1"],
["--linear-attn-decode-backend", "flashinfer"],
["--linear-attn-prefill-backend", "flashinfer"],
["--mamba-ssm-dtype", "bfloat16"],
["--max-running-requests", "2"],
["--mem-fraction-static", "0.80"],
["--mem-fraction-static", "0.8"],
["--reasoning-parser", "auto"],
["--scheduler-recv-interval", "10"],
["--speculative-algorithm", "NEXTN"],
@@ -542,10 +528,11 @@ export const agentx = {
["--speculative-num-draft-tokens", "4"],
["--speculative-num-steps", "3"],
["--stream-interval", "50"],
["--tensor-parallel-size", "1"],
["--tokenizer-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
["--tp", "1"],
["--trust-remote-code"]
]
],
"envRef": 0
}
]
},
@@ -728,14 +715,6 @@ export const agentx = {
"workers": 1,
"nodes_per_worker": 1,
"cmd": "python3 -m sglang.launch_server",
"env": {
"NCCL_CUMEM_ENABLE": "1",
"NCCL_NVLS_ENABLE": "1",
"SGLANG_ENABLE_FLASHINFER_GEMM": "true",
"SGLANG_TIMEOUT_KEEP_ALIVE": "1800",
"SGL_ENABLE_JIT_DEEPGEMM": "false",
"TORCH_CUDA_ARCH_LIST": "10.0"
},
"args": [
["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"],
@@ -760,7 +739,8 @@ export const agentx = {
["--tokenizer-worker-num", "6"],
["--tp", "1"],
["--trust-remote-code"]
]
],
"envRef": 0
}
]
},
@@ -1,52 +1,37 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed.
// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed.
export const agentx = {
"model": "DeepSeek-V4",
"generatedOn": "2026-10-20",
"generatedOn": "2026-10-21",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
"main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc",
"main": "746150a2786f5cfe6fc474220d4a3335365f14c4",
"prs": {
"4344": {
"4389": {
"run": null,
"sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64",
"status": "in-progress"
},
"4342": {
"run": null,
"sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa",
"status": "in-progress"
},
"4341": {
"run": null,
"sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9",
"status": "in-progress"
},
"4343": {
"run": null,
"sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27",
"status": "in-progress"
},
"4350": {
"run": null,
"sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70",
"sha": "e390de596bf5bae7a561308095ad44cf88b74f51",
"status": "in-progress"
},
"4346": {
"run": null,
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794",
"sha": "566c6e105131c0cbfd4b668b51cbc8a861336397",
"status": "in-progress"
},
"3930": {
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600",
"sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157",
"status": "verified-pr"
},
"4333": {
"run": null,
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
"sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
"status": "verified-pr"
},
"4303": {
"run": null,
"sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9",
"status": "in-progress"
},
"3930": {
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792",
"sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4",
"status": "verified-pr"
}
}
},
@@ -1452,9 +1437,9 @@ export const agentx = {
"detail": "DSpark 6",
"gpus": 8,
"configKey": "dsv4-fp4-gb200-dynamo-sglang-agentic-agg",
"image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9",
"imageStatus": "pruned",
"imageCommit": "c9a8fba9",
"image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469",
"imageStatus": "ok",
"imageCommit": null,
"submitted": "dynamo",
"setup": [
{
@@ -1480,23 +1465,25 @@ export const agentx = {
}
},
"prov": {
"sha": "d9185455e166dfd034dcdfaf9608bd8335618b54",
"run": "37656460117/attempts/1",
"pr": "3701",
"sha": "afe7fff22655671661b5673722147097876f72c3",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-variants.yaml",
"run": "38953283564/attempts/1",
"pr": "4196",
"status": "verified",
"date": "2026-10-12"
"date": "2026-10-21"
},
"points": [
{
"concs": [1],
"item": 0,
"override": "override_tp8_c1",
"goldenAL": "3.77",
"kv": "hicache",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml"
"kv": "hicache"
},
{
"concs": [4],
"item": 1,
"override": "override_tp8_c4",
"goldenAL": "3.77",
"kv": "hicache",
"patch": {
@@ -1520,8 +1507,7 @@ export const agentx = {
},
null
]
},
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml"
}
}
],
"routers": {
@@ -2520,10 +2506,9 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
["--hicache-ratio", "3"],
["--hicache-write-policy", "write_back"]
["--hicache-ratio", "2.75"]
],
"donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp",
"donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg",
"donorHardware": "b300"
}
},
@@ -2800,10 +2785,9 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
["--hicache-ratio", "3"],
["--hicache-write-policy", "write_back"]
["--hicache-ratio", "2.75"]
],
"donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp",
"donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg",
"donorHardware": "b300"
}
},
@@ -3301,18 +3285,19 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
["--hicache-ratio", "3"],
["--hicache-write-policy", "write_back"]
["--hicache-ratio", "2.75"]
],
"donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp",
"donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg",
"donorHardware": "b300"
}
},
"prov": {
"sha": "566c6e105131c0cbfd4b668b51cbc8a861336397",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp8-c1-mtp.yaml",
"run": "39534932794",
"pr": "4346",
"status": "in-progress"
"comment": "6331655172",
"status": "verified-pr"
},
"points": [
{
@@ -3455,10 +3440,9 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
["--hicache-ratio", "3"],
["--hicache-write-policy", "write_back"]
["--hicache-ratio", "2.75"]
],
"donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp",
"donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg",
"donorHardware": "b300"
}
},
@@ -3696,17 +3680,18 @@ export const agentx = {
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
["--hicache-ratio", "3"],
["--hicache-write-policy", "write_back"]
["--hicache-ratio", "2.75"]
],
"donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp",
"donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg",
"donorHardware": "b300"
}
},
"prov": {
"sha": "566c6e105131c0cbfd4b668b51cbc8a861336397",
"run": "39534932794",
"pr": "4346",
"status": "in-progress"
"comment": "6331655172",
"status": "verified-pr"
},
"points": [
{
@@ -4174,18 +4159,19 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
["--hicache-ratio", "3"],
["--hicache-write-policy", "write_back"]
],
"donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp",
"donor": "dsv4-fp4-b300-dynamo-sglang-agentic-disagg",
"donorHardware": "b300"
}
},
"prov": {
"sha": "566c6e105131c0cbfd4b668b51cbc8a861336397",
"run": "39534932794",
"pr": "4346",
"status": "in-progress"
"comment": "6331655172",
"status": "verified-pr"
},
"points": [
{
@@ -4446,19 +4432,20 @@ export const agentx = {
"args": [
["--enable-hierarchical-cache"],
["--hicache-io-backend", "direct"],
["--hicache-mem-layout", "page_first_direct"],
["--hicache-ratio", "3"],
["--hicache-write-policy", "write_back"]
],
"donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp",
"donor": "dsv4-fp4-b300-dynamo-sglang-agentic-disagg",
"donorHardware": "b300"
}
},
"prov": {
"sha": "566c6e105131c0cbfd4b668b51cbc8a861336397",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p2d-dep8-dep8-c64-mtp-kvoffload.yaml",
"run": "39534932794",
"pr": "4346",
"status": "in-progress"
"comment": "6331655172",
"status": "verified-pr"
},
"points": [
{
@@ -4603,8 +4590,10 @@ export const agentx = {
"prov": {
"sha": "566c6e105131c0cbfd4b668b51cbc8a861336397",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-dep8-c384-mtp-kvoffload.yaml",
"run": "39534932794",
"pr": "4346",
"status": "in-progress"
"comment": "6331655172",
"status": "verified-pr"
},
"points": [
{
@@ -4737,9 +4726,9 @@ export const agentx = {
"detail": "DSpark 6 · dsv4-fp4-b200-dynamo-sglang-agentic-agg",
"gpus": 8,
"configKey": "dsv4-fp4-b200-dynamo-sglang-agentic-agg",
"image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9",
"imageStatus": "pruned",
"imageCommit": "c9a8fba9",
"image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae",
"imageStatus": "ok",
"imageCommit": null,
"submitted": "dynamo",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -4759,23 +4748,25 @@ export const agentx = {
}
},
"prov": {
"sha": "1d0716bb364d49a450386c867d632f7409ff7ced",
"run": "35345548475/attempts/1",
"pr": "3257",
"sha": "e82118bb200aad9275f1d821b4265f251cadd7d2",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml",
"run": "38960572895/attempts/1",
"pr": "4203",
"status": "verified",
"date": "2026-09-22"
"date": "2026-10-21"
},
"points": [
{
"concs": [1],
"item": 0,
"override": "override_c1",
"goldenAL": "3.77",
"kv": "hicache",
"recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml"
"kv": "hicache"
},
{
"concs": [4],
"item": 1,
"override": "override_c4",
"goldenAL": "3.77",
"kv": "hicache",
"patch": {
@@ -4799,12 +4790,12 @@ export const agentx = {
},
null
]
},
"recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml"
}
},
{
"concs": [8],
"item": 2,
"override": "override_c8",
"goldenAL": "3.77",
"kv": "hicache",
"patch": {
@@ -4828,8 +4819,7 @@ export const agentx = {
},
null
]
},
"recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml"
}
}
],
"routers": {
@@ -4875,6 +4865,23 @@ export const agentx = {
"workers": 1,
"nodes_per_worker": 1,
"cmd": "python3 -m dynamo.sglang",
"env": {
"DYN_REQUEST_PLANE": "tcp",
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379",
"NATS_SERVER": "nats://$NATS_IP:4222",
"NCCL_CUMEM_ENABLE": "1",
"SGLANG_DEFAULT_THINKING": "1",
"SGLANG_DSV4_REASONING_EFFORT": "high",
"SGLANG_ENABLE_PREFILL_WAR_READ_DONE": "1",
"SGLANG_JIT_DEEPGEMM_PRECOMPILE": "1",
"SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1",
"SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2": "1",
"SGLANG_OPT_USE_JIT_INDEXER_METADATA": "1",
"SGLANG_OPT_USE_JIT_NORM": "1",
"SGLANG_OPT_USE_ONLINE_COMPRESS": "0",
"SGLANG_OPT_USE_TOPK_V2": "True",
"SGLANG_RAGGED_VERIFY_MODE": "static"
},
"args": [
["--model-path", "deepseek-ai/DeepSeek-V4-Pro-0813"],
["--served-model-name", "deepseek-ai/DeepSeek-V4-Pro-0813"],
@@ -4909,8 +4916,7 @@ export const agentx = {
["--trust-remote-code"],
["--watchdog-timeout", "1000000"],
["--weight-loader-prefetch-checkpoints"]
],
"envRef": 1
]
}
]
},
@@ -5573,9 +5579,9 @@ export const agentx = {
"detail": "DSpark 6",
"gpus": 16,
"configKey": "dsv4-fp4-b200-dynamo-sglang-agentic-disagg",
"image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9",
"imageStatus": "pruned",
"imageCommit": "c9a8fba9",
"image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469",
"imageStatus": "ok",
"imageCommit": null,
"submitted": "dynamo",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -5595,23 +5601,25 @@ export const agentx = {
}
},
"prov": {
"sha": "1d0716bb364d49a450386c867d632f7409ff7ced",
"run": "35345548475/attempts/1",
"pr": "3257",
"sha": "8b1ac122ff6b7e6583af377f190fec7321c22b90",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml",
"run": "38925477096",
"pr": "4176",
"status": "verified",
"date": "2026-09-22"
"date": "2026-10-21"
},
"points": [
{
"concs": [64],
"item": 0,
"override": "override_1p1d_c64",
"goldenAL": "3.77",
"kv": "hicache",
"recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml"
"kv": "hicache"
},
{
"concs": [128],
"item": 1,
"override": "override_1p1d_c128",
"goldenAL": "3.77",
"kv": "hicache",
"patch": {
@@ -5636,8 +5644,7 @@ export const agentx = {
},
null
]
},
"recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml"
}
}
],
"routers": {
@@ -5712,7 +5719,6 @@ export const agentx = {
"SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1",
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "9216",
"SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW": "1",
"SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1",
"SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1",
"SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2": "1",
"SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE": "1",
@@ -5787,7 +5793,6 @@ export const agentx = {
"SGLANG_OPT_DEEPGEMM_MEGA_MOE": "1",
"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096",
"SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW": "1",
"SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1",
"SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1",
"SGLANG_OPT_USE_ONLINE_COMPRESS": "0",
"SGLANG_RAGGED_VERIFY_MODE": "static",
@@ -5893,9 +5898,9 @@ export const agentx = {
"detail": "DSpark 6",
"gpus": 24,
"configKey": "dsv4-fp4-b200-dynamo-sglang-agentic-disagg",
"image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9",
"imageStatus": "pruned",
"imageCommit": "c9a8fba9",
"image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469",
"imageStatus": "ok",
"imageCommit": null,
"submitted": "dynamo",
"setup": [],
"disclosures": ["synthetic-acceptance"],
@@ -5915,17 +5920,18 @@ export const agentx = {
}
},
"prov": {
"sha": "1d0716bb364d49a450386c867d632f7409ff7ced",
"recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml",
"run": "35345548475/attempts/1",
"pr": "3257",
"sha": "8b1ac122ff6b7e6583af377f190fec7321c22b90",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml",
"run": "38925477096",
"pr": "4176",
"status": "verified",
"date": "2026-09-22"
"date": "2026-10-21"
},
"points": [
{
"concs": [256],
"item": 2,
"override": "override_2p1d_c256",
"goldenAL": "3.77",
"kv": "hicache"
}
File diff suppressed because it is too large Load Diff
@@ -1,52 +1,37 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed.
// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed.
export const agentx = {
"model": "Muse Spark Open",
"generatedOn": "2026-10-20",
"generatedOn": "2026-10-21",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
"main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc",
"main": "746150a2786f5cfe6fc474220d4a3335365f14c4",
"prs": {
"4344": {
"4389": {
"run": null,
"sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64",
"status": "in-progress"
},
"4342": {
"run": null,
"sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa",
"status": "in-progress"
},
"4341": {
"run": null,
"sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9",
"status": "in-progress"
},
"4343": {
"run": null,
"sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27",
"status": "in-progress"
},
"4350": {
"run": null,
"sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70",
"sha": "e390de596bf5bae7a561308095ad44cf88b74f51",
"status": "in-progress"
},
"4346": {
"run": null,
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794",
"sha": "566c6e105131c0cbfd4b668b51cbc8a861336397",
"status": "in-progress"
},
"3930": {
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600",
"sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157",
"status": "verified-pr"
},
"4333": {
"run": null,
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
"sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
"status": "verified-pr"
},
"4303": {
"run": null,
"sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9",
"status": "in-progress"
},
"3930": {
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792",
"sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4",
"status": "verified-pr"
}
}
},
@@ -173,18 +158,20 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-size", "120"]
["--hicache-size", "120"],
["--hicache-write-policy", "write_through"]
],
"donor": "musespark-fp4-gb300-sglang-agentic-dflash",
"donorHardware": "gb300",
"donorRef": "main"
"donorHardware": "gb300"
}
},
"prov": {
"sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb300-fp4-mtp/agentic.yaml",
"run": "39408192975/attempts/1",
"pr": "4342",
"status": "in-progress"
"status": "verified",
"date": "2026-10-22"
},
"points": [
{
@@ -672,18 +659,20 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-size", "120"]
["--hicache-size", "120"],
["--hicache-write-policy", "write_through"]
],
"donor": "musespark-fp4-gb300-sglang-agentic-dflash",
"donorHardware": "gb300",
"donorRef": "main"
"donorHardware": "gb300"
}
},
"prov": {
"sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml",
"run": "39427144732/attempts/4",
"pr": "4344",
"status": "in-progress"
"status": "verified",
"date": "2026-10-22"
},
"points": [
{
@@ -912,18 +901,20 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-size", "120"]
["--hicache-size", "120"],
["--hicache-write-policy", "write_through"]
],
"donor": "musespark-fp4-gb300-sglang-agentic-dflash",
"donorHardware": "gb300",
"donorRef": "main"
"donorHardware": "gb300"
}
},
"prov": {
"sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64",
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml",
"run": "39427144732/attempts/4",
"pr": "4344",
"status": "in-progress"
"status": "verified",
"date": "2026-10-22"
},
"points": [
{
@@ -1225,15 +1216,16 @@ export const agentx = {
["--hicache-size", "120"]
],
"donor": "musespark-fp4-gb200-sglang-agentic-dflash",
"donorHardware": "gb200",
"donorRef": "main"
"donorHardware": "gb200"
}
},
"prov": {
"sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml",
"run": "39404635036/attempts/1",
"pr": "4343",
"status": "in-progress"
"status": "verified",
"date": "2026-10-22"
},
"points": [
{
@@ -1835,18 +1827,19 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-size", "216"]
["--hicache-size", "170"]
],
"donor": "musespark-fp4-b300-sglang-agentic-dflash",
"donorHardware": "b300",
"donorRef": "main"
"donorHardware": "b300"
}
},
"prov": {
"sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml",
"run": "39385088584/attempts/1",
"pr": "4341",
"status": "in-progress"
"status": "verified",
"date": "2026-10-22"
},
"points": [
{
@@ -2070,18 +2063,19 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-size", "216"]
["--hicache-size", "170"]
],
"donor": "musespark-fp4-b300-sglang-agentic-dflash",
"donorHardware": "b300",
"donorRef": "main"
"donorHardware": "b300"
}
},
"prov": {
"sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml",
"run": "39385088584/attempts/1",
"pr": "4341",
"status": "in-progress"
"status": "verified",
"date": "2026-10-22"
},
"points": [
{
@@ -2495,18 +2489,20 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-size", "160"]
["--hicache-size", "120"],
["--hicache-write-policy", "write_through"]
],
"donor": "musespark-fp4-b200-sglang-agentic-dflash",
"donorHardware": "b200",
"donorRef": "main"
"donorHardware": "b200"
}
},
"prov": {
"sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml",
"run": "39449248229/attempts/1",
"pr": "4350",
"status": "in-progress"
"status": "verified",
"date": "2026-10-22"
},
"points": [
{
@@ -2867,18 +2863,20 @@ export const agentx = {
"hicache": {
"args": [
["--enable-hierarchical-cache"],
["--hicache-size", "160"]
["--hicache-size", "120"],
["--hicache-write-policy", "write_through"]
],
"donor": "musespark-fp4-b200-sglang-agentic-dflash",
"donorHardware": "b200",
"donorRef": "main"
"donorHardware": "b200"
}
},
"prov": {
"sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml",
"run": "39449248229/attempts/1",
"pr": "4350",
"status": "in-progress"
"status": "verified",
"date": "2026-10-22"
},
"points": [
{
File diff suppressed because it is too large Load Diff
+156 -41
View File
@@ -1,52 +1,37 @@
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed.
// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed.
export const agentx = {
"model": "GLM-5.3",
"generatedOn": "2026-10-20",
"generatedOn": "2026-10-21",
"source": {
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
"main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc",
"main": "746150a2786f5cfe6fc474220d4a3335365f14c4",
"prs": {
"4344": {
"4389": {
"run": null,
"sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64",
"status": "in-progress"
},
"4342": {
"run": null,
"sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa",
"status": "in-progress"
},
"4341": {
"run": null,
"sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9",
"status": "in-progress"
},
"4343": {
"run": null,
"sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27",
"status": "in-progress"
},
"4350": {
"run": null,
"sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70",
"sha": "e390de596bf5bae7a561308095ad44cf88b74f51",
"status": "in-progress"
},
"4346": {
"run": null,
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794",
"sha": "566c6e105131c0cbfd4b668b51cbc8a861336397",
"status": "in-progress"
},
"3930": {
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600",
"sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157",
"status": "verified-pr"
},
"4333": {
"run": null,
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627",
"sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf",
"status": "verified-pr"
},
"4303": {
"run": null,
"sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9",
"status": "in-progress"
},
"3930": {
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792",
"sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4",
"status": "verified-pr"
}
}
},
@@ -155,41 +140,155 @@ export const agentx = {
}
},
"prov": {
"sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157",
"sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4",
"recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3/sglang/b300-fp4-mtp/agentic.yaml",
"run": "39462624600",
"run": "39551262792",
"pr": "3930",
"comment": "6300890848",
"comment": "6330763860",
"status": "verified-pr"
},
"points": [
{
"concs": [1],
"item": 0,
"override": "override_c1",
"goldenAL": "2.99",
"kv": "none"
},
{
"concs": [4],
"item": 0,
"override": "override_c4",
"goldenAL": "2.99",
"kv": "none",
"patch": {
"sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["8"],
"--max-running-requests": ["8"]
}
}
],
"dynamo": [
null,
null,
{
"args": {
"--cuda-graph-max-bs-decode": ["8"],
"--max-running-requests": ["8"]
}
}
]
}
},
{
"concs": [8],
"item": 0,
"override": "override_c8",
"goldenAL": "2.99",
"kv": "none"
"kv": "none",
"patch": {
"sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--max-running-requests": ["256"]
}
}
],
"dynamo": [
null,
null,
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--max-running-requests": ["256"]
}
}
]
}
},
{
"concs": [10],
"item": 0,
"override": "override_c10",
"goldenAL": "2.99",
"kv": "none"
"kv": "none",
"patch": {
"sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--max-running-requests": ["256"]
}
}
],
"dynamo": [
null,
null,
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--max-running-requests": ["256"]
}
}
]
}
},
{
"concs": [13],
"item": 0,
"override": "override_c13",
"goldenAL": "2.99",
"kv": "none"
"kv": "none",
"patch": {
"sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--max-running-requests": ["256"]
}
}
],
"dynamo": [
null,
null,
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--max-running-requests": ["256"]
}
}
]
}
},
{
"concs": [14],
"item": 0,
"override": "override_c14",
"goldenAL": "2.99",
"kv": "none"
"kv": "none",
"patch": {
"sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--max-running-requests": ["256"]
}
}
],
"dynamo": [
null,
null,
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--max-running-requests": ["256"]
}
}
]
}
},
{
"concs": [22],
@@ -201,11 +300,13 @@ export const agentx = {
"sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--enable-hierarchical-cache": [],
"--hicache-io-backend": ["direct"],
"--hicache-mem-layout": ["page_first_direct"],
"--hicache-size": ["300"],
"--hicache-write-policy": ["write_back"],
"--max-running-requests": ["256"],
"--enable-torch-symm-mem": null
},
"env": {
@@ -218,11 +319,13 @@ export const agentx = {
null,
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--enable-hierarchical-cache": [],
"--hicache-io-backend": ["direct"],
"--hicache-mem-layout": ["page_first_direct"],
"--hicache-size": ["300"],
"--hicache-write-policy": ["write_back"],
"--max-running-requests": ["256"],
"--enable-torch-symm-mem": null
},
"env": {
@@ -242,11 +345,13 @@ export const agentx = {
"sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--enable-hierarchical-cache": [],
"--hicache-io-backend": ["direct"],
"--hicache-mem-layout": ["page_first_direct"],
"--hicache-size": ["300"],
"--hicache-write-policy": ["write_back"],
"--max-running-requests": ["256"],
"--enable-torch-symm-mem": null
},
"env": {
@@ -259,11 +364,13 @@ export const agentx = {
null,
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--enable-hierarchical-cache": [],
"--hicache-io-backend": ["direct"],
"--hicache-mem-layout": ["page_first_direct"],
"--hicache-size": ["300"],
"--hicache-write-policy": ["write_back"],
"--max-running-requests": ["256"],
"--enable-torch-symm-mem": null
},
"env": {
@@ -283,11 +390,13 @@ export const agentx = {
"sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--enable-hierarchical-cache": [],
"--hicache-io-backend": ["direct"],
"--hicache-mem-layout": ["page_first_direct"],
"--hicache-size": ["300"],
"--hicache-write-policy": ["write_back"],
"--max-running-requests": ["256"],
"--enable-torch-symm-mem": null
},
"env": {
@@ -300,11 +409,13 @@ export const agentx = {
null,
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--enable-hierarchical-cache": [],
"--hicache-io-backend": ["direct"],
"--hicache-mem-layout": ["page_first_direct"],
"--hicache-size": ["300"],
"--hicache-write-policy": ["write_back"],
"--max-running-requests": ["256"],
"--enable-torch-symm-mem": null
},
"env": {
@@ -324,11 +435,13 @@ export const agentx = {
"sglang": [
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--enable-hierarchical-cache": [],
"--hicache-io-backend": ["direct"],
"--hicache-mem-layout": ["page_first_direct"],
"--hicache-size": ["300"],
"--hicache-write-policy": ["write_back"],
"--max-running-requests": ["256"],
"--enable-torch-symm-mem": null
},
"env": {
@@ -341,11 +454,13 @@ export const agentx = {
null,
{
"args": {
"--cuda-graph-max-bs-decode": ["64"],
"--enable-hierarchical-cache": [],
"--hicache-io-backend": ["direct"],
"--hicache-mem-layout": ["page_first_direct"],
"--hicache-size": ["300"],
"--hicache-write-policy": ["write_back"],
"--max-running-requests": ["256"],
"--enable-torch-symm-mem": null
},
"env": {
@@ -379,11 +494,11 @@ export const agentx = {
["--port", "30000"],
["--chunked-prefill-size", "8192"],
["--context-length", "1048576"],
["--cuda-graph-max-bs-decode", "64"],
["--cuda-graph-max-bs-decode", "2"],
["--enable-torch-symm-mem"],
["--expert-parallel-size", "1"],
["--max-prefill-tokens", "8192"],
["--max-running-requests", "256"],
["--max-running-requests", "2"],
["--mem-fraction-static", "0.86"],
["--model-loader-extra-config", "{\"enable_multithread_load\":true}"],
["--quantization", "modelopt_fp4"],