From 12370da1157d9bfe4e41632dabd0420b2d4cc774 Mon Sep 17 00:00:00 2001 From: "Po-Han Huang (NVIDIA)" <53919306+nvpohanh@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:15:19 +0800 Subject: [PATCH] [Doc] cookbook: Refresh SemiAnalysis AgentX launches with five open InferenceX PRs (#45749) Co-authored-by: Claude Opus 5.5 --- docs/src/snippets/agentx/Qwen/qwen3.5.jsx | 760 +++++++---------- .../agentx/Qwen/qwen3.8-flash-next.jsx | 144 ++-- .../agentx/deepseek-ai/deepseek-v4.jsx | 212 ++--- .../agentx/deepseek-ai/deepseek-v4_1.jsx | 649 +++++++++++---- .../agentx/meta-models/muse-spark-open.jsx | 124 ++- docs/src/snippets/agentx/zai-org/glm-5.2.jsx | 768 ++++++++++++------ docs/src/snippets/agentx/zai-org/glm-5.3.jsx | 197 ++++- 7 files changed, 1689 insertions(+), 1165 deletions(-) diff --git a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx index 8318064e91..9891b2f832 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.5.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.5.jsx @@ -1,52 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed. export const agentx = { "model": "Qwen3.5", - "generatedOn": "2026-10-20", + "generatedOn": "2026-10-21", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc", + "main": "746150a2786f5cfe6fc474220d4a3335365f14c4", "prs": { - "4344": { + "4389": { "run": null, - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", - "status": "in-progress" - }, - "4342": { - "run": null, - "sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa", - "status": "in-progress" - }, - "4341": { - "run": null, - "sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9", - "status": "in-progress" - }, - "4343": { - "run": null, - "sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27", - "status": "in-progress" - }, - "4350": { - "run": null, - "sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70", + "sha": "e390de596bf5bae7a561308095ad44cf88b74f51", "status": "in-progress" }, "4346": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794", "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", - "status": "in-progress" - }, - "3930": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600", - "sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157", "status": "verified-pr" }, "4333": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "status": "verified-pr" + }, + "4303": { + "run": null, + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", "status": "in-progress" + }, + "3930": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792", + "sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4", + "status": "verified-pr" } } }, @@ -231,7 +216,7 @@ export const agentx = { ["--hicache-size", "32"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", "donorHardware": "gb300" } }, @@ -423,7 +408,7 @@ export const agentx = { ["--hicache-size", "32"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", "donorHardware": "gb300" } }, @@ -540,7 +525,7 @@ export const agentx = { ["--hicache-size", "32"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", "donorHardware": "gb300" } }, @@ -660,7 +645,7 @@ export const agentx = { ["--hicache-size", "32"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", "donorHardware": "gb300" } }, @@ -770,7 +755,7 @@ export const agentx = { ["--hicache-size", "32"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", "donorHardware": "gb300" } }, @@ -887,7 +872,7 @@ export const agentx = { ["--hicache-size", "32"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", "donorHardware": "gb300" } }, @@ -998,13 +983,12 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, @@ -1326,13 +1310,12 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, @@ -1505,13 +1488,12 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, @@ -1660,13 +1642,12 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, @@ -1818,13 +1799,12 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, @@ -1977,13 +1957,12 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, @@ -2137,13 +2116,12 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, @@ -2300,13 +2278,12 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, @@ -2466,15 +2443,17 @@ export const agentx = { ["--hicache-size", "32"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", "donorHardware": "gb300" } }, "prov": { "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "39355392627", "pr": "4333", - "status": "in-progress" + "comment": "6331658889", + "status": "verified-pr" }, "points": [ { @@ -2962,15 +2941,17 @@ export const agentx = { ["--hicache-size", "32"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg", "donorHardware": "gb300" } }, "prov": { "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "39355392627", "pr": "4333", - "status": "in-progress" + "comment": "6331658889", + "status": "verified-pr" }, "points": [ { @@ -3099,21 +3080,22 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, "prov": { "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "39355392627", "pr": "4333", - "status": "in-progress" + "comment": "6331658889", + "status": "verified-pr" }, "points": [ { @@ -3635,21 +3617,22 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, "prov": { "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "39355392627", "pr": "4333", - "status": "in-progress" + "comment": "6331658889", + "status": "verified-pr" }, "points": [ { @@ -3811,21 +3794,22 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, "prov": { "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "39355392627", "pr": "4333", - "status": "in-progress" + "comment": "6331658889", + "status": "verified-pr" }, "points": [ { @@ -4094,21 +4078,22 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, "prov": { "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "39355392627", "pr": "4333", - "status": "in-progress" + "comment": "6331658889", + "status": "verified-pr" }, "points": [ { @@ -4361,21 +4346,22 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-host-memory-mode", "cache"], ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.0"], + ["--hicache-mem-layout", "page_first"], + ["--hicache-ratio", "12"], ["--hicache-write-policy", "write_back"] ], - "donor": "qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve", + "donor": "qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300" } }, "prov": { "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "run": "39355392627", "pr": "4333", - "status": "in-progress" + "comment": "6331658889", + "status": "verified-pr" }, "points": [ { @@ -4541,9 +4527,9 @@ export const agentx = { "detail": "MTP 5-1-6", "gpus": 4, "configKey": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-20260818-c0b6474b", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", "imageStatus": "ok", - "imageCommit": "c0b6474b", + "imageCommit": null, "submitted": "dynamo", "setup": [ { @@ -4552,12 +4538,10 @@ export const agentx = { "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." } ], - "disclosures": [], + "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], @@ -4571,22 +4555,26 @@ export const agentx = { } }, "prov": { - "sha": "cc865fb3945848e9d0627ad3b94df0cface6a6fc", - "run": "34436606041/attempts/2", - "pr": "2945", + "sha": "258f9a6cef995a6437f89af24365cd714672460c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml", + "run": "39145685012/attempts/2", + "pr": "4278", "status": "verified", - "date": "2026-09-11" + "date": "2026-10-22" }, "points": [ { "concs": [1, 4, 8, 12, 20], "item": 0, - "kv": "none", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp.yaml" + "override": "override_tp4", + "goldenAL": "4.41", + "kv": "none" }, { "concs": [2], "item": 1, + "override": "override_tp4_no_symm", + "goldenAL": "4.41", "kv": "none", "patch": { "dynamo": [ @@ -4607,12 +4595,13 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-no-symm.yaml" + } }, { "concs": [24], "item": 2, + "override": "override_tp4_parity", + "goldenAL": "4.41", "kv": "none", "patch": { "dynamo": [ @@ -4623,8 +4612,8 @@ export const agentx = { "args": { "--enable-linear-replayssm-spec": [], "--linear-attn-prefill-backend": ["flashinfer"], - "--linear-replayssm-cache-len": ["8"], - "--mamba-scheduler-strategy": ["extra_buffer_lazy"], + "--linear-replayssm-cache-len": ["16"], + "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], "--mamba-track-interval": ["1048576"] } } @@ -4634,20 +4623,19 @@ export const agentx = { "args": { "--enable-linear-replayssm-spec": [], "--linear-attn-prefill-backend": ["flashinfer"], - "--linear-replayssm-cache-len": ["8"], - "--mamba-scheduler-strategy": ["extra_buffer_lazy"], + "--linear-replayssm-cache-len": ["16"], + "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], "--mamba-track-interval": ["1048576"] } }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp4-mtp-parity.yaml" + } } ], "routers": { "dynamo": { - "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "install": [], "files": ["nats.conf#48a4de48"], "blocks": [ { @@ -4710,7 +4698,7 @@ export const agentx = { ["--allow-auto-truncate"], ["--attention-backend", "trtllm_mha"], ["--chunked-prefill-size", "16384"], - ["--cuda-graph-max-bs", "64"], + ["--cuda-graph-max-bs-decode", "64"], ["--data-parallel-size", "1"], ["--enable-symm-mem"], ["--expert-parallel-size", "1"], @@ -4718,7 +4706,7 @@ export const agentx = { ["--kv-cache-dtype", "fp8_e4m3"], ["--linear-attn-decode-backend", "flashinfer"], ["--mamba-max-states-per-path", "1"], - ["--mamba-scheduler-strategy", "extra_buffer"], + ["--mamba-radix-cache-strategy", "extra_buffer"], ["--mamba-ssm-dtype", "bfloat16"], ["--mamba-track-interval", "8192"], ["--max-mamba-cache-size", "360"], @@ -4783,9 +4771,9 @@ export const agentx = { "detail": "MTP 5-1-6", "gpus": 2, "configKey": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-20260818-c0b6474b", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", "imageStatus": "ok", - "imageCommit": "c0b6474b", + "imageCommit": null, "submitted": "dynamo", "setup": [ { @@ -4794,12 +4782,10 @@ export const agentx = { "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." } ], - "disclosures": [], + "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], @@ -4813,23 +4799,25 @@ export const agentx = { } }, "prov": { - "sha": "cc865fb3945848e9d0627ad3b94df0cface6a6fc", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache.yaml", - "run": "34436606041/attempts/2", - "pr": "2945", + "sha": "258f9a6cef995a6437f89af24365cd714672460c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml", + "run": "39145685012/attempts/2", + "pr": "4278", "status": "verified", - "date": "2026-09-11" + "date": "2026-10-22" }, "points": [ { "concs": [22], "item": 3, + "override": "override_tp2ep2_hicache", + "goldenAL": "4.41", "kv": "hicache" } ], "routers": { "dynamo": { - "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "install": [], "files": ["nats.conf#48a4de48"], "blocks": [ { @@ -4876,7 +4864,7 @@ export const agentx = { "--hicache-ratio": ["2.5"], "--hicache-write-policy": ["write_through_selective"], "--linear-attn-prefill-backend": ["flashinfer"], - "--linear-replayssm-cache-len": ["8"], + "--linear-replayssm-cache-len": ["16"], "--mamba-radix-cache-strategy": ["extra_buffer_lazy"], "--mamba-track-interval": ["1048576"], "--max-mamba-cache-size": ["320"], @@ -4886,8 +4874,7 @@ export const agentx = { "--speculative-moe-runner-backend": ["flashinfer_trtllm"], "--tensor-parallel-size": ["2"], "--watchdog-timeout": ["1000000"], - "--enable-symm-mem": null, - "--mamba-scheduler-strategy": null + "--enable-symm-mem": null } }, "ref": [21, "dynamo", 3] @@ -4935,9 +4922,9 @@ export const agentx = { "detail": "MTP 3-1-4", "gpus": 2, "configKey": "qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-20260818-c0b6474b", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", "imageStatus": "ok", - "imageCommit": "c0b6474b", + "imageCommit": null, "submitted": "dynamo", "setup": [ { @@ -4946,12 +4933,10 @@ export const agentx = { "reason": "SA's GB200 Dynamo+SGLang lane runs srt-slurm's install-torchao.sh in every worker container before launch, so the measured environment had torchao installed." } ], - "disclosures": [], + "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], @@ -4965,22 +4950,26 @@ export const agentx = { } }, "prov": { - "sha": "cc865fb3945848e9d0627ad3b94df0cface6a6fc", - "run": "34436606041/attempts/2", - "pr": "2945", + "sha": "258f9a6cef995a6437f89af24365cd714672460c", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml", + "run": "39145685012/attempts/2", + "pr": "4278", "status": "verified", - "date": "2026-09-11" + "date": "2026-10-22" }, "points": [ { "concs": [32], "item": 4, - "kv": "hicache", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-k3-baseline.yaml" + "override": "override_tp2ep2_hicache_k3_baseline", + "goldenAL": "3.39", + "kv": "hicache" }, { "concs": [40, 48], "item": 5, + "override": "override_tp2ep2_hicache_cap48", + "goldenAL": "3.39", "kv": "hicache", "patch": { "dynamo": [ @@ -4989,7 +4978,7 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["48"], + "--cuda-graph-max-bs-decode": ["48"], "--max-mamba-cache-size": ["192"], "--max-running-requests": ["48"], "--tokenizer-worker-num": null @@ -4999,7 +4988,7 @@ export const agentx = { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["48"], + "--cuda-graph-max-bs-decode": ["48"], "--max-mamba-cache-size": ["192"], "--max-running-requests": ["48"], "--tokenizer-worker-num": null @@ -5007,13 +4996,12 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic/agg-gb200-tp2ep2-mtp-hicache-cap48.yaml" + } } ], "routers": { "dynamo": { - "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.4.2\" \"ai-dynamo==1.4.2\""], + "install": [], "files": ["nats.conf#48a4de48"], "blocks": [ { @@ -5971,9 +5959,9 @@ export const agentx = { "detail": "no spec", "gpus": 8, "configKey": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85", - "imageStatus": "pruned", - "imageCommit": "20518d85", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -5993,30 +5981,32 @@ export const agentx = { } }, "prov": { - "sha": "8eb9a40f3bdb57d9ffaaee56eb86e8aef213a6f5", - "run": "35628368631/attempts/1", - "pr": "3268", + "sha": "b8e528b7a4dbc8b95717a8a55c126925c0b28c0f", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml", + "run": "38997725541/attempts/2", + "pr": "4224", "status": "verified", - "date": "2026-09-21" + "date": "2026-10-21" }, "points": [ { "concs": [4], "item": 0, + "override": "override_tp4_tp4_c4", "goldenAL": "3.39", - "kv": "hicache", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c4-mtp-hicache.yaml" + "kv": "hicache" }, { "concs": [12], "item": 1, + "override": "override_tp4_tp4_c12", "goldenAL": "3.39", - "kv": "hicache", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c12-mtp-hicache.yaml" + "kv": "hicache" }, { "concs": [24], "item": 2, + "override": "override_tp4_tp4_c24", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -6039,8 +6029,7 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4-tp4-colocated-c24-mtp-hicache.yaml" + } } ], "routers": { @@ -6570,9 +6559,9 @@ export const agentx = { "detail": "no spec", "gpus": 4, "configKey": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85", - "imageStatus": "pruned", - "imageCommit": "20518d85", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -6592,23 +6581,25 @@ export const agentx = { } }, "prov": { - "sha": "8eb9a40f3bdb57d9ffaaee56eb86e8aef213a6f5", - "run": "35628368631/attempts/1", - "pr": "3268", + "sha": "b8e528b7a4dbc8b95717a8a55c126925c0b28c0f", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml", + "run": "38997725541/attempts/2", + "pr": "4224", "status": "verified", - "date": "2026-09-21" + "date": "2026-10-21" }, "points": [ { "concs": [32], "item": 4, + "override": "override_tp2ep2_tp2ep2_c32", "goldenAL": "3.39", - "kv": "hicache", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c32-mtp-hicache.yaml" + "kv": "hicache" }, { "concs": [40], "item": 5, + "override": "override_tp2ep2_tp2ep2_c40", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -6631,12 +6622,12 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c40-mtp-hicache.yaml" + } }, { "concs": [44], "item": 6, + "override": "override_tp2ep2_tp2ep2_c44", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -6659,12 +6650,12 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c44-mtp-hicache.yaml" + } }, { "concs": [48], "item": 7, + "override": "override_tp2ep2_tp2ep2_c48", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -6689,12 +6680,12 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c48-mtp-hicache.yaml" + } }, { "concs": [56], "item": 8, + "override": "override_tp2ep2_tp2ep2_c56_replayssm", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -6721,8 +6712,7 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp2ep2-tp2ep2-colocated-c56-replayssm-mtp-hicache.yaml" + } } ], "routers": { @@ -6847,9 +6837,9 @@ export const agentx = { "detail": "no spec", "gpus": 8, "configKey": "qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85", - "imageStatus": "pruned", - "imageCommit": "20518d85", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -6869,17 +6859,18 @@ export const agentx = { } }, "prov": { - "sha": "8eb9a40f3bdb57d9ffaaee56eb86e8aef213a6f5", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-1p1d-tp4ep4-tp4-colocated-c32-mtp-hicache.yaml", - "run": "35628368631/attempts/1", - "pr": "3268", + "sha": "b8e528b7a4dbc8b95717a8a55c126925c0b28c0f", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp8/agentx/disagg-variants.yaml", + "run": "38997725541/attempts/2", + "pr": "4224", "status": "verified", - "date": "2026-09-21" + "date": "2026-10-21" }, "points": [ { "concs": [32], "item": 3, + "override": "override_tp4ep4_tp4_c32", "goldenAL": "3.39", "kv": "hicache" } @@ -8807,9 +8798,9 @@ export const agentx = { "detail": "MTP 3-1-4", "gpus": 4, "configKey": "qwen3.5-fp8-b200-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", - "imageStatus": "pruned", - "imageCommit": "4358a161", + "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", + "imageStatus": "ok", + "imageCommit": null, "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -8829,24 +8820,25 @@ export const agentx = { } }, "prov": { - "sha": "21a63c471574e0cda725d4a61d3837e91405b522", - "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh", - "run": "35032235397/attempts/1", - "pr": "3142", + "sha": "7ee38eaa51fbde61fab6337417d373f82f0e0e67", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml", + "run": "38580932618/attempts/2", + "pr": "4058", "status": "verified", - "date": "2026-09-16" + "date": "2026-10-21" }, "points": [ { "concs": [1], "item": 2, + "override": "override_tp4_c1", "goldenAL": "3.39", - "kv": "none", - "job": "35032235397/job/104593683554" + "kv": "none" }, { "concs": [4], "item": 2, + "override": "override_tp4_c4", "goldenAL": "3.39", "kv": "none", "patch": { @@ -8868,12 +8860,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683369" + } }, { "concs": [8], "item": 2, + "override": "override_tp4_c8", "goldenAL": "3.39", "kv": "none", "patch": { @@ -8895,12 +8887,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684311" + } }, { "concs": [12], "item": 2, + "override": "override_tp4_c12", "goldenAL": "3.39", "kv": "none", "patch": { @@ -8922,12 +8914,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684310" + } }, { "concs": [14], "item": 2, + "override": "override_tp4_c14", "goldenAL": "3.39", "kv": "none", "patch": { @@ -8949,12 +8941,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684304" + } }, { "concs": [16], "item": 3, + "override": "override_tp4_hicache_c16", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -8988,12 +8980,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684012" + } }, { "concs": [18], "item": 3, + "override": "override_tp4_hicache_c18", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9027,12 +9019,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684349" + } }, { "concs": [20], "item": 3, + "override": "override_tp4_hicache_c20", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9066,12 +9058,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683966" + } }, { "concs": [22], "item": 3, + "override": "override_tp4_hicache_c22", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9105,12 +9097,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683751" + } }, { "concs": [24], "item": 3, + "override": "override_tp4_hicache_c24", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9144,12 +9136,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684216" + } }, { "concs": [28], "item": 3, + "override": "override_tp4_hicache_c28", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9183,12 +9175,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683411" + } }, { "concs": [32], "item": 3, + "override": "override_tp4_hicache_c32", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9222,8 +9214,7 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683809" + } } ], "routers": { @@ -9253,14 +9244,14 @@ export const agentx = { ["--attention-backend", "trtllm_mha"], ["--chunked-prefill-size", "16384"], ["--cuda-graph-max-bs-decode", "1"], - ["--dp", "1"], + ["--data-parallel-size", "1"], ["--enable-symm-mem"], - ["--ep-size", "1"], + ["--expert-parallel-size", "1"], ["--kv-cache-dtype", "fp8_e4m3"], ["--mamba-ssm-dtype", "bfloat16"], ["--max-prefill-tokens", "16384"], ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.80"], + ["--mem-fraction-static", "0.8"], ["--moe-runner-backend", "flashinfer_trtllm"], ["--quantization", "fp8"], ["--reasoning-parser", "qwen3"], @@ -9270,10 +9261,9 @@ export const agentx = { ["--speculative-num-draft-tokens", "4"], ["--speculative-num-steps", "3"], ["--stream-interval", "50"], - ["--tokenizer-path", "Qwen/Qwen3.5-397B-A17B-FP8"], + ["--tensor-parallel-size", "4"], ["--tokenizer-worker-num", "6"], ["--tool-call-parser", "qwen3_coder"], - ["--tp", "4"], ["--trust-remote-code"] ] } @@ -9332,9 +9322,9 @@ export const agentx = { "detail": "MTP 3-1-4 · measured with no KV offload", "gpus": 8, "configKey": "qwen3.5-fp8-b200-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", - "imageStatus": "pruned", - "imageCommit": "4358a161", + "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", + "imageStatus": "ok", + "imageCommit": null, "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -9354,24 +9344,25 @@ export const agentx = { } }, "prov": { - "sha": "21a63c471574e0cda725d4a61d3837e91405b522", - "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh", - "run": "35032235397/attempts/1", - "pr": "3142", + "sha": "7ee38eaa51fbde61fab6337417d373f82f0e0e67", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml", + "run": "38580932618/attempts/2", + "pr": "4058", "status": "verified", - "date": "2026-09-16" + "date": "2026-10-21" }, "points": [ { "concs": [1], "item": 0, + "override": "override_tp8_c1", "goldenAL": "3.39", - "kv": "none", - "job": "35032235397/job/104593684082" + "kv": "none" }, { "concs": [4], "item": 0, + "override": "override_tp8_c4", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9393,12 +9384,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684181" + } }, { "concs": [8], "item": 0, + "override": "override_tp8_c8", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9420,12 +9411,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683585" + } }, { "concs": [12], "item": 0, + "override": "override_tp8_c12", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9447,12 +9438,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684113" + } }, { "concs": [16], "item": 0, + "override": "override_tp8_c16", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9474,12 +9465,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683289" + } }, { "concs": [20], "item": 0, + "override": "override_tp8_c20", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9501,12 +9492,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684136" + } }, { "concs": [24], "item": 0, + "override": "override_tp8_c24", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9528,12 +9519,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683532" + } }, { "concs": [28], "item": 0, + "override": "override_tp8_c28", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9555,12 +9546,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683645" + } }, { "concs": [32], "item": 0, + "override": "override_tp8_c32", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9582,12 +9573,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683606" + } }, { "concs": [40], "item": 0, + "override": "override_tp8_c40", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9609,12 +9600,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684158" + } }, { "concs": [48], "item": 0, + "override": "override_tp8_c48", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9636,12 +9627,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684375" + } }, { "concs": [56], "item": 0, + "override": "override_tp8_c56", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9663,12 +9654,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683940" + } }, { "concs": [60], "item": 0, + "override": "override_tp8_c60", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9690,12 +9681,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683635" + } }, { "concs": [62], "item": 0, + "override": "override_tp8_c62", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9717,12 +9708,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684023" + } }, { "concs": [64], "item": 0, + "override": "override_tp8_c64", "goldenAL": "3.39", "kv": "none", "patch": { @@ -9744,8 +9735,7 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684272" + } } ], "routers": { @@ -9756,7 +9746,7 @@ export const agentx = { { "diff": { "args": { - "--tp": ["8"] + "--tensor-parallel-size": ["8"] } }, "ref": [37, "sglang", 0] @@ -9816,9 +9806,9 @@ export const agentx = { "detail": "no spec", "gpus": 8, "configKey": "qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85", - "imageStatus": "pruned", - "imageCommit": "20518d85", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -9838,23 +9828,25 @@ export const agentx = { } }, "prov": { - "sha": "0b75e138be098de0122959460c765860f79b95fd", - "run": "35404276619/attempts/2", - "pr": "3269", + "sha": "4fac0db38d4046dbe1e70621dfb167da74141299", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-variants.yaml", + "run": "39009568068/attempts/1", + "pr": "4235", "status": "verified", - "date": "2026-09-19" + "date": "2026-10-21" }, "points": [ { "concs": [8], "item": 0, + "override": "override_1p1d_p_tp4_d_tp4_c8", "goldenAL": "3.39", - "kv": "hicache", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c8-mtp.yaml" + "kv": "hicache" }, { "concs": [16], "item": 1, + "override": "override_1p1d_p_tp4_d_tp4_c16", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9877,12 +9869,12 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c16-mtp.yaml" + } }, { "concs": [24], "item": 2, + "override": "override_1p1d_p_tp4_d_tp4_c24", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9915,12 +9907,12 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c24-mtp.yaml" + } }, { "concs": [32], "item": 3, + "override": "override_1p1d_p_tp4_d_tp4_c32", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9953,12 +9945,12 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c32-mtp.yaml" + } }, { "concs": [48], "item": 4, + "override": "override_1p1d_p_tp4_d_tp4_c48_write_through", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -9997,12 +9989,12 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c48-write-through-mtp.yaml" + } }, { "concs": [64], "item": 5, + "override": "override_1p1d_p_tp4_d_tp4_c64_write_through", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -10041,8 +10033,7 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8/agentx/disagg-1p1d-p-tp4-d-tp4-hicache-c64-write-through-mtp.yaml" + } } ], "routers": { @@ -10233,9 +10224,9 @@ export const agentx = { "detail": "MTP 3-1-4 · measured with HiCache", "gpus": 8, "configKey": "qwen3.5-fp8-b200-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260914-4358a161", - "imageStatus": "pruned", - "imageCommit": "4358a161", + "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", + "imageStatus": "ok", + "imageCommit": null, "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -10255,24 +10246,25 @@ export const agentx = { } }, "prov": { - "sha": "21a63c471574e0cda725d4a61d3837e91405b522", - "recipe": "benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh", - "run": "35032235397/attempts/1", - "pr": "3142", + "sha": "7ee38eaa51fbde61fab6337417d373f82f0e0e67", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b200-fp8-mtp/agentic.yaml", + "run": "38580932618/attempts/2", + "pr": "4058", "status": "verified", - "date": "2026-09-16" + "date": "2026-10-21" }, "points": [ { "concs": [64], "item": 1, + "override": "override_tp8_hicache_c64", "goldenAL": "3.39", - "kv": "hicache", - "job": "35032235397/job/104593683504" + "kv": "hicache" }, { "concs": [66], "item": 1, + "override": "override_tp8_hicache_c66", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -10292,12 +10284,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593683262" + } }, { "concs": [68], "item": 1, + "override": "override_tp8_hicache_c68", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -10317,12 +10309,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684193" + } }, { "concs": [70], "item": 1, + "override": "override_tp8_hicache_c70", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -10342,12 +10334,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684126" + } }, { "concs": [72], "item": 1, + "override": "override_tp8_hicache_c72", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -10367,12 +10359,12 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684325" + } }, { "concs": [76], "item": 1, + "override": "override_tp8_hicache_c76", "goldenAL": "3.39", "kv": "hicache", "patch": { @@ -10392,8 +10384,7 @@ export const agentx = { } } ] - }, - "job": "35032235397/job/104593684071" + } } ], "routers": { @@ -11718,9 +11709,9 @@ export const agentx = { "detail": "MTP 3-1-4", "gpus": 8, "configKey": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9", - "imageStatus": "pruned", - "imageCommit": "c9a8fba9", + "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", + "imageStatus": "ok", + "imageCommit": null, "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -11740,12 +11731,12 @@ export const agentx = { } }, "prov": { - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", + "sha": "8e55584763cf8c1f3a1641e131b3d2d91efe64bb", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml", - "run": "38965319292/attempts/1", - "pr": "4209", + "run": "37942837688/attempts/1", + "pr": "3869", "status": "verified", - "date": "2026-10-19" + "date": "2026-10-21" }, "points": [ { @@ -12160,139 +12151,6 @@ export const agentx = { } } }, - { - "id": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp#1", - "hw": "h200", - "ckpt": "Qwen/Qwen3.5-397B-A17B-FP8", - "label": "Agg TP4·EP4", - "detail": "MTP 3-1-4", - "gpus": 4, - "configKey": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260916-c9a8fba9", - "imageStatus": "pruned", - "imageCommit": "c9a8fba9", - "submitted": "sglang", - "setup": [], - "disclosures": ["synthetic-acceptance"], - "kv": { - "available": ["none", "hicache", "mooncake"], - "unavailable": {}, - "hicache": { - "args": [ - ["--enable-hierarchical-cache"], - ["--hicache-io-backend", "kernel"], - ["--hicache-mem-layout", "page_first"], - ["--hicache-size", "77"], - ["--hicache-write-policy", "write_through_selective"] - ], - "donor": "qwen3.5-fp8-h200-sglang-agentic-hicache-mtp", - "donorHardware": "h200" - } - }, - "prov": { - "sha": "0dce380d1863c0b076ae624b0f9f05f4b98d4810", - "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/h200-fp8-mtp/agentic.yaml", - "run": "38965319292/attempts/1", - "pr": "4209", - "status": "verified", - "date": "2026-10-19" - }, - "points": [ - { - "concs": [8], - "item": 1, - "override": "override_hicache_tp4_c8", - "goldenAL": "3.39", - "kv": "hicache" - }, - { - "concs": [12], - "item": 2, - "override": "override_hicache_tp4_c12", - "goldenAL": "3.39", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--mem-fraction-static": ["0.86"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--mem-fraction-static": ["0.86"] - } - } - ] - } - } - ], - "routers": { - "sglang": { - "install": [], - "files": [], - "blocks": [ - { - "diff": { - "args": { - "--expert-parallel-size": ["4"], - "--mem-fraction-static": ["0.92"], - "--tensor-parallel-size": ["4"] - } - }, - "ref": [44, "sglang", 0] - } - ] - }, - "dynamo": { - "install": [], - "files": [], - "blocks": [ - { - "kind": "etcd", - "count_nodes": 1, - "cmd": "etcd", - "env": {}, - "args": [ - ["--data-dir", "/tmp/etcd"], - ["--listen-client-urls", "http://0.0.0.0:2379"], - ["--advertise-client-urls", "http://$ETCD_IP:2379"] - ] - }, - { - "kind": "dynamo-frontend", - "count": 1, - "cmd": "python3 -m dynamo.frontend", - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - }, - "args": [ - ["--http-port", "8000"], - ["--router-mode", "kv"] - ] - }, - { - "from": 0, - "cmd": "python3 -m dynamo.sglang", - "diff": { - "args": { - "--request-plane": ["tcp"] - }, - "env": { - "DYN_REQUEST_PLANE": "tcp", - "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" - } - } - } - ] - } - } - }, { "id": "qwen3.5-fp8-h100-sglang-agentic-mtp#0", "hw": "h100", @@ -12536,7 +12394,7 @@ export const agentx = { "--page-size": ["64"] } }, - "ref": [48, "sglang", 0] + "ref": [47, "sglang", 0] } ] }, diff --git a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx index f9e1c5a19f..cecb0a96c2 100644 --- a/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx +++ b/docs/src/snippets/agentx/Qwen/qwen3.8-flash-next.jsx @@ -1,52 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed. export const agentx = { "model": "Qwen3.8-Flash-Next", - "generatedOn": "2026-10-20", + "generatedOn": "2026-10-21", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc", + "main": "746150a2786f5cfe6fc474220d4a3335365f14c4", "prs": { - "4344": { + "4389": { "run": null, - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", - "status": "in-progress" - }, - "4342": { - "run": null, - "sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa", - "status": "in-progress" - }, - "4341": { - "run": null, - "sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9", - "status": "in-progress" - }, - "4343": { - "run": null, - "sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27", - "status": "in-progress" - }, - "4350": { - "run": null, - "sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70", + "sha": "e390de596bf5bae7a561308095ad44cf88b74f51", "status": "in-progress" }, "4346": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794", "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", - "status": "in-progress" - }, - "3930": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600", - "sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157", "status": "verified-pr" }, "4333": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "status": "verified-pr" + }, + "4303": { + "run": null, + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", "status": "in-progress" + }, + "3930": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792", + "sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4", + "status": "verified-pr" } } }, @@ -79,7 +64,16 @@ export const agentx = { "id": "nvidia/Qwen3.8-Flash-Next-NVFP4" } ], - "envs": [], + "envs": [ + { + "NCCL_CUMEM_ENABLE": "1", + "NCCL_NVLS_ENABLE": "1", + "SGLANG_ENABLE_FLASHINFER_GEMM": "true", + "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", + "SGL_ENABLE_JIT_DEEPGEMM": "false", + "TORCH_CUDA_ARCH_LIST": "10.0" + } + ], "kvMooncake": { "donor": "dsv4-fp4-gb300-dynamo-sglang-agentic-disagg", "donorHardware": "gb300", @@ -366,45 +360,45 @@ export const agentx = { "detail": "MTP 3-1-4", "gpus": 1, "configKey": "qwen3.8next-fp4-b300-sglang-agentic-mtp", - "image": "lmsysorg/sglang:qwen38flashnext", + "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", "imageStatus": "ok", "imageCommit": null, "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none"], + "available": ["none", "mooncake"], "unavailable": { - "hicache": "no verified HiCache settings for this model yet", - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" + "hicache": "no verified HiCache settings for this model yet" } }, "prov": { - "sha": "7caed5a556ee1ae97f072328fb2c1befa338caa4", - "recipe": "benchmarks/single_node/agentic/qwen3.8next_fp4_b300_sglang_mtp.sh", - "run": "33039186365/attempts/1", - "pr": "2758", + "sha": "3c0d91ceb2334554aef03a8ddfec7e0377a3f274", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/b300-fp4-mtp/agentic.yaml", + "run": "38635046800/attempts/1", + "pr": "4069", "status": "verified", - "date": "2026-08-27" + "date": "2026-10-21" }, "points": [ { "concs": [1], "item": 0, + "override": "override_c1", "goldenAL": "2.32", - "kv": "none", - "job": "33039186365/job/98408899989" + "kv": "none" }, { "concs": [4], "item": 0, + "override": "override_c4", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["4"], + "--cuda-graph-max-bs-decode": ["4"], "--max-running-requests": ["8"] } } @@ -414,24 +408,24 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["4"], + "--cuda-graph-max-bs-decode": ["4"], "--max-running-requests": ["8"] } } ] - }, - "job": "33039186365/job/98408899982" + } }, { "concs": [8], "item": 0, + "override": "override_c8", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["8"], + "--cuda-graph-max-bs-decode": ["8"], "--max-running-requests": ["16"] } } @@ -441,24 +435,24 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["8"], + "--cuda-graph-max-bs-decode": ["8"], "--max-running-requests": ["16"] } } ] - }, - "job": "33039186365/job/98408899939" + } }, { "concs": [12], "item": 0, + "override": "override_c12", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["12"], + "--cuda-graph-max-bs-decode": ["12"], "--max-running-requests": ["24"] } } @@ -468,24 +462,24 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["12"], + "--cuda-graph-max-bs-decode": ["12"], "--max-running-requests": ["24"] } } ] - }, - "job": "33039186365/job/98408899965" + } }, { "concs": [16], "item": 0, + "override": "override_c16", "goldenAL": "2.32", "kv": "none", "patch": { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["16"], + "--cuda-graph-max-bs-decode": ["16"], "--max-running-requests": ["32"] } } @@ -495,13 +489,12 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["16"], + "--cuda-graph-max-bs-decode": ["16"], "--max-running-requests": ["32"] } } ] - }, - "job": "33039186365/job/98408899999" + } } ], "routers": { @@ -515,26 +508,19 @@ export const agentx = { "workers": 1, "nodes_per_worker": 1, "cmd": "python3 -m sglang.launch_server", - "env": { - "NCCL_NVLS_ENABLE": "1", - "SGLANG_ENABLE_FLASHINFER_GEMM": "true", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", - "SGL_ENABLE_JIT_DEEPGEMM": "false", - "TORCH_CUDA_ARCH_LIST": "10.0" - }, "args": [ ["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], ["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], ["--host", "0.0.0.0"], ["--port", "30000"], - ["--cuda-graph-max-bs", "1"], - ["--dp", "1"], - ["--ep-size", "1"], + ["--cuda-graph-max-bs-decode", "1"], + ["--data-parallel-size", "1"], + ["--expert-parallel-size", "1"], ["--linear-attn-decode-backend", "flashinfer"], ["--linear-attn-prefill-backend", "flashinfer"], ["--mamba-ssm-dtype", "bfloat16"], ["--max-running-requests", "2"], - ["--mem-fraction-static", "0.80"], + ["--mem-fraction-static", "0.8"], ["--reasoning-parser", "auto"], ["--scheduler-recv-interval", "10"], ["--speculative-algorithm", "NEXTN"], @@ -542,10 +528,11 @@ export const agentx = { ["--speculative-num-draft-tokens", "4"], ["--speculative-num-steps", "3"], ["--stream-interval", "50"], + ["--tensor-parallel-size", "1"], ["--tokenizer-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], - ["--tp", "1"], ["--trust-remote-code"] - ] + ], + "envRef": 0 } ] }, @@ -728,14 +715,6 @@ export const agentx = { "workers": 1, "nodes_per_worker": 1, "cmd": "python3 -m sglang.launch_server", - "env": { - "NCCL_CUMEM_ENABLE": "1", - "NCCL_NVLS_ENABLE": "1", - "SGLANG_ENABLE_FLASHINFER_GEMM": "true", - "SGLANG_TIMEOUT_KEEP_ALIVE": "1800", - "SGL_ENABLE_JIT_DEEPGEMM": "false", - "TORCH_CUDA_ARCH_LIST": "10.0" - }, "args": [ ["--model-path", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], ["--served-model-name", "RadixArk/Qwen3.8-Flash-Next-NVFP4"], @@ -760,7 +739,8 @@ export const agentx = { ["--tokenizer-worker-num", "6"], ["--tp", "1"], ["--trust-remote-code"] - ] + ], + "envRef": 0 } ] }, diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx index 142631a126..1dc24f2dfa 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4.jsx @@ -1,52 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4", - "generatedOn": "2026-10-20", + "generatedOn": "2026-10-21", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc", + "main": "746150a2786f5cfe6fc474220d4a3335365f14c4", "prs": { - "4344": { + "4389": { "run": null, - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", - "status": "in-progress" - }, - "4342": { - "run": null, - "sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa", - "status": "in-progress" - }, - "4341": { - "run": null, - "sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9", - "status": "in-progress" - }, - "4343": { - "run": null, - "sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27", - "status": "in-progress" - }, - "4350": { - "run": null, - "sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70", + "sha": "e390de596bf5bae7a561308095ad44cf88b74f51", "status": "in-progress" }, "4346": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794", "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", - "status": "in-progress" - }, - "3930": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600", - "sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157", "status": "verified-pr" }, "4333": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "status": "verified-pr" + }, + "4303": { + "run": null, + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", "status": "in-progress" + }, + "3930": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792", + "sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4", + "status": "verified-pr" } } }, @@ -1452,9 +1437,9 @@ export const agentx = { "detail": "DSpark 6", "gpus": 8, "configKey": "dsv4-fp4-gb200-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9", - "imageStatus": "pruned", - "imageCommit": "c9a8fba9", + "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [ { @@ -1480,23 +1465,25 @@ export const agentx = { } }, "prov": { - "sha": "d9185455e166dfd034dcdfaf9608bd8335618b54", - "run": "37656460117/attempts/1", - "pr": "3701", + "sha": "afe7fff22655671661b5673722147097876f72c3", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-variants.yaml", + "run": "38953283564/attempts/1", + "pr": "4196", "status": "verified", - "date": "2026-10-12" + "date": "2026-10-21" }, "points": [ { "concs": [1], "item": 0, + "override": "override_tp8_c1", "goldenAL": "3.77", - "kv": "hicache", - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c1-mtp-hicache.yaml" + "kv": "hicache" }, { "concs": [4], "item": 1, + "override": "override_tp8_c4", "goldenAL": "3.77", "kv": "hicache", "patch": { @@ -1520,8 +1507,7 @@ export const agentx = { }, null ] - }, - "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb200-fp4/agentx/agg-gb200-tp8-c4-mtp-hicache.yaml" + } } ], "routers": { @@ -2520,10 +2506,9 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "3"], - ["--hicache-write-policy", "write_back"] + ["--hicache-ratio", "2.75"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg", "donorHardware": "b300" } }, @@ -2800,10 +2785,9 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "3"], - ["--hicache-write-policy", "write_back"] + ["--hicache-ratio", "2.75"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg", "donorHardware": "b300" } }, @@ -3301,18 +3285,19 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "3"], - ["--hicache-write-policy", "write_back"] + ["--hicache-ratio", "2.75"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg", "donorHardware": "b300" } }, "prov": { "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp8-c1-mtp.yaml", + "run": "39534932794", "pr": "4346", - "status": "in-progress" + "comment": "6331655172", + "status": "verified-pr" }, "points": [ { @@ -3455,10 +3440,9 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "3"], - ["--hicache-write-policy", "write_back"] + ["--hicache-ratio", "2.75"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg", "donorHardware": "b300" } }, @@ -3696,17 +3680,18 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "3"], - ["--hicache-write-policy", "write_back"] + ["--hicache-ratio", "2.75"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-dynamo-sglang-agentic-agg", "donorHardware": "b300" } }, "prov": { "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", + "run": "39534932794", "pr": "4346", - "status": "in-progress" + "comment": "6331655172", + "status": "verified-pr" }, "points": [ { @@ -4174,18 +4159,19 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], ["--hicache-ratio", "3"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-dynamo-sglang-agentic-disagg", "donorHardware": "b300" } }, "prov": { "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", + "run": "39534932794", "pr": "4346", - "status": "in-progress" + "comment": "6331655172", + "status": "verified-pr" }, "points": [ { @@ -4446,19 +4432,20 @@ export const agentx = { "args": [ ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], - ["--hicache-mem-layout", "page_first_direct"], ["--hicache-ratio", "3"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv4-fp4-b300-sglang-agentic-hicache-mtp", + "donor": "dsv4-fp4-b300-dynamo-sglang-agentic-disagg", "donorHardware": "b300" } }, "prov": { "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p2d-dep8-dep8-c64-mtp-kvoffload.yaml", + "run": "39534932794", "pr": "4346", - "status": "in-progress" + "comment": "6331655172", + "status": "verified-pr" }, "points": [ { @@ -4603,8 +4590,10 @@ export const agentx = { "prov": { "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-dep8-c384-mtp-kvoffload.yaml", + "run": "39534932794", "pr": "4346", - "status": "in-progress" + "comment": "6331655172", + "status": "verified-pr" }, "points": [ { @@ -4737,9 +4726,9 @@ export const agentx = { "detail": "DSpark 6 · dsv4-fp4-b200-dynamo-sglang-agentic-agg", "gpus": 8, "configKey": "dsv4-fp4-b200-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9", - "imageStatus": "pruned", - "imageCommit": "c9a8fba9", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -4759,23 +4748,25 @@ export const agentx = { } }, "prov": { - "sha": "1d0716bb364d49a450386c867d632f7409ff7ced", - "run": "35345548475/attempts/1", - "pr": "3257", + "sha": "e82118bb200aad9275f1d821b4265f251cadd7d2", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-variants.yaml", + "run": "38960572895/attempts/1", + "pr": "4203", "status": "verified", - "date": "2026-09-22" + "date": "2026-10-21" }, "points": [ { "concs": [1], "item": 0, + "override": "override_c1", "goldenAL": "3.77", - "kv": "hicache", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c1-mtp-hicache.yaml" + "kv": "hicache" }, { "concs": [4], "item": 1, + "override": "override_c4", "goldenAL": "3.77", "kv": "hicache", "patch": { @@ -4799,12 +4790,12 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c4-mtp-hicache.yaml" + } }, { "concs": [8], "item": 2, + "override": "override_c8", "goldenAL": "3.77", "kv": "hicache", "patch": { @@ -4828,8 +4819,7 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/agg-b200-tp8-c8-mtp-hicache.yaml" + } } ], "routers": { @@ -4875,6 +4865,23 @@ export const agentx = { "workers": 1, "nodes_per_worker": 1, "cmd": "python3 -m dynamo.sglang", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "NATS_SERVER": "nats://$NATS_IP:4222", + "NCCL_CUMEM_ENABLE": "1", + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DSV4_REASONING_EFFORT": "high", + "SGLANG_ENABLE_PREFILL_WAR_READ_DONE": "1", + "SGLANG_JIT_DEEPGEMM_PRECOMPILE": "1", + "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", + "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2": "1", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "1", + "SGLANG_OPT_USE_JIT_NORM": "1", + "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", + "SGLANG_OPT_USE_TOPK_V2": "True", + "SGLANG_RAGGED_VERIFY_MODE": "static" + }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4-Pro-0813"], ["--served-model-name", "deepseek-ai/DeepSeek-V4-Pro-0813"], @@ -4909,8 +4916,7 @@ export const agentx = { ["--trust-remote-code"], ["--watchdog-timeout", "1000000"], ["--weight-loader-prefetch-checkpoints"] - ], - "envRef": 1 + ] } ] }, @@ -5573,9 +5579,9 @@ export const agentx = { "detail": "DSpark 6", "gpus": 16, "configKey": "dsv4-fp4-b200-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9", - "imageStatus": "pruned", - "imageCommit": "c9a8fba9", + "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -5595,23 +5601,25 @@ export const agentx = { } }, "prov": { - "sha": "1d0716bb364d49a450386c867d632f7409ff7ced", - "run": "35345548475/attempts/1", - "pr": "3257", + "sha": "8b1ac122ff6b7e6583af377f190fec7321c22b90", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml", + "run": "38925477096", + "pr": "4176", "status": "verified", - "date": "2026-09-22" + "date": "2026-10-21" }, "points": [ { "concs": [64], "item": 0, + "override": "override_1p1d_c64", "goldenAL": "3.77", - "kv": "hicache", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml" + "kv": "hicache" }, { "concs": [128], "item": 1, + "override": "override_1p1d_c128", "goldenAL": "3.77", "kv": "hicache", "patch": { @@ -5636,8 +5644,7 @@ export const agentx = { }, null ] - }, - "recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-1p1d-dep8-dep8-c128-mtp-kvoffload.yaml" + } } ], "routers": { @@ -5712,7 +5719,6 @@ export const agentx = { "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "9216", "SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW": "1", - "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1", "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2": "1", "SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE": "1", @@ -5787,7 +5793,6 @@ export const agentx = { "SGLANG_OPT_DEEPGEMM_MEGA_MOE": "1", "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "4096", "SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW": "1", - "SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT": "1", "SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS": "1", "SGLANG_OPT_USE_ONLINE_COMPRESS": "0", "SGLANG_RAGGED_VERIFY_MODE": "static", @@ -5893,9 +5898,9 @@ export const agentx = { "detail": "DSpark 6", "gpus": 24, "configKey": "dsv4-fp4-b200-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9", - "imageStatus": "pruned", - "imageCommit": "c9a8fba9", + "image": "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469", + "imageStatus": "ok", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -5915,17 +5920,18 @@ export const agentx = { } }, "prov": { - "sha": "1d0716bb364d49a450386c867d632f7409ff7ced", - "recipe": "benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-b200-2p1d-dep8-dep8-c256-mtp-kvoffload.yaml", - "run": "35345548475/attempts/1", - "pr": "3257", + "sha": "8b1ac122ff6b7e6583af377f190fec7321c22b90", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b200-fp4/agentx/disagg-variants.yaml", + "run": "38925477096", + "pr": "4176", "status": "verified", - "date": "2026-09-22" + "date": "2026-10-21" }, "points": [ { "concs": [256], "item": 2, + "override": "override_2p1d_c256", "goldenAL": "3.77", "kv": "hicache" } diff --git a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx index 4d5ef51d4c..c7a0ba6a85 100644 --- a/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx +++ b/docs/src/snippets/agentx/deepseek-ai/deepseek-v4_1.jsx @@ -1,52 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed. export const agentx = { "model": "DeepSeek-V4.1", - "generatedOn": "2026-10-20", + "generatedOn": "2026-10-21", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc", + "main": "746150a2786f5cfe6fc474220d4a3335365f14c4", "prs": { - "4344": { + "4389": { "run": null, - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", - "status": "in-progress" - }, - "4342": { - "run": null, - "sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa", - "status": "in-progress" - }, - "4341": { - "run": null, - "sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9", - "status": "in-progress" - }, - "4343": { - "run": null, - "sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27", - "status": "in-progress" - }, - "4350": { - "run": null, - "sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70", + "sha": "e390de596bf5bae7a561308095ad44cf88b74f51", "status": "in-progress" }, "4346": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794", "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", - "status": "in-progress" - }, - "3930": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600", - "sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157", "status": "verified-pr" }, "4333": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "status": "verified-pr" + }, + "4303": { + "run": null, + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", "status": "in-progress" + }, + "3930": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792", + "sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4", + "status": "verified-pr" } } }, @@ -96,12 +81,17 @@ export const agentx = { "NATS_SERVER": "nats://$NATS_IP:4222" }, { - "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", - "SGLANG_DEFAULT_THINKING": "1", - "SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT": "per_rank", - "SGLANG_DSV41_REASONING_EFFORT": "high", - "SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE": "1", - "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + "DYN_EVENT_PLANE": "zmq", + "DYN_REQUEST_PLANE": "tcp", + "DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS": "3600", + "DYN_ROUTER_QUEUE_THRESHOLD": "None", + "DYN_ROUTER_TEMPERATURE": "0", + "DYN_TCP_CHANNEL_BUFFER": "128", + "DYN_TCP_REQUEST_TIMEOUT": "60", + "DYN_TOKENIZER": "fastokens", + "DYN_TOKENIZER_CACHE": "1", + "DYN_TOKENIZER_CACHE_BYTES": "8000000000", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" } ], "kvMooncake": { @@ -189,11 +179,11 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.02"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { @@ -432,11 +422,11 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.02"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { @@ -635,11 +625,11 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.02"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { @@ -771,11 +761,11 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.02"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { @@ -920,11 +910,11 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.02"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { @@ -1104,13 +1094,13 @@ export const agentx = { "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#0", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Disagg 4P1D · prefill DEP2 · decode DEP16 (4 nodes)", + "label": "Disagg 3P1D · prefill DEP2 · decode DEP16 (4 nodes)", "detail": "DSpark 5", - "gpus": 24, + "gpus": 22, "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560@sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb", "imageStatus": "ok", - "imageCommit": "a6a8c560", + "imageCommit": "c1c287a4", "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -1122,28 +1112,26 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.02"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "38911006463/attempts/1", - "pr": "4180", - "status": "verified", - "date": "2026-10-19" + "pr": "4303", + "status": "in-progress" }, "points": [ { "concs": [1228], "item": 0, - "override": "override_4p1d_dep2_dep16_c1228", + "override": "override_3p1d_dep2_dep16_c1228", "goldenAL": "3.51", - "kv": "hicache" + "kv": "mooncake" } ], "routers": { @@ -1162,13 +1150,6 @@ export const agentx = { ["--advertise-client-urls", "http://$ETCD_IP:2379"] ] }, - { - "kind": "nats", - "count_nodes": 1, - "cmd": "nats-server -js -sd /tmp/nats", - "env": {}, - "args": [] - }, { "kind": "dynamo-frontend", "count": 1, @@ -1181,25 +1162,51 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 0 + "envRef": 1 + }, + { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": { + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--eviction_high_watermark_ratio", "0.90"], + ["--client_ttl", "60"], + ["--rpc_thread_num", "64"] + ] }, { "kind": "worker", "role": "prefill", - "workers": 4, + "workers": 3, "nodes_per_worker": 1, "gpu_pinning": [ ["0,1"], ["2,3"], - ["0,1"], - ["2,3"] + [null] ], "cmd": "python3 -m dynamo.sglang", "env": { + "DYN_EVENT_PLANE": "zmq", "DYN_REQUEST_PLANE": "tcp", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", "MC_FORCE_MNNVL": "1", - "NATS_SERVER": "nats://$NATS_IP:4222", + "MC_TCP_BIND_ADDRESS": "$NODE_IP", + "MOONCAKE_DEVICE": "$IB_DEVICES", + "MOONCAKE_GLOBAL_SEGMENT_SIZE": "80gb", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_PROTOCOL": "rdma", + "MOONCAKE_STANDALONE_STORAGE": "0", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata", "NCCL_CUMEM_ENABLE": "1", "NCCL_MNNVL_ENABLE": "1", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", @@ -1209,7 +1216,8 @@ export const agentx = { "SGLANG_ENABLE_PREFILL_WAR_READ_DONE": "1", "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "32768", - "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "1" + "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "1", + "WITH_NVIDIA_PEERMEM": "0" }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4.1-Flash"], @@ -1223,27 +1231,32 @@ export const agentx = { ["--chunked-prefill-size", "65536"], ["--data-parallel-size", "2"], ["--disaggregation-transfer-backend", "mooncake"], - ["--dsv4-attn-backend", "flashmla"], + ["--dsv4-attn-backend", "flashmla_mega"], ["--enable-decoder-swa-bounded-replay"], ["--enable-dp-attention"], + ["--enable-dp-attention-local-control-broadcast"], ["--enable-dp-lm-head"], - ["--enable-hierarchical-cache"], + ["--enable-unified-cache-external-linker"], ["--expert-parallel-size", "2"], - ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], ["--hicache-ratio", "1.02"], - ["--hicache-write-policy", "write_back"], + ["--hicache-storage-backend-extra-config", "{\"enable_group_semantics\":true}"], ["--max-running-requests", "128"], ["--mem-fraction-static", "0.85"], ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], ["--moe-a2a-backend", "megamoe"], ["--reasoning-parser", "auto"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], ["--speculative-algorithm", "DSPARK"], ["--speculative-dspark-block-size", "5"], + ["--speculative-num-steps", "5"], ["--swa-prefix-tails", "2048"], ["--tensor-parallel-size", "2"], + ["--tokenizer-backend", "fastokens"], ["--tool-call-parser", "auto"], ["--trust-remote-code"], + ["--unified-cache-external-linker-backend", "mooncake"], ["--watchdog-timeout", "3600"], ["--weight-loader-prefetch-checkpoints"] ] @@ -1255,11 +1268,20 @@ export const agentx = { "nodes_per_worker": 4, "cmd": "python3 -m dynamo.sglang", "env": { + "DYN_EVENT_PLANE": "zmq", "DYN_REQUEST_PLANE": "tcp", "DYN_SGL_GATEWAY_WORKERS": "8", "ETCD_ENDPOINTS": "http://$ETCD_IP:2379", + "MC_ENABLE_DEST_DEVICE_AFFINITY": "1", "MC_FORCE_MNNVL": "1", - "NATS_SERVER": "nats://$NATS_IP:4222", + "MC_TCP_BIND_ADDRESS": "$NODE_IP", + "MOONCAKE_DEVICE": "$IB_DEVICES", + "MOONCAKE_GLOBAL_SEGMENT_SIZE": "140gb", + "MOONCAKE_LOCAL_HOSTNAME": "$NODE_IP", + "MOONCAKE_MASTER": "$MOONCAKE_MASTER_IP:8700", + "MOONCAKE_PROTOCOL": "rdma", + "MOONCAKE_STANDALONE_STORAGE": "0", + "MOONCAKE_TE_META_DATA_SERVER": "http://$MOONCAKE_MASTER_IP:8701/metadata", "NCCL_CUMEM_ENABLE": "1", "NCCL_MNNVL_ENABLE": "1", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", @@ -1267,8 +1289,9 @@ export const agentx = { "SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT": "per_rank", "SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE": "1", "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", - "SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH": "1", - "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "16384" + "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK": "16384", + "SGLANG_OPT_DSV41_BF16_INDEX_LOGITS": "1", + "WITH_NVIDIA_PEERMEM": "0" }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4.1-Flash"], @@ -1283,21 +1306,32 @@ export const agentx = { ["--cuda-graph-max-bs-decode", "24"], ["--data-parallel-size", "16"], ["--disaggregation-decode-extra-slots", "36"], + ["--disaggregation-decode-polling-interval", "8"], ["--disaggregation-transfer-backend", "mooncake"], - ["--dsv4-attn-backend", "flashmla"], + ["--dsv4-attn-backend", "flashmla_mega"], + ["--enable-decoder-swa-bounded-replay"], ["--enable-dp-attention"], + ["--enable-dp-attention-local-control-broadcast"], ["--enable-dp-lm-head"], + ["--enable-unified-cache-external-linker"], ["--expert-parallel-size", "16"], + ["--load-balance-method", "total_tokens"], ["--max-running-requests", "384"], ["--mem-fraction-static", "0.85"], ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], ["--moe-a2a-backend", "megamoe"], ["--reasoning-parser", "auto"], + ["--router-mode", "round-robin"], ["--speculative-algorithm", "DSPARK"], ["--speculative-dspark-block-size", "5"], + ["--speculative-num-steps", "5"], + ["--stream-interval", "32"], + ["--swa-prefix-tails", "2048"], ["--tensor-parallel-size", "16"], + ["--tokenizer-backend", "fastokens"], ["--tool-call-parser", "auto"], ["--trust-remote-code"], + ["--unified-cache-external-linker-backend", "mooncake"], ["--watchdog-timeout", "3600"], ["--weight-loader-prefetch-checkpoints"] ] @@ -1308,6 +1342,22 @@ export const agentx = { "install": [], "files": [], "blocks": [ + { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": {}, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--eviction_high_watermark_ratio", "0.90"], + ["--client_ttl", "60"], + ["--rpc_thread_num", "64"] + ] + }, { "from": 3, "cmd": "python3 -m sglang.launch_server", @@ -1315,12 +1365,14 @@ export const agentx = { "args": { "--skip-server-warmup": [], "--request-plane": null, - "--kv-events-config": null + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { + "DYN_EVENT_PLANE": null, "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null, - "NATS_SERVER": null + "ETCD_ENDPOINTS": null } } }, @@ -1330,13 +1382,14 @@ export const agentx = { "diff": { "args": { "--skip-server-warmup": [], - "--request-plane": null + "--request-plane": null, + "--router-mode": null }, "env": { + "DYN_EVENT_PLANE": null, "DYN_REQUEST_PLANE": null, "DYN_SGL_GATEWAY_WORKERS": null, - "ETCD_ENDPOINTS": null, - "NATS_SERVER": null + "ETCD_ENDPOINTS": null } } }, @@ -1346,7 +1399,6 @@ export const agentx = { "env": {}, "args": [ ["--decode", "http://$DECODE_0_HEAD_IP:30000"], - ["--prefill", "http://$PREFILL_3_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], @@ -1364,13 +1416,13 @@ export const agentx = { "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#1", "hw": "gb300", "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", - "label": "Disagg 6P1D · prefill DEP2 · decode DEP16 (4 nodes)", + "label": "Disagg 5P1D · prefill DEP2 · decode DEP16 (4 nodes)", "detail": "DSpark 5", - "gpus": 28, + "gpus": 26, "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261015-a6a8c560@sha256:0a1ba33aa4e032b8c5bcebdc69c45664074c032074add4f7db01629ec1fcc381", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb", "imageStatus": "ok", - "imageCommit": "a6a8c560", + "imageCommit": "c1c287a4", "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -1382,28 +1434,26 @@ export const agentx = { ["--enable-hierarchical-cache"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], - ["--hicache-ratio", "1.02"], + ["--hicache-ratio", "2"], ["--hicache-write-policy", "write_back"] ], - "donor": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", - "donorHardware": "gb300" + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" } }, "prov": { - "sha": "2bc599fc4149e0d960c65d4deb528fd55cf282af", + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", - "run": "38911006463/attempts/1", - "pr": "4180", - "status": "verified", - "date": "2026-10-19" + "pr": "4303", + "status": "in-progress" }, "points": [ { "concs": [2884], "item": 1, - "override": "override_6p1d_dep2_dep16_c2884", + "override": "override_5p1d_dep2_dep16_c2884", "goldenAL": "3.51", - "kv": "hicache" + "kv": "mooncake" } ], "routers": { @@ -1422,13 +1472,6 @@ export const agentx = { ["--advertise-client-urls", "http://$ETCD_IP:2379"] ] }, - { - "kind": "nats", - "count_nodes": 1, - "cmd": "nats-server -js -sd /tmp/nats", - "env": {}, - "args": [] - }, { "kind": "dynamo-frontend", "count": 1, @@ -1441,21 +1484,38 @@ export const agentx = { ["--active-prefill-tokens-threshold", "None"], ["--active-prefill-tokens-threshold-frac", "None"] ], - "envRef": 0 + "envRef": 1 }, { - "workers": 6, + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": { + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--eviction_high_watermark_ratio", "0.90"], + ["--client_ttl", "60"], + ["--rpc_thread_num", "64"] + ] + }, + { + "workers": 5, "gpu_pinning": [ ["0,1"], ["2,3"], ["0,1"], ["2,3"], - ["0,1"], - ["2,3"] + [null] ], "diff": { "args": { - "--hicache-ratio": ["1.1"] + "--max-prefill-tokens": ["32768"] } }, "ref": [5, "dynamo", 3] @@ -1476,6 +1536,22 @@ export const agentx = { "install": [], "files": [], "blocks": [ + { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": {}, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--eviction_high_watermark_ratio", "0.90"], + ["--client_ttl", "60"], + ["--rpc_thread_num", "64"] + ] + }, { "from": 3, "cmd": "python3 -m sglang.launch_server", @@ -1483,12 +1559,14 @@ export const agentx = { "args": { "--skip-server-warmup": [], "--request-plane": null, - "--kv-events-config": null + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null }, "env": { + "DYN_EVENT_PLANE": null, "DYN_REQUEST_PLANE": null, - "ETCD_ENDPOINTS": null, - "NATS_SERVER": null + "ETCD_ENDPOINTS": null } } }, @@ -1498,13 +1576,14 @@ export const agentx = { "diff": { "args": { "--skip-server-warmup": [], - "--request-plane": null + "--request-plane": null, + "--router-mode": null }, "env": { + "DYN_EVENT_PLANE": null, "DYN_REQUEST_PLANE": null, "DYN_SGL_GATEWAY_WORKERS": null, - "ETCD_ENDPOINTS": null, - "NATS_SERVER": null + "ETCD_ENDPOINTS": null } } }, @@ -1514,6 +1593,202 @@ export const agentx = { "env": {}, "args": [ ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_3_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_2_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_1_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_0_HEAD_IP:30000", "8998"], + ["--pd-disaggregation"], + ["--host", "0.0.0.0"], + ["--port", "8000"], + ["--policy", "cache_aware"] + ] + } + ] + } + } + }, + { + "id": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg#2", + "hw": "gb300", + "ckpt": "deepseek-ai/DeepSeek-V4.1-Flash", + "label": "Disagg 8P1D · prefill DEP2 · decode DEP16 (4 nodes)", + "detail": "DSpark 5", + "gpus": 32, + "configKey": "dsv41flash-fp4-gb300-dynamo-sglang-agentic-dspark-disagg", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb", + "imageStatus": "ok", + "imageCommit": "c1c287a4", + "submitted": "dynamo", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-ratio", "2"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "dsv41flash-fp4-b200-sglang-agentic-dspark", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", + "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4/agentx/disagg-variants.yaml", + "pr": "4303", + "status": "in-progress" + }, + "points": [ + { + "concs": [4096], + "item": 2, + "override": "override_8p1d_dep2_dep16_c4096", + "goldenAL": "3.51", + "kv": "mooncake" + } + ], + "routers": { + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"], + ["--router-session-affinity-ttl-secs", "14400"], + ["--active-decode-blocks-threshold", "None"], + ["--active-prefill-tokens-threshold", "None"], + ["--active-prefill-tokens-threshold-frac", "None"] + ], + "envRef": 1 + }, + { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": { + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--eviction_high_watermark_ratio", "0.90"], + ["--client_ttl", "60"], + ["--rpc_thread_num", "64"] + ] + }, + { + "workers": 8, + "gpu_pinning": [ + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"], + ["0,1"], + ["2,3"] + ], + "ref": [6, "dynamo", 3] + }, + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--disaggregation-decode-extra-slots": ["128"], + "--max-running-requests": ["1024"] + } + }, + "ref": [5, "dynamo", 4] + } + ] + }, + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "mooncake-master", + "count_nodes": 1, + "ports": ["8700"], + "cmd": "mooncake_master", + "env": {}, + "args": [ + ["--port", "8700"], + ["--enable_http_metadata_server", "true"], + ["--http_metadata_server_port", "8701"], + ["--default_kv_lease_ttl", "10000"], + ["--eviction_high_watermark_ratio", "0.90"], + ["--client_ttl", "60"], + ["--rpc_thread_num", "64"] + ] + }, + { + "from": 3, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--kv-events-config": null, + "--router-mode": null, + "--router-session-affinity-ttl-secs": null + }, + "env": { + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "from": 4, + "cmd": "python3 -m sglang.launch_server", + "diff": { + "args": { + "--skip-server-warmup": [], + "--request-plane": null, + "--router-mode": null + }, + "env": { + "DYN_EVENT_PLANE": null, + "DYN_REQUEST_PLANE": null, + "DYN_SGL_GATEWAY_WORKERS": null, + "ETCD_ENDPOINTS": null + } + } + }, + { + "kind": "sglang-router", + "cmd": "python3 -m sglang_router.launch_router", + "env": {}, + "args": [ + ["--decode", "http://$DECODE_0_HEAD_IP:30000"], + ["--prefill", "http://$PREFILL_7_HEAD_IP:30000", "8998"], + ["--prefill", "http://$PREFILL_6_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_5_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_4_HEAD_IP:30000", "8998"], ["--prefill", "http://$PREFILL_3_HEAD_IP:30000", "8998"], @@ -1840,7 +2115,7 @@ export const agentx = { "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null } }, - "ref": [7, "sglang", 0] + "ref": [8, "sglang", 0] } ] }, @@ -2076,7 +2351,7 @@ export const agentx = { "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [8, "sglang", 0] + "ref": [9, "sglang", 0] }, { "kind": "sglang-router", @@ -2556,7 +2831,7 @@ export const agentx = { "--tensor-parallel-size": ["2"] } }, - "ref": [10, "sglang", 0] + "ref": [11, "sglang", 0] } ] }, @@ -2720,7 +2995,7 @@ export const agentx = { "SGLANG_RAGGED_VERIFY_MODE": "static" } }, - "ref": [10, "sglang", 0] + "ref": [11, "sglang", 0] }, { "kind": "sglang-router", @@ -2851,7 +3126,7 @@ export const agentx = { "--tensor-parallel-size": ["4"] } }, - "ref": [12, "sglang", 0] + "ref": [13, "sglang", 0] }, { "kind": "sglang-router", @@ -2925,9 +3200,9 @@ export const agentx = { "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb", "imageStatus": "ok", - "imageCommit": "684fea8a", + "imageCommit": "c1c287a4", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -2947,12 +3222,12 @@ export const agentx = { } }, "prov": { - "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", + "sha": "c1457ce06a7622258b011d4ab775c5b86883943e", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "38749770474/attempts/1", - "pr": "4111", + "run": "39606211010/attempts/1", + "pr": "4365", "status": "verified", - "date": "2026-10-16" + "date": "2026-10-22" }, "points": [ { @@ -3166,9 +3441,9 @@ export const agentx = { "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb", "imageStatus": "ok", - "imageCommit": "684fea8a", + "imageCommit": "c1c287a4", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -3188,12 +3463,12 @@ export const agentx = { } }, "prov": { - "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", + "sha": "c1457ce06a7622258b011d4ab775c5b86883943e", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "38749770474/attempts/1", - "pr": "4111", + "run": "39606211010/attempts/1", + "pr": "4365", "status": "verified", - "date": "2026-10-16" + "date": "2026-10-22" }, "points": [ { @@ -3256,7 +3531,7 @@ export const agentx = { "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": null } }, - "ref": [14, "sglang", 0] + "ref": [15, "sglang", 0] } ] }, @@ -3313,9 +3588,9 @@ export const agentx = { "detail": "DSpark 5", "gpus": 4, "configKey": "dsv41flash-fp4-b200-sglang-agentic-dspark", - "image": "lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8", + "image": "lmsysorg/sglang:nightly-dev-cu13-20261021-c1c287a4@sha256:b0473834763b2753e069cb96c1acaf235ba7fd632ccd9bbeb44da809c437fdeb", "imageStatus": "ok", - "imageCommit": "684fea8a", + "imageCommit": "c1c287a4", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -3335,12 +3610,12 @@ export const agentx = { } }, "prov": { - "sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae", + "sha": "c1457ce06a7622258b011d4ab775c5b86883943e", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b200-fp4-mtp/agentic.yaml", - "run": "38749770474/attempts/1", - "pr": "4111", + "run": "39606211010/attempts/1", + "pr": "4365", "status": "verified", - "date": "2026-10-16" + "date": "2026-10-22" }, "points": [ { @@ -3516,7 +3791,7 @@ export const agentx = { "SGLANG_OPT_DSV41_MEGA_MHC_PREFILL": "0" } }, - "ref": [15, "sglang", 0] + "ref": [16, "sglang", 0] }, { "kind": "sglang-router", @@ -3615,7 +3890,7 @@ export const agentx = { "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml", "run": "39318221321/attempts/1", - "pr": "XXX", + "pr": "4327", "status": "verified", "date": "2026-10-20" }, @@ -3833,7 +4108,7 @@ export const agentx = { "sha": "5bfd55593e80a10bb9a0e6dd22ec33fd606cf574", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h200-fp4-mtp/agentic.yaml", "run": "39318221321/attempts/1", - "pr": "XXX", + "pr": "4327", "status": "verified", "date": "2026-10-20" }, @@ -3957,7 +4232,7 @@ export const agentx = { "--tensor-parallel-size": ["8"] } }, - "ref": [17, "sglang", 0] + "ref": [18, "sglang", 0] } ] }, @@ -4198,6 +4473,14 @@ export const agentx = { "workers": 1, "nodes_per_worker": 1, "cmd": "python3 -m sglang.launch_server", + "env": { + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT": "per_rank", + "SGLANG_DSV41_REASONING_EFFORT": "high", + "SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE": "1", + "SGLANG_TIMEOUT_KEEP_ALIVE": "900" + }, "args": [ ["--model-path", "deepseek-ai/DeepSeek-V4.1-Flash"], ["--served-model-name", "deepseek-ai/DeepSeek-V4.1-Flash"], @@ -4218,8 +4501,7 @@ export const agentx = { ["--tp", "8"], ["--trust-remote-code"], ["--watchdog-timeout", "3600"] - ], - "envRef": 1 + ] } ] }, @@ -4276,9 +4558,9 @@ export const agentx = { "detail": "DSpark 5", "gpus": 8, "configKey": "dsv41flash-fp4-h100-sglang-agentic-dspark-dpa", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:0e1b14e302619a42ef5581b87db6804651d4f946301cf543d74a3a1eb1c33b40", - "imageStatus": "digest-only", - "imageCommit": "582389ce", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", + "imageStatus": "ok", + "imageCommit": null, "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -4298,24 +4580,25 @@ export const agentx = { } }, "prov": { - "sha": "510568c51a127f46c6fcc54d66e925395a512ed1", - "recipe": "benchmarks/single_node/agentic/dsv41flash_fp4_h100_sglang_mtp.sh", - "run": "35690159495/attempts/2", - "pr": "3345", + "sha": "32753f81a43aedd21dda124a92fe10ac8d98b0d8", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/h100-fp4-mtp/agentic.yaml", + "run": "39339355047/attempts/1", + "pr": "4324", "status": "verified", - "date": "2026-09-24" + "date": "2026-10-21" }, "points": [ { "concs": [4], "item": 0, + "override": "override_dp8_c4", "goldenAL": "3.51", - "kv": "none", - "job": "35690159495/job/106785886368" + "kv": "none" }, { "concs": [8], "item": 0, + "override": "override_dp8_c8", "goldenAL": "3.51", "kv": "none", "patch": { @@ -4336,12 +4619,12 @@ export const agentx = { } } ] - }, - "job": "35690159495/job/106785882300" + } }, { "concs": [16], "item": 0, + "override": "override_dp8_c16", "goldenAL": "3.51", "kv": "none", "patch": { @@ -4362,12 +4645,12 @@ export const agentx = { } } ] - }, - "job": "35690159495/job/106785881433" + } }, { "concs": [20], "item": 0, + "override": "override_dp8_c20", "goldenAL": "3.51", "kv": "none", "patch": { @@ -4388,8 +4671,7 @@ export const agentx = { } } ] - }, - "job": "35690159495/job/106785882144" + } } ], "routers": { @@ -4400,31 +4682,38 @@ export const agentx = { { "diff": { "args": { - "--dp-size": ["8"], + "--data-parallel-size": ["8"], "--enable-dp-attention": [], "--enable-dp-lm-head": [], + "--expert-parallel-size": ["8"], "--max-running-requests": ["8"], - "--swa-prefix-tails": ["64"] + "--swa-prefix-tails": ["64"], + "--tensor-parallel-size": ["8"], + "--ep-size": null, + "--tp": null + }, + "env": { + "NCCL_CUMEM_ENABLE": "1" } }, - "ref": [19, "sglang", 0] + "ref": [20, "sglang", 0] }, { "kind": "sglang-router", "cmd": "python3 -m sglang_router.launch_router", + "env": {}, "args": [ + ["--host", "0.0.0.0"], + ["--port", "8000"], ["--policy", "consistent_hashing"], ["--request-id-headers", "x-correlation-id"], ["--dp-aware"], - ["--host", "0.0.0.0"], - ["--port", "8000"], ["--connect-timeout-secs", "900"], ["--request-timeout-secs", "14400"], ["--disable-health-check"], ["--disable-retries"], ["--worker-urls", "http://127.0.0.1:30000"] - ], - "envRef": 1 + ] } ] }, diff --git a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx index 40f5d78119..7d0d644b5e 100644 --- a/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx +++ b/docs/src/snippets/agentx/meta-models/muse-spark-open.jsx @@ -1,52 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed. export const agentx = { "model": "Muse Spark Open", - "generatedOn": "2026-10-20", + "generatedOn": "2026-10-21", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc", + "main": "746150a2786f5cfe6fc474220d4a3335365f14c4", "prs": { - "4344": { + "4389": { "run": null, - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", - "status": "in-progress" - }, - "4342": { - "run": null, - "sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa", - "status": "in-progress" - }, - "4341": { - "run": null, - "sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9", - "status": "in-progress" - }, - "4343": { - "run": null, - "sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27", - "status": "in-progress" - }, - "4350": { - "run": null, - "sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70", + "sha": "e390de596bf5bae7a561308095ad44cf88b74f51", "status": "in-progress" }, "4346": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794", "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", - "status": "in-progress" - }, - "3930": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600", - "sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157", "status": "verified-pr" }, "4333": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "status": "verified-pr" + }, + "4303": { + "run": null, + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", "status": "in-progress" + }, + "3930": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792", + "sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4", + "status": "verified-pr" } } }, @@ -173,18 +158,20 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "120"] + ["--hicache-size", "120"], + ["--hicache-write-policy", "write_through"] ], "donor": "musespark-fp4-gb300-sglang-agentic-dflash", - "donorHardware": "gb300", - "donorRef": "main" + "donorHardware": "gb300" } }, "prov": { "sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb300-fp4-mtp/agentic.yaml", + "run": "39408192975/attempts/1", "pr": "4342", - "status": "in-progress" + "status": "verified", + "date": "2026-10-22" }, "points": [ { @@ -672,18 +659,20 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "120"] + ["--hicache-size", "120"], + ["--hicache-write-policy", "write_through"] ], "donor": "musespark-fp4-gb300-sglang-agentic-dflash", - "donorHardware": "gb300", - "donorRef": "main" + "donorHardware": "gb300" } }, "prov": { "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "39427144732/attempts/4", "pr": "4344", - "status": "in-progress" + "status": "verified", + "date": "2026-10-22" }, "points": [ { @@ -912,18 +901,20 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "120"] + ["--hicache-size", "120"], + ["--hicache-write-policy", "write_through"] ], "donor": "musespark-fp4-gb300-sglang-agentic-dflash", - "donorHardware": "gb300", - "donorRef": "main" + "donorHardware": "gb300" } }, "prov": { "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/musespark/sglang/gb300-fp4/agentx/agg-variants.yaml", + "run": "39427144732/attempts/4", "pr": "4344", - "status": "in-progress" + "status": "verified", + "date": "2026-10-22" }, "points": [ { @@ -1225,15 +1216,16 @@ export const agentx = { ["--hicache-size", "120"] ], "donor": "musespark-fp4-gb200-sglang-agentic-dflash", - "donorHardware": "gb200", - "donorRef": "main" + "donorHardware": "gb200" } }, "prov": { "sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/gb200-fp4-mtp/agentic.yaml", + "run": "39404635036/attempts/1", "pr": "4343", - "status": "in-progress" + "status": "verified", + "date": "2026-10-22" }, "points": [ { @@ -1835,18 +1827,19 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "216"] + ["--hicache-size", "170"] ], "donor": "musespark-fp4-b300-sglang-agentic-dflash", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { "sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml", + "run": "39385088584/attempts/1", "pr": "4341", - "status": "in-progress" + "status": "verified", + "date": "2026-10-22" }, "points": [ { @@ -2070,18 +2063,19 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "216"] + ["--hicache-size", "170"] ], "donor": "musespark-fp4-b300-sglang-agentic-dflash", - "donorHardware": "b300", - "donorRef": "main" + "donorHardware": "b300" } }, "prov": { "sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b300-fp4-mtp/agentic.yaml", + "run": "39385088584/attempts/1", "pr": "4341", - "status": "in-progress" + "status": "verified", + "date": "2026-10-22" }, "points": [ { @@ -2495,18 +2489,20 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "160"] + ["--hicache-size", "120"], + ["--hicache-write-policy", "write_through"] ], "donor": "musespark-fp4-b200-sglang-agentic-dflash", - "donorHardware": "b200", - "donorRef": "main" + "donorHardware": "b200" } }, "prov": { "sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml", + "run": "39449248229/attempts/1", "pr": "4350", - "status": "in-progress" + "status": "verified", + "date": "2026-10-22" }, "points": [ { @@ -2867,18 +2863,20 @@ export const agentx = { "hicache": { "args": [ ["--enable-hierarchical-cache"], - ["--hicache-size", "160"] + ["--hicache-size", "120"], + ["--hicache-write-policy", "write_through"] ], "donor": "musespark-fp4-b200-sglang-agentic-dflash", - "donorHardware": "b200", - "donorRef": "main" + "donorHardware": "b200" } }, "prov": { "sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/b200-fp4-mtp/agentic.yaml", + "run": "39449248229/attempts/1", "pr": "4350", - "status": "in-progress" + "status": "verified", + "date": "2026-10-22" }, "points": [ { diff --git a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx index 0ade777811..b09c02de67 100644 --- a/docs/src/snippets/agentx/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/agentx/zai-org/glm-5.2.jsx @@ -1,52 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed. export const agentx = { "model": "GLM-5.2", - "generatedOn": "2026-10-20", + "generatedOn": "2026-10-21", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc", + "main": "746150a2786f5cfe6fc474220d4a3335365f14c4", "prs": { - "4344": { + "4389": { "run": null, - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", - "status": "in-progress" - }, - "4342": { - "run": null, - "sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa", - "status": "in-progress" - }, - "4341": { - "run": null, - "sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9", - "status": "in-progress" - }, - "4343": { - "run": null, - "sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27", - "status": "in-progress" - }, - "4350": { - "run": null, - "sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70", + "sha": "e390de596bf5bae7a561308095ad44cf88b74f51", "status": "in-progress" }, "4346": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794", "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", - "status": "in-progress" - }, - "3930": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600", - "sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157", "status": "verified-pr" }, "4333": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "status": "verified-pr" + }, + "4303": { + "run": null, + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", "status": "in-progress" + }, + "3930": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792", + "sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4", + "status": "verified-pr" } } }, @@ -1774,9 +1759,9 @@ export const agentx = { "detail": "MTP 4-1-5", "gpus": 8, "configKey": "glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260805-211ee642", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", "imageStatus": "ok", - "imageCommit": "211ee642", + "imageCommit": null, "submitted": "dynamo", "setup": [ { @@ -1787,10 +1772,8 @@ export const agentx = { ], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], @@ -1803,12 +1786,12 @@ export const agentx = { } }, "prov": { - "sha": "d8e7cc8e22bab51332c18b3f4f1bb19b003a12f5", + "sha": "b16dbcd73117ea2f0bddd9ddc2bbe17e7ada6b51", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/agg-mtp-variants.yaml", - "run": "37506762704/attempts/1", - "pr": "3401", + "run": "38997451681", + "pr": "4227", "status": "verified", - "date": "2026-10-07" + "date": "2026-10-21" }, "points": [ { @@ -1835,7 +1818,7 @@ export const agentx = { ], "routers": { "dynamo": { - "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/71eb001e17fa73c742f0afe1a6ed96836cb135fd"], + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b4540b8767aaa8d98582251bbe9e2ceaba1ed1b1"], "files": ["nats.conf#7d9e9ee1", "nginx.conf#748c9006"], "blocks": [ { @@ -1883,15 +1866,27 @@ export const agentx = { { "diff": { "args": { + "--cuda-graph-max-bs-decode": ["10"], + "--dsa-decode-backend": ["trtllm"], + "--dsa-prefill-backend": ["trtllm"], "--mem-fraction-static": ["0.8"], "--speculative-num-draft-tokens": ["5"], "--speculative-num-steps": ["4"], "--bf16-gemm-backend": null, + "--cuda-graph-max-bs": null, "--hicache-mem-layout": null, + "--nsa-decode-backend": null, + "--nsa-prefill-backend": null, "--watchdog-timeout": null }, "env": { + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DSV4_REASONING_EFFORT": "max", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", "NVSHMEM_REMOTE_TRANSPORT": null, + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": null, + "SGLANG_ENABLE_THINKING": null, + "SGLANG_REASONING_EFFORT": null, "SGLANG_TIMEOUT_KEEP_ALIVE": null, "UCX_TLS": null } @@ -1941,7 +1936,7 @@ export const agentx = { "detail": "MTP 3-1-4", "gpus": 8, "configKey": "glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg", - "image": "lmsysorg/sglang:v0.5.18-cu130@sha256:db37df8aded8fa169f7264404f3c893672f8046609a4a43d15bedcb4f63d692a", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", "imageStatus": "ok", "imageCommit": null, "submitted": "dynamo", @@ -1954,10 +1949,8 @@ export const agentx = { ], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], @@ -1971,12 +1964,12 @@ export const agentx = { } }, "prov": { - "sha": "d8e7cc8e22bab51332c18b3f4f1bb19b003a12f5", + "sha": "2d4231ec49540d6036c836f07b4422c100dc9373", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/gb200-fp4/agentx/disagg-mtp-variants.yaml", - "run": "37506762704/attempts/1", - "pr": "3401", + "run": "39134797422/attempts/1", + "pr": "4277", "status": "verified", - "date": "2026-10-07" + "date": "2026-10-21" }, "points": [ { @@ -1989,7 +1982,7 @@ export const agentx = { ], "routers": { "dynamo": { - "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/71eb001e17fa73c742f0afe1a6ed96836cb135fd"], + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b4540b8767aaa8d98582251bbe9e2ceaba1ed1b1"], "files": ["nats.conf#7d9e9ee1", "nginx.conf#748c9006"], "blocks": [ { @@ -2051,15 +2044,15 @@ export const agentx = { "NCCL_CUMEM_ENABLE": "1", "NCCL_MNNVL_ENABLE": "1", "NVSHMEM_REMOTE_TRANSPORT": "none", - "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_DEFAULT_THINKING": "1", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", - "SGLANG_ENABLE_THINKING": "1", + "SGLANG_DSV4_REASONING_EFFORT": "max", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1", "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "2048", "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", - "SGLANG_REASONING_EFFORT": "max", "SGLANG_TIMEOUT_KEEP_ALIVE": "900", "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800", @@ -2078,11 +2071,13 @@ export const agentx = { ["--bf16-gemm-backend", "cutedsl"], ["--chunked-prefill-size", "32768"], ["--context-length", "1048576"], - ["--cuda-graph-max-bs", "64"], + ["--cuda-graph-max-bs-decode", "64"], ["--data-parallel-size", "1"], ["--disable-cuda-graph"], ["--disable-shared-experts-fusion"], ["--disaggregation-transfer-backend", "nixl"], + ["--dsa-decode-backend", "trtllm"], + ["--dsa-prefill-backend", "trtllm"], ["--enable-flashinfer-allreduce-fusion"], ["--enable-hierarchical-cache"], ["--expert-parallel-size", "1"], @@ -2099,8 +2094,6 @@ export const agentx = { ["--mem-fraction-static", "0.83"], ["--model-loader-extra-config", "{\"enable_multithread_load\": true}"], ["--moe-runner-backend", "flashinfer_trtllm"], - ["--nsa-decode-backend", "trtllm"], - ["--nsa-prefill-backend", "trtllm"], ["--quantization", "modelopt_fp4"], ["--speculative-algorithm", "EAGLE"], ["--speculative-eagle-topk", "1"], @@ -2128,16 +2121,16 @@ export const agentx = { "NVSHMEM_REMOTE_TRANSPORT": "none", "SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "8", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "512", - "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_DEFAULT_THINKING": "1", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "100000", "SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE": "100000", "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "100000", - "SGLANG_ENABLE_THINKING": "1", + "SGLANG_DSV4_REASONING_EFFORT": "max", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "2048", "SGLANG_MOE_NVFP4_DISPATCH": "1", "SGLANG_MOONCAKE_CUSTOM_MEM_POOL": "True", "SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE": "0", - "SGLANG_REASONING_EFFORT": "max", "SGLANG_TIMEOUT_KEEP_ALIVE": "900", "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0", "TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800", @@ -2154,11 +2147,13 @@ export const agentx = { ["--bf16-gemm-backend", "cutedsl"], ["--chunked-prefill-size", "64"], ["--context-length", "1048576"], - ["--cuda-graph-max-bs", "64"], + ["--cuda-graph-max-bs-decode", "64"], ["--data-parallel-size", "1"], ["--disable-radix-cache"], ["--disaggregation-decode-extra-slots", "0"], ["--disaggregation-transfer-backend", "nixl"], + ["--dsa-decode-backend", "trtllm"], + ["--dsa-prefill-backend", "trtllm"], ["--enable-flashinfer-allreduce-fusion"], ["--expert-parallel-size", "1"], ["--fp4-gemm-backend", "flashinfer_trtllm"], @@ -2167,8 +2162,6 @@ export const agentx = { ["--mem-fraction-static", "0.9"], ["--model-loader-extra-config", "{\"enable_multithread_load\": true}"], ["--moe-runner-backend", "flashinfer_trtllm"], - ["--nsa-decode-backend", "trtllm"], - ["--nsa-prefill-backend", "trtllm"], ["--quantization", "modelopt_fp4"], ["--skip-tokenizer-init"], ["--speculative-algorithm", "EAGLE"], @@ -2356,19 +2349,30 @@ export const agentx = { "--max-running-requests": ["16"], "--mem-fraction-static": ["0.8"], "--moe-runner-backend": ["flashinfer_cutlass"], + "--nsa-decode-backend": ["trtllm"], + "--nsa-prefill-backend": ["trtllm"], "--speculative-num-draft-tokens": ["2"], "--speculative-num-steps": ["1"], "--tensor-parallel-size": ["8"], "--kv-events-config": null, "--attn-cp-size": null, "--bf16-gemm-backend": null, + "--cuda-graph-max-bs-decode": null, "--disable-shared-experts-fusion": null, + "--dsa-decode-backend": null, + "--dsa-prefill-backend": null, "--hicache-mem-layout": null, "--max-total-tokens": null, "--watchdog-timeout": null }, "env": { + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_ENABLE_THINKING": "1", + "SGLANG_REASONING_EFFORT": "max", "UCX_TLS": "cuda_copy,cuda_ipc,sm,self,tcp", + "SGLANG_DEFAULT_THINKING": null, + "SGLANG_DSV4_REASONING_EFFORT": null, + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": null, "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": null, "SGLANG_TIMEOUT_KEEP_ALIVE": null } @@ -2382,14 +2386,25 @@ export const agentx = { "--cuda-graph-max-bs": ["16"], "--fp4-gemm-backend": ["flashinfer_cutlass"], "--max-running-requests": ["16"], + "--nsa-decode-backend": ["trtllm"], + "--nsa-prefill-backend": ["trtllm"], "--speculative-num-draft-tokens": ["3"], "--speculative-num-steps": ["2"], "--kv-events-config": null, "--bf16-gemm-backend": null, + "--cuda-graph-max-bs-decode": null, + "--dsa-decode-backend": null, + "--dsa-prefill-backend": null, "--watchdog-timeout": null }, "env": { + "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK": "1", + "SGLANG_ENABLE_THINKING": "1", + "SGLANG_REASONING_EFFORT": "max", "UCX_TLS": "cuda_copy,cuda_ipc,sm,self,tcp", + "SGLANG_DEFAULT_THINKING": null, + "SGLANG_DSV4_REASONING_EFFORT": null, + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": null, "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK": null, "SGLANG_TIMEOUT_KEEP_ALIVE": null } @@ -4329,17 +4344,15 @@ export const agentx = { "detail": "MTP 5-1-6", "gpus": 4, "configKey": "glm5.2-fp4-b200-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", "imageStatus": "ok", - "imageCommit": "07c8f729", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], @@ -4353,12 +4366,12 @@ export const agentx = { } }, "prov": { - "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", + "sha": "97faef328278aade1f46f2bf91ad59e161e068b1", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml", - "run": "39166018772/attempts/1", - "pr": "4286", + "run": "39626904306/attempts/1", + "pr": "4370", "status": "verified", - "date": "2026-10-19" + "date": "2026-10-22" }, "points": [ { @@ -4381,7 +4394,7 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["16"], + "--cuda-graph-max-bs-decode": ["16"], "--max-running-requests": ["16"] } } @@ -4389,7 +4402,7 @@ export const agentx = { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["16"], + "--cuda-graph-max-bs-decode": ["16"], "--max-running-requests": ["16"] } }, @@ -4410,7 +4423,7 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["32"], + "--cuda-graph-max-bs-decode": ["32"], "--max-running-requests": ["32"] } } @@ -4418,7 +4431,7 @@ export const agentx = { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["32"], + "--cuda-graph-max-bs-decode": ["32"], "--max-running-requests": ["32"] } }, @@ -4439,7 +4452,7 @@ export const agentx = { null, { "args": { - "--cuda-graph-max-bs": ["32"], + "--cuda-graph-max-bs-decode": ["32"], "--max-running-requests": ["32"] } } @@ -4447,7 +4460,7 @@ export const agentx = { "sglang": [ { "args": { - "--cuda-graph-max-bs": ["32"], + "--cuda-graph-max-bs-decode": ["32"], "--max-running-requests": ["32"] } }, @@ -4458,7 +4471,7 @@ export const agentx = { ], "routers": { "dynamo": { - "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.5.0.dev20260903\" \"ai-dynamo==1.5.0.dev20260903\""], + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b4540b8767aaa8d98582251bbe9e2ceaba1ed1b1"], "files": ["nats.conf#7d9e9ee1"], "blocks": [ { @@ -4511,7 +4524,7 @@ export const agentx = { ["--request-plane", "tcp"], ["--bf16-gemm-backend", "cutedsl"], ["--chunked-prefill-size", "8192"], - ["--cuda-graph-max-bs", "1"], + ["--cuda-graph-max-bs-decode", "1"], ["--enable-hierarchical-cache"], ["--expert-parallel-size", "1"], ["--hicache-io-backend", "direct"], @@ -4580,17 +4593,15 @@ export const agentx = { "detail": "MTP 5-1-6", "gpus": 8, "configKey": "glm5.2-fp4-b200-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", "imageStatus": "ok", - "imageCommit": "07c8f729", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], @@ -4604,12 +4615,12 @@ export const agentx = { } }, "prov": { - "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", + "sha": "97faef328278aade1f46f2bf91ad59e161e068b1", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml", - "run": "39166018772/attempts/1", - "pr": "4286", + "run": "39626904306/attempts/1", + "pr": "4370", "status": "verified", - "date": "2026-10-19" + "date": "2026-10-22" }, "points": [ { @@ -4622,7 +4633,7 @@ export const agentx = { ], "routers": { "dynamo": { - "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.5.0.dev20260903\" \"ai-dynamo==1.5.0.dev20260903\""], + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b4540b8767aaa8d98582251bbe9e2ceaba1ed1b1"], "files": ["nats.conf#7d9e9ee1"], "blocks": [ { @@ -4664,7 +4675,7 @@ export const agentx = { { "diff": { "args": { - "--cuda-graph-max-bs": ["2"], + "--cuda-graph-max-bs-decode": ["2"], "--max-running-requests": ["2"], "--tensor-parallel-size": ["8"] } @@ -4878,12 +4889,14 @@ export const agentx = { { "diff": { "args": { + "--cuda-graph-max-bs": ["2"], "--ep-size": ["1"], "--hicache-ratio": ["0.75"], "--mem-fraction-static": ["0.83"], "--speculative-num-draft-tokens": ["4"], "--speculative-num-steps": ["3"], "--tp": ["8"], + "--cuda-graph-max-bs-decode": null, "--expert-parallel-size": null, "--hicache-size": null, "--model-loader-extra-config": null, @@ -4952,17 +4965,15 @@ export const agentx = { "detail": "MTP 5-1-6", "gpus": 8, "configKey": "glm5.2-fp4-b200-dynamo-sglang-agentic-agg", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729", + "image": "lmsysorg/sglang:v0.5.22-cu130@sha256:e3a4f06001a29542fc4a8638536d9ffc5eb9c9942b08603c09b19a56eae4a1ae", "imageStatus": "ok", - "imageCommit": "07c8f729", + "imageCommit": null, "submitted": "dynamo", "setup": [], "disclosures": ["synthetic-acceptance"], "kv": { - "available": ["none", "hicache"], - "unavailable": { - "mooncake": "this image's SGLang build predates --enable-unified-cache-external-linker" - }, + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, "hicache": { "args": [ ["--enable-hierarchical-cache"], @@ -4976,12 +4987,12 @@ export const agentx = { } }, "prov": { - "sha": "3f3f6f5c78ec10dff2133e2d0d5c29416e41b1ec", + "sha": "97faef328278aade1f46f2bf91ad59e161e068b1", "recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b200-fp4/agentx/agg-variants.yaml", - "run": "39166018772/attempts/1", - "pr": "4286", + "run": "39626904306/attempts/1", + "pr": "4370", "status": "verified", - "date": "2026-10-19" + "date": "2026-10-22" }, "points": [ { @@ -4994,7 +5005,7 @@ export const agentx = { ], "routers": { "dynamo": { - "install": ["pip install --pre --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.5.0.dev20260903\" \"ai-dynamo==1.5.0.dev20260903\""], + "install": ["# Dynamo built from source at https://github.com/ai-dynamo/dynamo/commit/b4540b8767aaa8d98582251bbe9e2ceaba1ed1b1"], "files": ["nats.conf#7d9e9ee1"], "blocks": [ { @@ -5037,7 +5048,7 @@ export const agentx = { "diff": { "args": { "--chunked-prefill-size": ["65536"], - "--cuda-graph-max-bs": ["16"], + "--cuda-graph-max-bs-decode": ["16"], "--data-parallel-size": ["8"], "--enable-dp-attention": [], "--enable-dp-lm-head": [], @@ -5552,16 +5563,16 @@ export const agentx = { } }, { - "id": "glm5.2-fp8-b200-sglang-agentic-mtp#0", + "id": "glm5.2-fp8-b200-sglang-agentic-mtp#0@c1", "hw": "b200", "ckpt": "zai-org/GLM-5.2-FP8", "label": "Agg TP8", - "detail": "MTP 3-1-4", + "detail": "MTP 6-1-7", "gpus": 8, "configKey": "glm5.2-fp8-b200-sglang-agentic-mtp", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260908-20ca564b", - "imageStatus": "pruned", - "imageCommit": "20ca564b", + "image": "lmsysorg/sglang:nightly-dev-20261020-ab580b2e@sha256:0af411bb320a95bb5e1c58b4c20c9852920a19aa9f12f148f69731b0dca3b576", + "imageStatus": "ok", + "imageCommit": "ab580b2e", "submitted": "sglang", "setup": [], "disclosures": ["synthetic-acceptance"], @@ -5581,136 +5592,18 @@ export const agentx = { } }, "prov": { - "sha": "a30c9cb26d6d2a218ef87aadcaf854123ffcf71e", - "recipe": "benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh", - "run": "35013710860/attempts/1", - "pr": "2863", - "status": "verified", - "date": "2026-09-17" + "sha": "e390de596bf5bae7a561308095ad44cf88b74f51", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml", + "pr": "4389", + "status": "in-progress" }, "points": [ { "concs": [1], "item": 0, - "goldenAL": "2.99", - "kv": "hicache", - "job": "35013710860/job/104532588543" - }, - { - "concs": [4], - "item": 0, - "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs": ["8"], - "--max-running-requests": ["8"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs": ["8"], - "--max-running-requests": ["8"] - } - } - ] - }, - "job": "35013710860/job/104532588624" - }, - { - "concs": [8], - "item": 0, - "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs": ["16"], - "--max-running-requests": ["16"] - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs": ["16"], - "--max-running-requests": ["16"] - } - } - ] - }, - "job": "35013710860/job/104532588912" - }, - { - "concs": [12], - "item": 0, - "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs": ["24"], - "--hicache-size": ["169"], - "--max-running-requests": ["24"], - "--hicache-ratio": null - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs": ["24"], - "--hicache-size": ["169"], - "--max-running-requests": ["24"], - "--hicache-ratio": null - } - } - ] - }, - "job": "35013710860/job/104532588733" - }, - { - "concs": [16], - "item": 0, - "goldenAL": "2.99", - "kv": "hicache", - "patch": { - "sglang": [ - { - "args": { - "--cuda-graph-max-bs": ["32"], - "--hicache-size": ["169"], - "--max-running-requests": ["32"], - "--hicache-ratio": null - } - } - ], - "dynamo": [ - null, - null, - { - "args": { - "--cuda-graph-max-bs": ["32"], - "--hicache-size": ["169"], - "--max-running-requests": ["32"], - "--hicache-ratio": null - } - } - ] - }, - "job": "35013710860/job/104532588655" + "override": "override_c1", + "goldenAL": "3.78", + "kv": "hicache" } ], "routers": { @@ -5736,9 +5629,9 @@ export const agentx = { ["--port", "30000"], ["--bf16-gemm-backend", "cutedsl"], ["--chunked-prefill-size", "8192"], - ["--cuda-graph-max-bs", "2"], + ["--cuda-graph-max-bs-decode", "2"], ["--enable-hierarchical-cache"], - ["--ep-size", "1"], + ["--expert-parallel-size", "1"], ["--hicache-io-backend", "direct"], ["--hicache-mem-layout", "page_first_direct"], ["--hicache-ratio", "0.75"], @@ -5747,16 +5640,18 @@ export const agentx = { ["--max-prefill-tokens", "8192"], ["--max-running-requests", "2"], ["--mem-fraction-static", "0.83"], + ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], ["--quantization", "fp8"], ["--reasoning-parser", "glm45"], ["--speculative-algorithm", "EAGLE"], ["--speculative-eagle-topk", "1"], - ["--speculative-num-draft-tokens", "4"], - ["--speculative-num-steps", "3"], + ["--speculative-num-draft-tokens", "7"], + ["--speculative-num-steps", "6"], + ["--tensor-parallel-size", "8"], ["--tool-call-parser", "glm47"], - ["--tp", "8"], ["--trust-remote-code"], - ["--watchdog-timeout", "1800"] + ["--watchdog-timeout", "1800"], + ["--weight-loader-prefetch-checkpoints"] ] } ] @@ -5806,6 +5701,389 @@ export const agentx = { } } }, + { + "id": "glm5.2-fp8-b200-sglang-agentic-mtp#0@c4", + "hw": "b200", + "ckpt": "zai-org/GLM-5.2-FP8", + "label": "Agg TP8", + "detail": "MTP 5-1-6", + "gpus": 8, + "configKey": "glm5.2-fp8-b200-sglang-agentic-mtp", + "image": "lmsysorg/sglang:nightly-dev-20261020-ab580b2e@sha256:0af411bb320a95bb5e1c58b4c20c9852920a19aa9f12f148f69731b0dca3b576", + "imageStatus": "ok", + "imageCommit": "ab580b2e", + "submitted": "sglang", + "setup": [], + "disclosures": ["synthetic-acceptance"], + "kv": { + "available": ["none", "hicache", "mooncake"], + "unavailable": {}, + "hicache": { + "args": [ + ["--enable-hierarchical-cache"], + ["--hicache-io-backend", "direct"], + ["--hicache-mem-layout", "page_first_direct"], + ["--hicache-size", "170"], + ["--hicache-write-policy", "write_back"] + ], + "donor": "glm5.2-fp4-b200-dynamo-sglang-agentic-agg", + "donorHardware": "b200" + } + }, + "prov": { + "sha": "e390de596bf5bae7a561308095ad44cf88b74f51", + "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/sglang/b200-fp8-mtp/agentic.yaml", + "pr": "4389", + "status": "in-progress" + }, + "points": [ + { + "concs": [4], + "item": 0, + "override": "override_c4", + "goldenAL": "3.61", + "kv": "hicache" + }, + { + "concs": [8], + "item": 0, + "override": "override_c8", + "goldenAL": "3.61", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["16"], + "--max-running-requests": ["16"] + } + } + ] + } + }, + { + "concs": [12], + "item": 0, + "override": "override_c12", + "goldenAL": "3.61", + "kv": "hicache", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["24"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["24"], + "--max-running-requests": ["24"] + } + } + ] + } + }, + { + "concs": [16], + "item": 1, + "override": "override_c16", + "goldenAL": "3.61", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["32"], + "--dcp-size": ["8"], + "--enable-prefill-delayer": [], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["32"], + "--prefill-delayer-max-delay-ms": ["800"], + "--prefill-delayer-max-delay-passes": ["32"], + "--prefill-delayer-queue-min-ratio": ["0.5"], + "--prefill-max-requests": ["2"], + "--enable-hierarchical-cache": null, + "--hicache-io-backend": null, + "--hicache-mem-layout": null, + "--hicache-size": null, + "--hicache-write-policy": null + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["32"], + "--dcp-size": ["8"], + "--enable-prefill-delayer": [], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["32"], + "--prefill-delayer-max-delay-ms": ["800"], + "--prefill-delayer-max-delay-passes": ["32"], + "--prefill-delayer-queue-min-ratio": ["0.5"], + "--prefill-max-requests": ["2"], + "--enable-hierarchical-cache": null, + "--hicache-io-backend": null, + "--hicache-mem-layout": null, + "--hicache-size": null, + "--hicache-write-policy": null + } + } + ] + } + }, + { + "concs": [17], + "item": 1, + "override": "override_c17", + "goldenAL": "3.61", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["32"], + "--dcp-size": ["8"], + "--enable-prefill-delayer": [], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.83"], + "--prefill-delayer-max-delay-ms": ["750"], + "--prefill-delayer-max-delay-passes": ["40"], + "--prefill-delayer-queue-min-ratio": ["0.5"], + "--prefill-max-requests": ["3"], + "--enable-hierarchical-cache": null, + "--hicache-io-backend": null, + "--hicache-mem-layout": null, + "--hicache-size": null, + "--hicache-write-policy": null + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["32"], + "--dcp-size": ["8"], + "--enable-prefill-delayer": [], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.83"], + "--prefill-delayer-max-delay-ms": ["750"], + "--prefill-delayer-max-delay-passes": ["40"], + "--prefill-delayer-queue-min-ratio": ["0.5"], + "--prefill-max-requests": ["3"], + "--enable-hierarchical-cache": null, + "--hicache-io-backend": null, + "--hicache-mem-layout": null, + "--hicache-size": null, + "--hicache-write-policy": null + } + } + ] + } + }, + { + "concs": [18], + "item": 1, + "override": "override_c18", + "goldenAL": "3.61", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["32"], + "--dcp-size": ["8"], + "--enable-prefill-delayer": [], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.83"], + "--prefill-delayer-max-delay-ms": ["750"], + "--prefill-delayer-max-delay-passes": ["40"], + "--prefill-delayer-queue-min-ratio": ["0.5"], + "--prefill-max-requests": ["3"], + "--enable-hierarchical-cache": null, + "--hicache-io-backend": null, + "--hicache-mem-layout": null, + "--hicache-size": null, + "--hicache-write-policy": null + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["32"], + "--dcp-size": ["8"], + "--enable-prefill-delayer": [], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.83"], + "--prefill-delayer-max-delay-ms": ["750"], + "--prefill-delayer-max-delay-passes": ["40"], + "--prefill-delayer-queue-min-ratio": ["0.5"], + "--prefill-max-requests": ["3"], + "--enable-hierarchical-cache": null, + "--hicache-io-backend": null, + "--hicache-mem-layout": null, + "--hicache-size": null, + "--hicache-write-policy": null + } + } + ] + } + }, + { + "concs": [24], + "item": 1, + "override": "override_c24", + "goldenAL": "3.61", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["32"], + "--dcp-size": ["8"], + "--enable-prefill-delayer": [], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.83"], + "--prefill-delayer-max-delay-ms": ["800"], + "--prefill-delayer-max-delay-passes": ["32"], + "--prefill-delayer-queue-min-ratio": ["0.5"], + "--prefill-max-requests": ["2"], + "--enable-hierarchical-cache": null, + "--hicache-io-backend": null, + "--hicache-mem-layout": null, + "--hicache-size": null, + "--hicache-write-policy": null + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--chunked-prefill-size": ["16384"], + "--cuda-graph-max-bs-decode": ["32"], + "--dcp-size": ["8"], + "--enable-prefill-delayer": [], + "--max-prefill-tokens": ["16384"], + "--max-running-requests": ["32"], + "--mem-fraction-static": ["0.83"], + "--prefill-delayer-max-delay-ms": ["800"], + "--prefill-delayer-max-delay-passes": ["32"], + "--prefill-delayer-queue-min-ratio": ["0.5"], + "--prefill-max-requests": ["2"], + "--enable-hierarchical-cache": null, + "--hicache-io-backend": null, + "--hicache-mem-layout": null, + "--hicache-size": null, + "--hicache-write-policy": null + } + } + ] + } + } + ], + "routers": { + "sglang": { + "install": [], + "files": [], + "blocks": [ + { + "diff": { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--hicache-size": ["169"], + "--hicache-write-policy": ["write_through"], + "--max-running-requests": ["8"], + "--mem-fraction-static": ["0.88"], + "--speculative-num-draft-tokens": ["6"], + "--speculative-num-steps": ["5"], + "--hicache-ratio": null + } + }, + "ref": [27, "sglang", 0] + } + ] + }, + "dynamo": { + "install": [], + "files": [], + "blocks": [ + { + "kind": "etcd", + "count_nodes": 1, + "cmd": "etcd", + "env": {}, + "args": [ + ["--data-dir", "/tmp/etcd"], + ["--listen-client-urls", "http://0.0.0.0:2379"], + ["--advertise-client-urls", "http://$ETCD_IP:2379"] + ] + }, + { + "kind": "dynamo-frontend", + "count": 1, + "cmd": "python3 -m dynamo.frontend", + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + }, + "args": [ + ["--http-port", "8000"], + ["--router-mode", "kv"] + ] + }, + { + "from": 0, + "cmd": "python3 -m dynamo.sglang", + "diff": { + "args": { + "--request-plane": ["tcp"] + }, + "env": { + "DYN_REQUEST_PLANE": "tcp", + "ETCD_ENDPOINTS": "http://$ETCD_IP:2379" + } + } + } + ] + } + } + }, { "id": "glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg#0", "hw": "h200", @@ -6446,7 +6724,7 @@ export const agentx = { "--page-size": null } }, - "ref": [29, "dynamo", 4] + "ref": [30, "dynamo", 4] }, { "workers": 2, @@ -6456,7 +6734,7 @@ export const agentx = { "--speculative-num-steps": ["3"] } }, - "ref": [29, "dynamo", 5] + "ref": [30, "dynamo", 5] } ] }, diff --git a/docs/src/snippets/agentx/zai-org/glm-5.3.jsx b/docs/src/snippets/agentx/zai-org/glm-5.3.jsx index 2497a3e468..55b5044244 100644 --- a/docs/src/snippets/agentx/zai-org/glm-5.3.jsx +++ b/docs/src/snippets/agentx/zai-org/glm-5.3.jsx @@ -1,52 +1,37 @@ // GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand. // Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions -// (SemiAnalysisAI/InferenceX main@6046414d2d1a plus open PRs #3930, #4333, #4341, #4342, #4343, #4344, #4346, #4350), with SA-specific settings removed. +// (SemiAnalysisAI/InferenceX main@746150a2786f plus open PRs #3930, #4303, #4333, #4346, #4389), with SA-specific settings removed. export const agentx = { "model": "GLM-5.3", - "generatedOn": "2026-10-20", + "generatedOn": "2026-10-21", "source": { "repo": "https://github.com/SemiAnalysisAI/InferenceX", - "main": "6046414d2d1ad2fb17aa857f01fd3146f429b4fc", + "main": "746150a2786f5cfe6fc474220d4a3335365f14c4", "prs": { - "4344": { + "4389": { "run": null, - "sha": "cc9338f6a37f977b8e931011eb4d17febf78fe64", - "status": "in-progress" - }, - "4342": { - "run": null, - "sha": "be8cdbac6be4bcdf9b1c4496760354099f1e0aaa", - "status": "in-progress" - }, - "4341": { - "run": null, - "sha": "37ce29641296bf4e64eaf8e121e17a4f04d4e5b9", - "status": "in-progress" - }, - "4343": { - "run": null, - "sha": "bdfb83f2e0d95441eebd2b1dfb5928b76d8d0f27", - "status": "in-progress" - }, - "4350": { - "run": null, - "sha": "5e7fa9201892aee7f037806bab0e1a3b83daef70", + "sha": "e390de596bf5bae7a561308095ad44cf88b74f51", "status": "in-progress" }, "4346": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39534932794", "sha": "566c6e105131c0cbfd4b668b51cbc8a861336397", - "status": "in-progress" - }, - "3930": { - "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39462624600", - "sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157", "status": "verified-pr" }, "4333": { - "run": null, + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39355392627", "sha": "02b72dc0bf88eca6f6638300739732d4c7640ddf", + "status": "verified-pr" + }, + "4303": { + "run": null, + "sha": "a0e5be74248f09552f88904f90dc71e8f190a3e9", "status": "in-progress" + }, + "3930": { + "run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39551262792", + "sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4", + "status": "verified-pr" } } }, @@ -155,41 +140,155 @@ export const agentx = { } }, "prov": { - "sha": "eee5985fcce4b74b0443bcbfd2cd4193c3e24157", + "sha": "c093d2052d2e4f6aeaa8162e9e81219f31556fe4", "recipe": "inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.3/sglang/b300-fp4-mtp/agentic.yaml", - "run": "39462624600", + "run": "39551262792", "pr": "3930", - "comment": "6300890848", + "comment": "6330763860", "status": "verified-pr" }, "points": [ + { + "concs": [1], + "item": 0, + "override": "override_c1", + "goldenAL": "2.99", + "kv": "none" + }, + { + "concs": [4], + "item": 0, + "override": "override_c4", + "goldenAL": "2.99", + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["8"], + "--max-running-requests": ["8"] + } + } + ] + } + }, { "concs": [8], "item": 0, "override": "override_c8", "goldenAL": "2.99", - "kv": "none" + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["256"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["256"] + } + } + ] + } }, { "concs": [10], "item": 0, "override": "override_c10", "goldenAL": "2.99", - "kv": "none" + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["256"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["256"] + } + } + ] + } }, { "concs": [13], "item": 0, "override": "override_c13", "goldenAL": "2.99", - "kv": "none" + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["256"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["256"] + } + } + ] + } }, { "concs": [14], "item": 0, "override": "override_c14", "goldenAL": "2.99", - "kv": "none" + "kv": "none", + "patch": { + "sglang": [ + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["256"] + } + } + ], + "dynamo": [ + null, + null, + { + "args": { + "--cuda-graph-max-bs-decode": ["64"], + "--max-running-requests": ["256"] + } + } + ] + } }, { "concs": [22], @@ -201,11 +300,13 @@ export const agentx = { "sglang": [ { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--enable-hierarchical-cache": [], "--hicache-io-backend": ["direct"], "--hicache-mem-layout": ["page_first_direct"], "--hicache-size": ["300"], "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["256"], "--enable-torch-symm-mem": null }, "env": { @@ -218,11 +319,13 @@ export const agentx = { null, { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--enable-hierarchical-cache": [], "--hicache-io-backend": ["direct"], "--hicache-mem-layout": ["page_first_direct"], "--hicache-size": ["300"], "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["256"], "--enable-torch-symm-mem": null }, "env": { @@ -242,11 +345,13 @@ export const agentx = { "sglang": [ { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--enable-hierarchical-cache": [], "--hicache-io-backend": ["direct"], "--hicache-mem-layout": ["page_first_direct"], "--hicache-size": ["300"], "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["256"], "--enable-torch-symm-mem": null }, "env": { @@ -259,11 +364,13 @@ export const agentx = { null, { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--enable-hierarchical-cache": [], "--hicache-io-backend": ["direct"], "--hicache-mem-layout": ["page_first_direct"], "--hicache-size": ["300"], "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["256"], "--enable-torch-symm-mem": null }, "env": { @@ -283,11 +390,13 @@ export const agentx = { "sglang": [ { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--enable-hierarchical-cache": [], "--hicache-io-backend": ["direct"], "--hicache-mem-layout": ["page_first_direct"], "--hicache-size": ["300"], "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["256"], "--enable-torch-symm-mem": null }, "env": { @@ -300,11 +409,13 @@ export const agentx = { null, { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--enable-hierarchical-cache": [], "--hicache-io-backend": ["direct"], "--hicache-mem-layout": ["page_first_direct"], "--hicache-size": ["300"], "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["256"], "--enable-torch-symm-mem": null }, "env": { @@ -324,11 +435,13 @@ export const agentx = { "sglang": [ { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--enable-hierarchical-cache": [], "--hicache-io-backend": ["direct"], "--hicache-mem-layout": ["page_first_direct"], "--hicache-size": ["300"], "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["256"], "--enable-torch-symm-mem": null }, "env": { @@ -341,11 +454,13 @@ export const agentx = { null, { "args": { + "--cuda-graph-max-bs-decode": ["64"], "--enable-hierarchical-cache": [], "--hicache-io-backend": ["direct"], "--hicache-mem-layout": ["page_first_direct"], "--hicache-size": ["300"], "--hicache-write-policy": ["write_back"], + "--max-running-requests": ["256"], "--enable-torch-symm-mem": null }, "env": { @@ -379,11 +494,11 @@ export const agentx = { ["--port", "30000"], ["--chunked-prefill-size", "8192"], ["--context-length", "1048576"], - ["--cuda-graph-max-bs-decode", "64"], + ["--cuda-graph-max-bs-decode", "2"], ["--enable-torch-symm-mem"], ["--expert-parallel-size", "1"], ["--max-prefill-tokens", "8192"], - ["--max-running-requests", "256"], + ["--max-running-requests", "2"], ["--mem-fraction-static", "0.86"], ["--model-loader-extra-config", "{\"enable_multithread_load\":true}"], ["--quantization", "modelopt_fp4"],