[Doc] cookbook: Add SemiAnalysis AgentX launches to Muse Spark Open, refresh DeepSeek-V4.1-Flash (#44654)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
5117bfbaeb
commit
fd4b75c696
@@ -25,7 +25,7 @@ Choose your **Hardware Platform**, **Strategy**, and **Quantization** below. The
|
||||
import { Deployment } from "/src/snippets/_deployment.jsx";
|
||||
import { config } from "/src/snippets/configs/meta-models/muse-spark.jsx";
|
||||
|
||||
<Deployment config={config} />
|
||||
<Deployment config={config} agenticLink="agentic-long-context-deployment" />
|
||||
|
||||
## Playground
|
||||
|
||||
@@ -35,6 +35,19 @@ import { Playground } from "/src/snippets/_playground.jsx";
|
||||
|
||||
<Playground config={config} />
|
||||
|
||||
## Agentic Long-Context Deployment
|
||||
|
||||
Agentic workloads, such as coding agents and other multi-turn, tool-using assistants, send long prompts whose prefixes grow and repeat from turn to turn. Serving them well depends on reusing the KV cache across turns, offloading it beyond GPU memory, and keeping each session on the workers that already hold its prefix. This section gives complete launch commands for such deployments, each benchmarked in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) and linked to the run that measured it. Pick a hardware platform, checkpoint, deployment shape, concurrency, KV cache offloading mechanism and router to get every process of the deployment as a raw command, one tab per process in launch order.
|
||||
|
||||
- **Router.** Each configuration was submitted behind one front door: Dynamo (`dynamo.frontend` with `dynamo.sglang` workers) or SGLang (`sglang.launch_server`, behind `sglang_router` when there are several workers or DP-attention ranks).
|
||||
- **KV offload.** Choose among these KV cache offloading mechanisms: none (GPU KV only), HiCache (host DRAM), or a Mooncake store attached through the external linker. The Mooncake option needs an image whose SGLang build includes `--enable-unified-cache-external-linker`.
|
||||
- **Container image.** Run the commands inside the image listed for each configuration; flag names follow that image.
|
||||
|
||||
import { AgentX } from "/src/snippets/_agentx.jsx";
|
||||
import { agentx } from "/src/snippets/agentx/meta-models/muse-spark-open.jsx";
|
||||
|
||||
<AgentX data={agentx} />
|
||||
|
||||
## 1. Model introduction
|
||||
|
||||
This cookbook provides deployment recipes for the Muse Spark Open 816B-A42B checkpoints:
|
||||
|
||||
@@ -1,21 +1,36 @@
|
||||
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
|
||||
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
|
||||
// (SemiAnalysisAI/InferenceX main@6fff051b9f0d plus open PRs #3927, #3953), with SA-specific settings removed.
|
||||
// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed.
|
||||
export const agentx = {
|
||||
"model": "Qwen3.5",
|
||||
"generatedOn": "2026-10-13",
|
||||
"generatedOn": "2026-10-15",
|
||||
"source": {
|
||||
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
|
||||
"main": "6fff051b9f0dd578b1bd0a11dc84dfc502c31e5d",
|
||||
"main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff",
|
||||
"prs": {
|
||||
"3953": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38356572105",
|
||||
"sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125",
|
||||
"4146": {
|
||||
"run": null,
|
||||
"sha": "a23546bea6a5f1d34b75261728ac110403a86540",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4141": {
|
||||
"run": null,
|
||||
"sha": "d3850c4581d03e908bb0a27c39830185a4591465",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4112": {
|
||||
"run": null,
|
||||
"sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4111": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474",
|
||||
"sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae",
|
||||
"status": "verified-pr"
|
||||
},
|
||||
"3927": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38266927261",
|
||||
"sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78",
|
||||
"4094": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139",
|
||||
"sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac",
|
||||
"status": "verified-pr"
|
||||
}
|
||||
}
|
||||
@@ -7393,6 +7408,188 @@ export const agentx = {
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#4",
|
||||
"hw": "b300",
|
||||
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
|
||||
"label": "Agg TP4",
|
||||
"detail": "MTP 6-1-7",
|
||||
"gpus": 4,
|
||||
"configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto",
|
||||
"image": "lmsysorg/sglang:nightly-dev-20261013-7224fce1",
|
||||
"imageStatus": "ok",
|
||||
"imageCommit": "7224fce1",
|
||||
"submitted": "sglang",
|
||||
"setup": [],
|
||||
"disclosures": ["synthetic-acceptance"],
|
||||
"kv": {
|
||||
"available": ["none", "hicache", "mooncake"],
|
||||
"unavailable": {},
|
||||
"hicache": {
|
||||
"args": [
|
||||
["--enable-hierarchical-cache"],
|
||||
["--hicache-io-backend", "kernel"],
|
||||
["--hicache-mem-layout", "page_first"],
|
||||
["--hicache-size", "144"],
|
||||
["--hicache-write-policy", "write_through_selective"]
|
||||
],
|
||||
"donor": "qwen3.5-fp4-b300-sglang-agentic-mtp",
|
||||
"donorHardware": "b300"
|
||||
}
|
||||
},
|
||||
"prov": {
|
||||
"sha": "836a0c38ae5f14b56ce40307a0f3f619f96846cc",
|
||||
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml",
|
||||
"run": "38390415472/attempts/1",
|
||||
"pr": "2978",
|
||||
"status": "verified",
|
||||
"date": "2026-10-14"
|
||||
},
|
||||
"points": [
|
||||
{
|
||||
"concs": [4],
|
||||
"item": 4,
|
||||
"override": "base",
|
||||
"goldenAL": "4.8",
|
||||
"kv": "hicache"
|
||||
}
|
||||
],
|
||||
"routers": {
|
||||
"sglang": {
|
||||
"install": [],
|
||||
"files": [],
|
||||
"blocks": [
|
||||
{
|
||||
"kind": "worker",
|
||||
"role": "agg",
|
||||
"workers": 1,
|
||||
"nodes_per_worker": 1,
|
||||
"cmd": "python3 -m sglang.launch_server",
|
||||
"env": {
|
||||
"FLASHINFER_DISABLE_VERSION_CHECK": "1",
|
||||
"NCCL_CUMEM_ENABLE": "1",
|
||||
"NCCL_MNNVL_ENABLE": "1",
|
||||
"NCCL_NVLS_ENABLE": "0",
|
||||
"PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
|
||||
"SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION": "0",
|
||||
"SGLANG_ENABLE_JIT_DEEPGEMM": "true",
|
||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
|
||||
"SGLANG_FLASHINFER_AUTOTUNE_CACHE": "1",
|
||||
"SGLANG_HEALTH_CHECK_TIMEOUT": "1800",
|
||||
"SGLANG_MOE_NVFP4_DISPATCH": "1",
|
||||
"SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "1",
|
||||
"SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "1",
|
||||
"SGLANG_USE_MESSAGE_QUEUE_BROADCASTER": "0",
|
||||
"TORCH_DISTRIBUTED_DEFAULT_TIMEOUT": "1800"
|
||||
},
|
||||
"args": [
|
||||
["--model-path", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
|
||||
["--served-model-name", "nvidia/Qwen3.5-397B-A17B-NVFP4"],
|
||||
["--host", "0.0.0.0"],
|
||||
["--port", "30000"],
|
||||
["--attention-backend", "trtllm_mha"],
|
||||
["--chunked-prefill-size", "8192"],
|
||||
["--context-length", "262144"],
|
||||
["--cuda-graph-bs-decode", "1"],
|
||||
["--cuda-graph-max-bs-decode", "1"],
|
||||
["--data-parallel-size", "1"],
|
||||
["--disable-attn-tp-gather"],
|
||||
["--disable-prefill-cuda-graph"],
|
||||
["--disable-shared-experts-fusion"],
|
||||
["--enable-hierarchical-cache"],
|
||||
["--enable-linear-replayssm-spec"],
|
||||
["--expert-parallel-size", "1"],
|
||||
["--hicache-mem-layout", "page_first_direct"],
|
||||
["--hicache-size", "32"],
|
||||
["--hicache-write-policy", "write_back"],
|
||||
["--kv-cache-dtype", "fp8_e4m3"],
|
||||
["--linear-attn-decode-backend", "triton"],
|
||||
["--mamba-max-states-per-path", "3"],
|
||||
["--mamba-radix-cache-strategy", "extra_buffer"],
|
||||
["--mamba-ssm-dtype", "bfloat16"],
|
||||
["--mamba-track-interval", "8192"],
|
||||
["--max-mamba-cache-size", "512"],
|
||||
["--max-prefill-tokens", "8192"],
|
||||
["--max-running-requests", "1"],
|
||||
["--mem-fraction-static", "0.85"],
|
||||
["--moe-a2a-backend", "none"],
|
||||
["--moe-runner-backend", "flashinfer_cutedsl"],
|
||||
["--page-size", "64"],
|
||||
["--pipeline-parallel-size", "1"],
|
||||
["--pp-max-micro-batch-size", "1"],
|
||||
["--prefill-max-requests", "1"],
|
||||
["--quantization", "modelopt_fp4"],
|
||||
["--speculative-algorithm", "NEXTN"],
|
||||
["--speculative-eagle-topk", "1"],
|
||||
["--speculative-moe-a2a-backend", "none"],
|
||||
["--speculative-moe-runner-backend", "flashinfer_cutedsl"],
|
||||
["--speculative-num-draft-tokens", "7"],
|
||||
["--speculative-num-steps", "6"],
|
||||
["--stream-interval", "20"],
|
||||
["--tensor-parallel-size", "4"],
|
||||
["--trust-remote-code"],
|
||||
["--watchdog-timeout", "1000000"],
|
||||
["--weight-loader-prefetch-checkpoints"],
|
||||
["--weight-loader-prefetch-num-threads", "4"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"kind": "sglang-router",
|
||||
"cmd": "python3 -m sglang_router.launch_router",
|
||||
"env": {},
|
||||
"args": [
|
||||
["--host", "0.0.0.0"],
|
||||
["--port", "8000"],
|
||||
["--worker-urls", "http://127.0.0.1:30000"]
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dynamo": {
|
||||
"install": [],
|
||||
"files": [],
|
||||
"blocks": [
|
||||
{
|
||||
"kind": "etcd",
|
||||
"count_nodes": 1,
|
||||
"cmd": "etcd",
|
||||
"env": {},
|
||||
"args": [
|
||||
["--data-dir", "/tmp/etcd"],
|
||||
["--listen-client-urls", "http://0.0.0.0:2379"],
|
||||
["--advertise-client-urls", "http://$ETCD_IP:2379"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"kind": "dynamo-frontend",
|
||||
"count": 1,
|
||||
"cmd": "python3 -m dynamo.frontend",
|
||||
"env": {
|
||||
"DYN_REQUEST_PLANE": "tcp",
|
||||
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
|
||||
},
|
||||
"args": [
|
||||
["--http-port", "8000"],
|
||||
["--router-mode", "kv"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"from": 0,
|
||||
"cmd": "python3 -m dynamo.sglang",
|
||||
"diff": {
|
||||
"args": {
|
||||
"--request-plane": ["tcp"]
|
||||
},
|
||||
"env": {
|
||||
"DYN_REQUEST_PLANE": "tcp",
|
||||
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "qwen3.5-fp4-b300-sglang-agentic-mtp#2",
|
||||
"hw": "b300",
|
||||
@@ -7724,6 +7921,438 @@ export const agentx = {
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#1",
|
||||
"hw": "b300",
|
||||
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
|
||||
"label": "Agg TP2",
|
||||
"detail": "MTP 7-1-8",
|
||||
"gpus": 2,
|
||||
"configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto",
|
||||
"image": "lmsysorg/sglang:nightly-dev-20261013-7224fce1",
|
||||
"imageStatus": "ok",
|
||||
"imageCommit": "7224fce1",
|
||||
"submitted": "sglang",
|
||||
"setup": [],
|
||||
"disclosures": ["synthetic-acceptance"],
|
||||
"kv": {
|
||||
"available": ["none", "hicache", "mooncake"],
|
||||
"unavailable": {},
|
||||
"hicache": {
|
||||
"args": [
|
||||
["--enable-hierarchical-cache"],
|
||||
["--hicache-io-backend", "kernel"],
|
||||
["--hicache-mem-layout", "page_first"],
|
||||
["--hicache-size", "144"],
|
||||
["--hicache-write-policy", "write_through_selective"]
|
||||
],
|
||||
"donor": "qwen3.5-fp4-b300-sglang-agentic-mtp",
|
||||
"donorHardware": "b300"
|
||||
}
|
||||
},
|
||||
"prov": {
|
||||
"sha": "836a0c38ae5f14b56ce40307a0f3f619f96846cc",
|
||||
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml",
|
||||
"run": "38390415472/attempts/1",
|
||||
"pr": "2978",
|
||||
"status": "verified",
|
||||
"date": "2026-10-14"
|
||||
},
|
||||
"points": [
|
||||
{
|
||||
"concs": [36],
|
||||
"item": 2,
|
||||
"override": "zip_override_running_cap[0]",
|
||||
"goldenAL": "5.04",
|
||||
"kv": "hicache"
|
||||
},
|
||||
{
|
||||
"concs": [44],
|
||||
"item": 1,
|
||||
"override": "zip_override_running_cap[2]",
|
||||
"goldenAL": "5.04",
|
||||
"kv": "hicache",
|
||||
"patch": {
|
||||
"sglang": [
|
||||
{
|
||||
"args": {
|
||||
"--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8"],
|
||||
"--cuda-graph-max-bs-decode": ["8"],
|
||||
"--max-running-requests": ["8"],
|
||||
"--pp-max-micro-batch-size": ["8"],
|
||||
"--prefill-max-requests": ["8"]
|
||||
}
|
||||
},
|
||||
null
|
||||
],
|
||||
"dynamo": [
|
||||
null,
|
||||
null,
|
||||
{
|
||||
"args": {
|
||||
"--cuda-graph-bs-decode": ["1", "2", "3", "4", "5", "6", "7", "8"],
|
||||
"--cuda-graph-max-bs-decode": ["8"],
|
||||
"--max-running-requests": ["8"],
|
||||
"--pp-max-micro-batch-size": ["8"],
|
||||
"--prefill-max-requests": ["8"]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"routers": {
|
||||
"sglang": {
|
||||
"install": [],
|
||||
"files": [],
|
||||
"blocks": [
|
||||
{
|
||||
"diff": {
|
||||
"args": {
|
||||
"--cuda-graph-bs-decode": ["1", "2", "3", "4"],
|
||||
"--cuda-graph-max-bs-decode": ["4"],
|
||||
"--hicache-size": ["128"],
|
||||
"--max-mamba-cache-size": ["1536"],
|
||||
"--max-running-requests": ["4"],
|
||||
"--moe-runner-backend": ["flashinfer_trtllm"],
|
||||
"--pp-max-micro-batch-size": ["4"],
|
||||
"--prefill-decode-interval": ["0"],
|
||||
"--prefill-max-requests": ["4"],
|
||||
"--speculative-moe-runner-backend": ["flashinfer_trtllm"],
|
||||
"--speculative-num-draft-tokens": ["8"],
|
||||
"--speculative-num-steps": ["7"],
|
||||
"--tensor-parallel-size": ["2"],
|
||||
"--disable-prefill-cuda-graph": null
|
||||
}
|
||||
},
|
||||
"ref": [33, "sglang", 0]
|
||||
},
|
||||
{
|
||||
"kind": "sglang-router",
|
||||
"cmd": "python3 -m sglang_router.launch_router",
|
||||
"env": {},
|
||||
"args": [
|
||||
["--host", "0.0.0.0"],
|
||||
["--port", "8000"],
|
||||
["--worker-urls", "http://127.0.0.1:30000"]
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dynamo": {
|
||||
"install": [],
|
||||
"files": [],
|
||||
"blocks": [
|
||||
{
|
||||
"kind": "etcd",
|
||||
"count_nodes": 1,
|
||||
"cmd": "etcd",
|
||||
"env": {},
|
||||
"args": [
|
||||
["--data-dir", "/tmp/etcd"],
|
||||
["--listen-client-urls", "http://0.0.0.0:2379"],
|
||||
["--advertise-client-urls", "http://$ETCD_IP:2379"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"kind": "dynamo-frontend",
|
||||
"count": 1,
|
||||
"cmd": "python3 -m dynamo.frontend",
|
||||
"env": {
|
||||
"DYN_REQUEST_PLANE": "tcp",
|
||||
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
|
||||
},
|
||||
"args": [
|
||||
["--http-port", "8000"],
|
||||
["--router-mode", "kv"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"from": 0,
|
||||
"cmd": "python3 -m dynamo.sglang",
|
||||
"diff": {
|
||||
"args": {
|
||||
"--request-plane": ["tcp"]
|
||||
},
|
||||
"env": {
|
||||
"DYN_REQUEST_PLANE": "tcp",
|
||||
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#3",
|
||||
"hw": "b300",
|
||||
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
|
||||
"label": "Agg TP2",
|
||||
"detail": "MTP 5-1-6",
|
||||
"gpus": 2,
|
||||
"configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto",
|
||||
"image": "lmsysorg/sglang:nightly-dev-20261013-7224fce1",
|
||||
"imageStatus": "ok",
|
||||
"imageCommit": "7224fce1",
|
||||
"submitted": "sglang",
|
||||
"setup": [],
|
||||
"disclosures": ["synthetic-acceptance"],
|
||||
"kv": {
|
||||
"available": ["none", "hicache", "mooncake"],
|
||||
"unavailable": {},
|
||||
"hicache": {
|
||||
"args": [
|
||||
["--enable-hierarchical-cache"],
|
||||
["--hicache-io-backend", "kernel"],
|
||||
["--hicache-mem-layout", "page_first"],
|
||||
["--hicache-size", "144"],
|
||||
["--hicache-write-policy", "write_through_selective"]
|
||||
],
|
||||
"donor": "qwen3.5-fp4-b300-sglang-agentic-mtp",
|
||||
"donorHardware": "b300"
|
||||
}
|
||||
},
|
||||
"prov": {
|
||||
"sha": "836a0c38ae5f14b56ce40307a0f3f619f96846cc",
|
||||
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml",
|
||||
"run": "38390415472/attempts/1",
|
||||
"pr": "2978",
|
||||
"status": "verified",
|
||||
"date": "2026-10-14"
|
||||
},
|
||||
"points": [
|
||||
{
|
||||
"concs": [40],
|
||||
"item": 3,
|
||||
"goldenAL": "4.41",
|
||||
"kv": "hicache"
|
||||
}
|
||||
],
|
||||
"routers": {
|
||||
"sglang": {
|
||||
"install": [],
|
||||
"files": [],
|
||||
"blocks": [
|
||||
{
|
||||
"diff": {
|
||||
"args": {
|
||||
"--cuda-graph-bs-decode": ["1", "2"],
|
||||
"--cuda-graph-max-bs-decode": ["2"],
|
||||
"--max-running-requests": ["2"],
|
||||
"--pp-max-micro-batch-size": ["2"],
|
||||
"--prefill-max-requests": ["2"],
|
||||
"--speculative-num-draft-tokens": ["6"],
|
||||
"--speculative-num-steps": ["5"]
|
||||
}
|
||||
},
|
||||
"ref": [35, "sglang", 0]
|
||||
},
|
||||
{
|
||||
"kind": "sglang-router",
|
||||
"cmd": "python3 -m sglang_router.launch_router",
|
||||
"env": {},
|
||||
"args": [
|
||||
["--host", "0.0.0.0"],
|
||||
["--port", "8000"],
|
||||
["--worker-urls", "http://127.0.0.1:30000"]
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dynamo": {
|
||||
"install": [],
|
||||
"files": [],
|
||||
"blocks": [
|
||||
{
|
||||
"kind": "etcd",
|
||||
"count_nodes": 1,
|
||||
"cmd": "etcd",
|
||||
"env": {},
|
||||
"args": [
|
||||
["--data-dir", "/tmp/etcd"],
|
||||
["--listen-client-urls", "http://0.0.0.0:2379"],
|
||||
["--advertise-client-urls", "http://$ETCD_IP:2379"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"kind": "dynamo-frontend",
|
||||
"count": 1,
|
||||
"cmd": "python3 -m dynamo.frontend",
|
||||
"env": {
|
||||
"DYN_REQUEST_PLANE": "tcp",
|
||||
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
|
||||
},
|
||||
"args": [
|
||||
["--http-port", "8000"],
|
||||
["--router-mode", "kv"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"from": 0,
|
||||
"cmd": "python3 -m dynamo.sglang",
|
||||
"diff": {
|
||||
"args": {
|
||||
"--request-plane": ["tcp"]
|
||||
},
|
||||
"env": {
|
||||
"DYN_REQUEST_PLANE": "tcp",
|
||||
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto#0",
|
||||
"hw": "b300",
|
||||
"ckpt": "nvidia/Qwen3.5-397B-A17B-NVFP4",
|
||||
"label": "Agg TP2·EP2",
|
||||
"detail": "MTP 3-1-4",
|
||||
"gpus": 2,
|
||||
"configKey": "qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto",
|
||||
"image": "lmsysorg/sglang:nightly-dev-20261013-7224fce1",
|
||||
"imageStatus": "ok",
|
||||
"imageCommit": "7224fce1",
|
||||
"submitted": "sglang",
|
||||
"setup": [],
|
||||
"disclosures": ["synthetic-acceptance"],
|
||||
"kv": {
|
||||
"available": ["none", "hicache", "mooncake"],
|
||||
"unavailable": {},
|
||||
"hicache": {
|
||||
"args": [
|
||||
["--enable-hierarchical-cache"],
|
||||
["--hicache-io-backend", "kernel"],
|
||||
["--hicache-mem-layout", "page_first"],
|
||||
["--hicache-size", "144"],
|
||||
["--hicache-write-policy", "write_through_selective"]
|
||||
],
|
||||
"donor": "qwen3.5-fp4-b300-sglang-agentic-mtp",
|
||||
"donorHardware": "b300"
|
||||
}
|
||||
},
|
||||
"prov": {
|
||||
"sha": "836a0c38ae5f14b56ce40307a0f3f619f96846cc",
|
||||
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml",
|
||||
"run": "38390415472/attempts/1",
|
||||
"pr": "2978",
|
||||
"status": "verified",
|
||||
"date": "2026-10-14"
|
||||
},
|
||||
"points": [
|
||||
{
|
||||
"concs": [64],
|
||||
"item": 0,
|
||||
"goldenAL": "3.39",
|
||||
"kv": "hicache"
|
||||
}
|
||||
],
|
||||
"routers": {
|
||||
"sglang": {
|
||||
"install": [],
|
||||
"files": [],
|
||||
"blocks": [
|
||||
{
|
||||
"diff": {
|
||||
"args": {
|
||||
"--allow-auto-truncate": [],
|
||||
"--chunked-prefill-size": ["16384"],
|
||||
"--cuda-graph-max-bs-decode": ["224"],
|
||||
"--expert-parallel-size": ["2"],
|
||||
"--hicache-ratio": ["1.8"],
|
||||
"--linear-attn-prefill-backend": ["flashinfer"],
|
||||
"--mamba-max-states-per-path": ["1"],
|
||||
"--mamba-track-interval": ["1048576"],
|
||||
"--max-mamba-cache-size": ["512"],
|
||||
"--max-prefill-tokens": ["16384"],
|
||||
"--max-running-requests": ["224"],
|
||||
"--moe-runner-backend": ["flashinfer_cutedsl"],
|
||||
"--scheduler-recv-interval": ["10"],
|
||||
"--speculative-num-draft-tokens": ["4"],
|
||||
"--speculative-num-steps": ["3"],
|
||||
"--stream-interval": ["50"],
|
||||
"--tokenizer-worker-num": ["6"],
|
||||
"--weight-loader-drop-cache-after-load": [],
|
||||
"--cuda-graph-bs-decode": null,
|
||||
"--disable-attn-tp-gather": null,
|
||||
"--disable-shared-experts-fusion": null,
|
||||
"--hicache-size": null,
|
||||
"--linear-attn-decode-backend": null,
|
||||
"--moe-a2a-backend": null,
|
||||
"--pp-max-micro-batch-size": null,
|
||||
"--prefill-decode-interval": null,
|
||||
"--prefill-max-requests": null,
|
||||
"--speculative-moe-a2a-backend": null,
|
||||
"--weight-loader-prefetch-checkpoints": null,
|
||||
"--weight-loader-prefetch-num-threads": null
|
||||
},
|
||||
"env": {
|
||||
"SGLANG_ENABLE_JIT_DEEPGEMM": "false",
|
||||
"SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS": "2"
|
||||
}
|
||||
},
|
||||
"ref": [35, "sglang", 0]
|
||||
},
|
||||
{
|
||||
"kind": "sglang-router",
|
||||
"cmd": "python3 -m sglang_router.launch_router",
|
||||
"env": {},
|
||||
"args": [
|
||||
["--host", "0.0.0.0"],
|
||||
["--port", "8000"],
|
||||
["--worker-urls", "http://127.0.0.1:30000"]
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"dynamo": {
|
||||
"install": [],
|
||||
"files": [],
|
||||
"blocks": [
|
||||
{
|
||||
"kind": "etcd",
|
||||
"count_nodes": 1,
|
||||
"cmd": "etcd",
|
||||
"env": {},
|
||||
"args": [
|
||||
["--data-dir", "/tmp/etcd"],
|
||||
["--listen-client-urls", "http://0.0.0.0:2379"],
|
||||
["--advertise-client-urls", "http://$ETCD_IP:2379"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"kind": "dynamo-frontend",
|
||||
"count": 1,
|
||||
"cmd": "python3 -m dynamo.frontend",
|
||||
"env": {
|
||||
"DYN_REQUEST_PLANE": "tcp",
|
||||
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
|
||||
},
|
||||
"args": [
|
||||
["--http-port", "8000"],
|
||||
["--router-mode", "kv"]
|
||||
]
|
||||
},
|
||||
{
|
||||
"from": 0,
|
||||
"cmd": "python3 -m dynamo.sglang",
|
||||
"diff": {
|
||||
"args": {
|
||||
"--request-plane": ["tcp"]
|
||||
},
|
||||
"env": {
|
||||
"DYN_REQUEST_PLANE": "tcp",
|
||||
"ETCD_ENDPOINTS": "http://$ETCD_IP:2379"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "qwen3.5-fp8-b200-sglang-agentic-mtp#2",
|
||||
"hw": "b200",
|
||||
@@ -8684,7 +9313,7 @@ export const agentx = {
|
||||
"--tp": ["8"]
|
||||
}
|
||||
},
|
||||
"ref": [34, "sglang", 0]
|
||||
"ref": [38, "sglang", 0]
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -9339,7 +9968,7 @@ export const agentx = {
|
||||
"--page-size": ["64"]
|
||||
}
|
||||
},
|
||||
"ref": [35, "sglang", 0]
|
||||
"ref": [39, "sglang", 0]
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -10346,7 +10975,7 @@ export const agentx = {
|
||||
"--tp": ["4"]
|
||||
}
|
||||
},
|
||||
"ref": [38, "sglang", 0]
|
||||
"ref": [42, "sglang", 0]
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -10586,7 +11215,7 @@ export const agentx = {
|
||||
"--page-size": ["64"]
|
||||
}
|
||||
},
|
||||
"ref": [39, "sglang", 0]
|
||||
"ref": [43, "sglang", 0]
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -10902,7 +11531,7 @@ export const agentx = {
|
||||
"--expert-parallel-size": ["8"]
|
||||
}
|
||||
},
|
||||
"ref": [41, "sglang", 0]
|
||||
"ref": [45, "sglang", 0]
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -11036,7 +11665,7 @@ export const agentx = {
|
||||
"--page-size": null
|
||||
}
|
||||
},
|
||||
"ref": [42, "sglang", 0]
|
||||
"ref": [46, "sglang", 0]
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -11328,7 +11957,7 @@ export const agentx = {
|
||||
"--page-size": ["64"]
|
||||
}
|
||||
},
|
||||
"ref": [44, "sglang", 0]
|
||||
"ref": [48, "sglang", 0]
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -1,21 +1,36 @@
|
||||
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
|
||||
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
|
||||
// (SemiAnalysisAI/InferenceX main@6fff051b9f0d plus open PRs #3927, #3953), with SA-specific settings removed.
|
||||
// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed.
|
||||
export const agentx = {
|
||||
"model": "Qwen3.8-Flash-Next",
|
||||
"generatedOn": "2026-10-13",
|
||||
"generatedOn": "2026-10-15",
|
||||
"source": {
|
||||
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
|
||||
"main": "6fff051b9f0dd578b1bd0a11dc84dfc502c31e5d",
|
||||
"main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff",
|
||||
"prs": {
|
||||
"3953": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38356572105",
|
||||
"sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125",
|
||||
"4146": {
|
||||
"run": null,
|
||||
"sha": "a23546bea6a5f1d34b75261728ac110403a86540",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4141": {
|
||||
"run": null,
|
||||
"sha": "d3850c4581d03e908bb0a27c39830185a4591465",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4112": {
|
||||
"run": null,
|
||||
"sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4111": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474",
|
||||
"sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae",
|
||||
"status": "verified-pr"
|
||||
},
|
||||
"3927": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38266927261",
|
||||
"sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78",
|
||||
"4094": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139",
|
||||
"sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac",
|
||||
"status": "verified-pr"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,21 +1,36 @@
|
||||
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
|
||||
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
|
||||
// (SemiAnalysisAI/InferenceX main@6fff051b9f0d plus open PRs #3927, #3953), with SA-specific settings removed.
|
||||
// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed.
|
||||
export const agentx = {
|
||||
"model": "DeepSeek-V4",
|
||||
"generatedOn": "2026-10-13",
|
||||
"generatedOn": "2026-10-15",
|
||||
"source": {
|
||||
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
|
||||
"main": "6fff051b9f0dd578b1bd0a11dc84dfc502c31e5d",
|
||||
"main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff",
|
||||
"prs": {
|
||||
"3953": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38356572105",
|
||||
"sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125",
|
||||
"4146": {
|
||||
"run": null,
|
||||
"sha": "a23546bea6a5f1d34b75261728ac110403a86540",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4141": {
|
||||
"run": null,
|
||||
"sha": "d3850c4581d03e908bb0a27c39830185a4591465",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4112": {
|
||||
"run": null,
|
||||
"sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4111": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474",
|
||||
"sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae",
|
||||
"status": "verified-pr"
|
||||
},
|
||||
"3927": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38266927261",
|
||||
"sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78",
|
||||
"4094": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139",
|
||||
"sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac",
|
||||
"status": "verified-pr"
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,21 +1,36 @@
|
||||
// GENERATED by the sa-infx-cookbook sync scripts — do not edit by hand.
|
||||
// Launch commands mirrored from SemiAnalysis InferenceX AgentX submissions
|
||||
// (SemiAnalysisAI/InferenceX main@6fff051b9f0d plus open PRs #3927, #3953), with SA-specific settings removed.
|
||||
// (SemiAnalysisAI/InferenceX main@c08eb51d3bd5 plus open PRs #4094, #4111, #4112, #4141, #4146), with SA-specific settings removed.
|
||||
export const agentx = {
|
||||
"model": "GLM-5.2",
|
||||
"generatedOn": "2026-10-13",
|
||||
"generatedOn": "2026-10-15",
|
||||
"source": {
|
||||
"repo": "https://github.com/SemiAnalysisAI/InferenceX",
|
||||
"main": "6fff051b9f0dd578b1bd0a11dc84dfc502c31e5d",
|
||||
"main": "c08eb51d3bd536ae3aac8d9b1843b363411611ff",
|
||||
"prs": {
|
||||
"3953": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38356572105",
|
||||
"sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125",
|
||||
"4146": {
|
||||
"run": null,
|
||||
"sha": "a23546bea6a5f1d34b75261728ac110403a86540",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4141": {
|
||||
"run": null,
|
||||
"sha": "d3850c4581d03e908bb0a27c39830185a4591465",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4112": {
|
||||
"run": null,
|
||||
"sha": "b260d0e4ef31e76bacba68cce9a8107ba4922c72",
|
||||
"status": "in-progress"
|
||||
},
|
||||
"4111": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38749770474",
|
||||
"sha": "f4d96b32737490edcb2249533f4fc8be33e9b8ae",
|
||||
"status": "verified-pr"
|
||||
},
|
||||
"3927": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38266927261",
|
||||
"sha": "e5d1fa07b165c344a6fb32b739f5fd1697564e78",
|
||||
"4094": {
|
||||
"run": "https://github.com/SemiAnalysisAI/InferenceX/actions/runs/38705241139",
|
||||
"sha": "e668c5b9153925a0a50f0adfcd2168d7b331afac",
|
||||
"status": "verified-pr"
|
||||
}
|
||||
}
|
||||
@@ -2479,10 +2494,10 @@ export const agentx = {
|
||||
"prov": {
|
||||
"sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125",
|
||||
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4/agentx/agg-tp4-variants.yaml",
|
||||
"run": "38356572105",
|
||||
"run": "38356572105/attempts/2",
|
||||
"pr": "3953",
|
||||
"comment": "6161711288",
|
||||
"status": "verified-pr"
|
||||
"status": "verified",
|
||||
"date": "2026-10-14"
|
||||
},
|
||||
"points": [
|
||||
{
|
||||
@@ -3210,10 +3225,10 @@ export const agentx = {
|
||||
"prov": {
|
||||
"sha": "caa2199eb65d4ef05d010f34c88b1c8614a0d125",
|
||||
"recipe": "inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/sglang/b300-fp4/agentx/agg-tp4-variants.yaml",
|
||||
"run": "38356572105",
|
||||
"run": "38356572105/attempts/2",
|
||||
"pr": "3953",
|
||||
"comment": "6161711288",
|
||||
"status": "verified-pr"
|
||||
"status": "verified",
|
||||
"date": "2026-10-14"
|
||||
},
|
||||
"points": [
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user