diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx index bfe1bd2566..4445e13803 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx @@ -29,7 +29,7 @@ Choose your hardware, then choose the operating point that matches your workload Every listed hardware platform exposes both strategies. A **Verified** badge means that exact hardware and command were tested. **Final Verification In Progress** means the recipe runs and is queued for measurement on the final weights. **Not Verified** means the command is a supported starting point that still needs workload validation. A choice is disabled only when the underlying runtime combination is known to be unsupported. -The recommended selection is only a starting point. The same panel also lets you override the KV/DSA pairing, multimodal feature transport, Breakable Cuda Graph, and HiCache tiers. The MI355X commands are marked **Not Verified** because the selector defaults to HiCache off, whereas the measured AgentX configuration used a host tier, and the merged source has not been remeasured. Do not treat the CUDA benchmark rows as MI355X measurements. +The recommended selection is only a starting point. The same panel also lets you override the KV/DSA pairing, multimodal feature transport, Breakable Cuda Graph, and HiCache tiers. The MI355X commands are marked **Not Verified** because the selector defaults to HiCache off, whereas the measured AgentX configuration adds the host tier described in [Extend the cache hierarchy](#extend-the-cache-hierarchy). Do not treat the CUDA benchmark rows as MI355X measurements. **Breakable Cuda Graph** defaults to **Off**. On NVIDIA, select **On** to add `--cuda-graph-backend-prefill breakable` to the generated command. This requires a build that includes [PR #38522](https://github.com/sgl-project/sglang/pull/38522); the option is disabled on MI355X. @@ -126,6 +126,8 @@ On Blackwell, the recipes default to an FP8 KV cache with TRT-LLM DSA: on GB300 Keep **HiCache** off when GPU memory is sufficient. Select **L1 + L2** to spill reusable cache entries into host memory. Select **+ L3** only after configuring Mooncake on every serving node; the generated command exposes the required configuration path. These options remain selectable but are marked **Not Verified** until the resulting command is validated on the chosen hardware. +For agentic MXFP4 serving on MI355X, select **Low Latency** with **L1 + L2**. The host tier uses `--hicache-size 180` (about 190 GB of host memory per rank once the KDA state and DSA indexer pools are included, 750 GB per node) and requires a build that includes [PR #42178](https://github.com/sgl-project/sglang/pull/42178). With less free host memory, keep HiCache off; the server does not start when the host tier does not fit. + ### Multimodal memory All strategies enable multimodal serving. Image and video input need a Transformers build that ships `Glm5NextProcessor` (the MI355X checks used commit `e4052f55`, which also requires `tokenizers>=0.23.1`). With the pinned `transformers==5.12.1`, the server loads only the tokenizer and silently answers image requests from the text alone. On MI355X, text, image, and an image-bearing follow-up were qualified with that build; video was not evaluated. The processor samples video at 2 FPS and caps video input at 240,000 visual tokens. Install `torchcodec` in the serving environment before sending video requests. For very long videos on 4x GB300, use encoder disaggregation to isolate the vision encoder's memory spikes from language decoding. diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx index c1524a0df6..ab25202218 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx @@ -136,12 +136,11 @@ export const config = { label: "L1 + L2", subtitle: "Host memory", flags: (s) => s.hw === "mi355x" - ? ["--enable-hierarchical-cache", "--hicache-ratio 0.50", - "--hicache-write-policy write_through", "--hicache-io-backend direct", - "--hicache-mem-layout page_first_direct"] + ? ["--enable-hierarchical-cache", "--hicache-size 180", + "--hicache-write-policy write_through"] : ["--enable-hierarchical-cache", "--hicache-size 32"], hints: (s) => s.hw === "mi355x" - ? ["Host cache matches the measured 0.50-ratio, write-through MI355X recipe; check available RAM."] + ? ["About 750 GB of host memory per node; the measured MI355X agentic recipe."] : ["32 GB host tier; the default ratio can demand more host RAM than the node has free."], }, { @@ -394,9 +393,9 @@ sgl-eval run gsm8k \\ }, cells: [ - // Four MI355X VFs, TP4/EP4. The fixed-source AgentX comparison also used - // HiCache L2 and CPU multimodal transport. These cookbook commands are - // supported starting points, not post-merge measurements of every overlay. + // Four MI355X VFs, TP4/EP4. The MXFP4 Low Latency chunk and memory fraction + // come from AgentX with the 180 GB/rank HiCache L2 tier; the other cells + // are supported starting points, not measurements of every overlay. { match: { hw: "mi355x", strategy: "low-latency", quant: "fp8" }, nnodes: 1, @@ -445,9 +444,9 @@ sgl-eval run gsm8k \\ "--quantization quark", "--tp-size 4", "--ep-size 4", "--page-size 64", - "--mem-fraction-static 0.82", "--max-running-requests 96", + "--mem-fraction-static 0.85", "--max-running-requests 96", "--cuda-graph-max-bs-decode 96", "--min-free-slots-delay 1", - "--chunked-prefill-size 8192", "--max-prefill-tokens 8192", + "--chunked-prefill-size 65536", "--max-prefill-tokens 65536", "--prefill-decode-interval 0", "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--linear-attn-verify-backend triton", "--kv-cache-dtype fp8_e4m3",