[docs][cookbook] Laguna-M.1 playground: add HiCache; refresh EP / DP-Attention notes (#28774)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Zijie Xia <zijie.xia@radixark.ai>
This commit is contained in:
Jimmy Shong
2026-06-22 03:58:42 +00:00
committed by GitHub
co-authored by Claude Opus 4.8 Zijie Xia
parent 886b96621d
commit be774d0acd
@@ -118,9 +118,13 @@ sgl-eval run gsm8k \\
playgroundFeatures: {
// M.1 is global-attention (no SWA); expose TP + DP-Attention. No CP.
// DP-Attention is a Playground experiment only — ~15% slower than plain TP on this GQA
// model (8 KV heads), so it is NOT in the shipped Balanced recipe.
// M.1 is global-attention (no SWA); expose TP + DP-Attention here. No CP: the default
// trtllm_mha backend has no CP-aware KV-store (crashes), and the engine's built-in attention CP
// knob emits NSA flags (--enable-nsa-prefill-context-parallel) that apply to DeepSeek-family
// models, not M.1. (CP works only via the fa3 backend, which is Hopper SM90 — left out here.)
// DP-Attention: VERIFIED functionally correct on 8×B200 BF16 (GSM8K 0.94, identical to the TP
// baseline) but ~15–28% slower on this GQA model (8 KV heads). Playground experiment only —
// deliberately NOT in the shipped Balanced recipe.
attention: {
knobs: [
{ id: "tp", label: "TP", values: [null, 1, 2, 4, 8] },
@@ -129,14 +133,12 @@ sgl-eval run gsm8k \\
],
},
// 256-expert top-16 MoE. DeepEP all-to-all + EP degree.
// 256-expert top-16 MoE — EP degree only.
// EP: VERIFIED on 8×B200 BF16 (--ep-size 8, GSM8K 0.94, identical to the TP baseline).
// DeepEP intentionally NOT exposed — it does not work on M.1: top-16 routing exceeds DeepEP's
// low-latency internode kernel cap of 11 (internode_ll.cu kNumMaxTopK=11) → assert at decode
// CUDA-graph capture, and `--deepep-mode normal` is NotImplemented for unquantized weights. Use EP.
moe: {
backend: {
options: [
{ id: null, label: "Inherited" },
{ id: "deepep", label: "DeepEP", flags: ["--moe-a2a-backend deepep"] },
],
},
ep: { label: "EP", values: [null, 1, 2, 4, 8] },
},
@@ -149,6 +151,20 @@ sgl-eval run gsm8k \\
],
},
// HiCache (hierarchical KV cache). VERIFIED on 8×B200 BF16: enabling the host L2 tier on a
// zipfian shared-prefix workload cut mean TTFT ~36% (median ~43%) and lifted throughput ~19% vs
// GPU-only, with ~1.14M tokens served from the host tier (TPOT unchanged — the win is on prefill
// / prefix reuse). Biggest gains on reuse-heavy traffic: shared system prompts, multi-turn
// agentic coding, repeated long contexts. "Enable" emits --enable-hierarchical-cache (+host L2);
// Write policy is optional. (L3 storage backends exist but were not validated, so none exposed.)
hicache: {
writePolicies: [
{ id: "auto", label: "Auto" },
{ id: "write_through", label: "Write-through" },
{ id: "write_back", label: "Write-back" },
],
},
// Prefill-Decode disaggregation (§3.3). M.1 is standard-KV (global attention, no sparse
// index buffer), so it disaggregates with just the --disaggregation-* flags — no model-specific
// backend pinning. Verified on 2×8×H200 (TP8+TP8, BF16) over InfiniBand.