[docs][cookbook] Laguna-M.1 playground: add HiCache; refresh EP / DP-Attention notes (#28774)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Zijie Xia <zijie.xia@radixark.ai>
This commit is contained in:
co-authored by
Claude Opus 4.8
Zijie Xia
parent
886b96621d
commit
be774d0acd
@@ -118,9 +118,13 @@ sgl-eval run gsm8k \\
|
||||
|
||||
playgroundFeatures: {
|
||||
|
||||
// M.1 is global-attention (no SWA); expose TP + DP-Attention. No CP.
|
||||
// DP-Attention is a Playground experiment only — ~15% slower than plain TP on this GQA
|
||||
// model (8 KV heads), so it is NOT in the shipped Balanced recipe.
|
||||
// M.1 is global-attention (no SWA); expose TP + DP-Attention here. No CP: the default
|
||||
// trtllm_mha backend has no CP-aware KV-store (crashes), and the engine's built-in attention CP
|
||||
// knob emits NSA flags (--enable-nsa-prefill-context-parallel) that apply to DeepSeek-family
|
||||
// models, not M.1. (CP works only via the fa3 backend, which is Hopper SM90 — left out here.)
|
||||
// DP-Attention: VERIFIED functionally correct on 8×B200 BF16 (GSM8K 0.94, identical to the TP
|
||||
// baseline) but ~15–28% slower on this GQA model (8 KV heads). Playground experiment only —
|
||||
// deliberately NOT in the shipped Balanced recipe.
|
||||
attention: {
|
||||
knobs: [
|
||||
{ id: "tp", label: "TP", values: [null, 1, 2, 4, 8] },
|
||||
@@ -129,14 +133,12 @@ sgl-eval run gsm8k \\
|
||||
],
|
||||
},
|
||||
|
||||
// 256-expert top-16 MoE. DeepEP all-to-all + EP degree.
|
||||
// 256-expert top-16 MoE — EP degree only.
|
||||
// EP: VERIFIED on 8×B200 BF16 (--ep-size 8, GSM8K 0.94, identical to the TP baseline).
|
||||
// DeepEP intentionally NOT exposed — it does not work on M.1: top-16 routing exceeds DeepEP's
|
||||
// low-latency internode kernel cap of 11 (internode_ll.cu kNumMaxTopK=11) → assert at decode
|
||||
// CUDA-graph capture, and `--deepep-mode normal` is NotImplemented for unquantized weights. Use EP.
|
||||
moe: {
|
||||
backend: {
|
||||
options: [
|
||||
{ id: null, label: "Inherited" },
|
||||
{ id: "deepep", label: "DeepEP", flags: ["--moe-a2a-backend deepep"] },
|
||||
],
|
||||
},
|
||||
ep: { label: "EP", values: [null, 1, 2, 4, 8] },
|
||||
},
|
||||
|
||||
@@ -149,6 +151,20 @@ sgl-eval run gsm8k \\
|
||||
],
|
||||
},
|
||||
|
||||
// HiCache (hierarchical KV cache). VERIFIED on 8×B200 BF16: enabling the host L2 tier on a
|
||||
// zipfian shared-prefix workload cut mean TTFT ~36% (median ~43%) and lifted throughput ~19% vs
|
||||
// GPU-only, with ~1.14M tokens served from the host tier (TPOT unchanged — the win is on prefill
|
||||
// / prefix reuse). Biggest gains on reuse-heavy traffic: shared system prompts, multi-turn
|
||||
// agentic coding, repeated long contexts. "Enable" emits --enable-hierarchical-cache (+host L2);
|
||||
// Write policy is optional. (L3 storage backends exist but were not validated, so none exposed.)
|
||||
hicache: {
|
||||
writePolicies: [
|
||||
{ id: "auto", label: "Auto" },
|
||||
{ id: "write_through", label: "Write-through" },
|
||||
{ id: "write_back", label: "Write-back" },
|
||||
],
|
||||
},
|
||||
|
||||
// Prefill-Decode disaggregation (§3.3). M.1 is standard-KV (global attention, no sparse
|
||||
// index buffer), so it disaggregates with just the --disaggregation-* flags — no model-specific
|
||||
// backend pinning. Verified on 2×8×H200 (TP8+TP8, BF16) over InfiniBand.
|
||||
|
||||
Reference in New Issue
Block a user