From 1d17066bad10281af50852d554444dd828c349bc Mon Sep 17 00:00:00 2001 From: caihuali95 <42954765+caihuali95@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:37:24 -0700 Subject: [PATCH] [Perf] gdn: Select FlashInfer prefill for page-major pools (#43171) Co-authored-by: Caihua Li --- python/sglang/srt/arg_groups/kv_cache_hook.py | 7 ++++--- .../layers/attention/linear/gdn_backend.py | 1 - .../test_gdn_prefill_backend_policy.py | 20 ++++++++++++++++++- .../test_page_major_backend_allowlist.py | 5 +++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 85ab055ee4..65c942dfe8 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -832,14 +832,15 @@ def handle_page_major_kv_layout(server_args: Any): # stride as a free int64); helion (specializes KDA state strides 0-3 # and rejects a non-unit innermost stride); cutedsl (KDA fused sigmoid- # gating update is stride-safe) on KDA-hybrid models only. - # - prefill: triton; flashkda (the wrapper gathers/scatters a contiguous - # per-slot copy); helion; cutedsl (kernel_h compiles h0/ht with dynamic + # - prefill: triton; flashinfer (GDN gathers/scatters contiguous state); + # flashkda (the wrapper gathers/scatters a contiguous per-slot copy); + # helion; cutedsl (kernel_h compiles h0/ht with dynamic # int64 strides), with the same KDA-only caveat. # - mamba (mamba2/short-conv state): triton only. # use_mla_backend() distinguishes the KDA-hybrid family (K3/KimiLinear # are MLA-hybrid) from GDN models (GQA-hybrid) for the KDA-only caveat. decode_allowed = {"triton", "flashinfer"} - prefill_allowed = {"triton", "flashkda"} + prefill_allowed = {"triton", "flashkda", "flashinfer"} if use_mla_backend(server_args): decode_allowed.update({"cutedsl", "helion"}) prefill_allowed.update({"cutedsl", "helion"}) diff --git a/python/sglang/srt/layers/attention/linear/gdn_backend.py b/python/sglang/srt/layers/attention/linear/gdn_backend.py index 75782942ff..16f92f4ff3 100644 --- a/python/sglang/srt/layers/attention/linear/gdn_backend.py +++ b/python/sglang/srt/layers/attention/linear/gdn_backend.py @@ -195,7 +195,6 @@ def flashinfer_gdn_prefill_default(model_runner: ModelRunner) -> Optional[str]: get_exec().mamba.linear_attn_prefill_backend is not None or get_exec().mamba.linear_attn_backend != "triton" or get_exec().deterministic.enable_deterministic_inference - or get_memory().enable_page_major_kv_layout or sm_major not in (9, 10) ): return None diff --git a/test/registered/unit/layers/attention/test_gdn_prefill_backend_policy.py b/test/registered/unit/layers/attention/test_gdn_prefill_backend_policy.py index 30cd0402e8..d5895c0e07 100644 --- a/test/registered/unit/layers/attention/test_gdn_prefill_backend_policy.py +++ b/test/registered/unit/layers/attention/test_gdn_prefill_backend_policy.py @@ -267,6 +267,25 @@ class TestFlashInferGDNPrefillBackendPolicy(CustomTestCase): self.apply_policy(make_runner(self, **runner_args), **hardware) ) + def test_selects_flashinfer_for_a_page_major_pool(self): + """The unified pool runs page-major. Its strided state reaches every + prefill kernel as contiguous per-sequence copies, so FlashInfer prefill + qualifies there exactly as on the contiguous pool.""" + cases = ( + ("sm100", {}, {}), + ( + "sm90", + {"state_dtype": torch.float32}, + {"capability": (9, 0), "cuda_version": "12.9"}, + ), + ) + for name, runner_args, hardware in cases: + with self.subTest(name=name): + runner = make_runner( + self, enable_page_major_kv_layout=True, **runner_args + ) + self.assertEqual(self.apply_policy(runner, **hardware), "flashinfer") + def test_rejects_gdn_config_without_qwen_head_dims(self): runner = make_runner(self) runner.hybrid_gdn_config = SimpleNamespace() @@ -275,7 +294,6 @@ class TestFlashInferGDNPrefillBackendPolicy(CustomTestCase): def test_rejects_unvalidated_runtime_modes(self): cases = ( ("non_triton_base", {"linear_attn_backend": "cutedsl"}), - ("page_major_kv", {"enable_page_major_kv_layout": True}), ("dynamic_chunk", {"enable_dynamic_chunking": True}), ("unchunked", {"chunked_prefill_size": -1}), ("unknown_chunk", {"chunked_prefill_size": None}), diff --git a/test/registered/unit/server_args/test_page_major_backend_allowlist.py b/test/registered/unit/server_args/test_page_major_backend_allowlist.py index 60d9e6b5fa..d80fa7c9db 100644 --- a/test/registered/unit/server_args/test_page_major_backend_allowlist.py +++ b/test/registered/unit/server_args/test_page_major_backend_allowlist.py @@ -103,6 +103,11 @@ class TestPageMajorBackendAllowlist(unittest.TestCase): # No virtual-to-physical id wiring anywhere: must stay rejected until they get one. UNWIRED_BACKENDS = ("aiter",) + def test_explicit_flashinfer_gdn_prefill_matches_the_default(self): + """Pinning the automatically selected GDN prefill backend must not + make an otherwise valid unified-memory configuration fail at startup.""" + self.assertTrue(_accepts("triton", use_mla=False, linear_prefill="flashinfer")) + def test_triton_allowed_on_every_arm(self): """Triton reads both view families, so it is the one backend neither the MLA nor the MHA arm can narrow away."""