[Perf] gdn: Select FlashInfer prefill for page-major pools (#43171)
Co-authored-by: Caihua Li <caihua.li@bytedance.com>
This commit is contained in:
@@ -832,14 +832,15 @@ def handle_page_major_kv_layout(server_args: Any):
|
||||
# stride as a free int64); helion (specializes KDA state strides 0-3
|
||||
# and rejects a non-unit innermost stride); cutedsl (KDA fused sigmoid-
|
||||
# gating update is stride-safe) on KDA-hybrid models only.
|
||||
# - prefill: triton; flashkda (the wrapper gathers/scatters a contiguous
|
||||
# per-slot copy); helion; cutedsl (kernel_h compiles h0/ht with dynamic
|
||||
# - prefill: triton; flashinfer (GDN gathers/scatters contiguous state);
|
||||
# flashkda (the wrapper gathers/scatters a contiguous per-slot copy);
|
||||
# helion; cutedsl (kernel_h compiles h0/ht with dynamic
|
||||
# int64 strides), with the same KDA-only caveat.
|
||||
# - mamba (mamba2/short-conv state): triton only.
|
||||
# use_mla_backend() distinguishes the KDA-hybrid family (K3/KimiLinear
|
||||
# are MLA-hybrid) from GDN models (GQA-hybrid) for the KDA-only caveat.
|
||||
decode_allowed = {"triton", "flashinfer"}
|
||||
prefill_allowed = {"triton", "flashkda"}
|
||||
prefill_allowed = {"triton", "flashkda", "flashinfer"}
|
||||
if use_mla_backend(server_args):
|
||||
decode_allowed.update({"cutedsl", "helion"})
|
||||
prefill_allowed.update({"cutedsl", "helion"})
|
||||
|
||||
@@ -195,7 +195,6 @@ def flashinfer_gdn_prefill_default(model_runner: ModelRunner) -> Optional[str]:
|
||||
get_exec().mamba.linear_attn_prefill_backend is not None
|
||||
or get_exec().mamba.linear_attn_backend != "triton"
|
||||
or get_exec().deterministic.enable_deterministic_inference
|
||||
or get_memory().enable_page_major_kv_layout
|
||||
or sm_major not in (9, 10)
|
||||
):
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user