[Fix] MiniMax-M3: default to the breakable prefill CUDA graph (#41845)
Co-authored-by: Hao Phan <htphan@nvidia.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Hao Phan
Po-Han Huang
Kevin Mi
Po-Han Huang
Claude Opus 5.5
parent
70e82f7b73
commit
fbf8d730fd
@@ -2396,6 +2396,8 @@ multimodal_breakable_cuda_graph_supported_model_archs = [
|
||||
"MuseGlimmerForConditionalGeneration",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK25ForConditionalGeneration",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
"MiniMaxM3SparseForConditionalGeneration",
|
||||
]
|
||||
|
||||
if external_mm_model_arch := envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.get():
|
||||
|
||||
@@ -188,10 +188,6 @@ class RadixAttention(nn.Module):
|
||||
and (torch.compiler.is_compiling() or not _force_eager_attn.get())
|
||||
):
|
||||
if kwargs.get("idx_q") is not None:
|
||||
if is_in_breakable_cuda_graph():
|
||||
return get_attn_backend().forward(
|
||||
q, k, v, self, forward_batch, save_kv_cache, **kwargs
|
||||
)
|
||||
idx_q = kwargs["idx_q"]
|
||||
idx_k = kwargs["idx_k"]
|
||||
idx_v = kwargs.get("idx_v")
|
||||
@@ -199,7 +195,12 @@ class RadixAttention(nn.Module):
|
||||
(q.shape[0], self.tp_q_head_num * self.v_head_dim)
|
||||
)
|
||||
idx_out = q.new_empty((q.shape[0], idx_q.shape[1] * idx_q.shape[2]))
|
||||
unified_sparse_attention_with_output(
|
||||
op = (
|
||||
breakable_unified_sparse_attention_with_output
|
||||
if is_in_breakable_cuda_graph()
|
||||
else unified_sparse_attention_with_output
|
||||
)
|
||||
op(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
@@ -742,6 +743,9 @@ breakable_unified_attention_with_output = eager_on_graph(True)(
|
||||
breakable_unified_attention_with_output_and_lse = eager_on_graph(True)(
|
||||
unified_attention_with_output_and_lse
|
||||
)
|
||||
breakable_unified_sparse_attention_with_output = eager_on_graph(True)(
|
||||
unified_sparse_attention_with_output
|
||||
)
|
||||
|
||||
|
||||
def attention_with_output_extra_kwargs(
|
||||
|
||||
Reference in New Issue
Block a user