From f7b8c6fe404ca4f29c3188cb50692f7735d45a99 Mon Sep 17 00:00:00 2001 From: Shu Wang Date: Wed, 21 Oct 2026 00:36:54 -0700 Subject: [PATCH] [Fix] glm-5.2: Record side-stream work right before its join to keep CUDA-graph replay on one stream (#45349) Co-authored-by: Claude Sonnet 5.5 Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> --- python/sglang/srt/layers/attention/dsa/dsa_indexer.py | 10 ++++++---- .../attention_forward_methods/forward_mla.py | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py index e808efcfb1..6587e5fbe9 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py @@ -1107,10 +1107,6 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): # on the GEMMs it consumes, not on the other side's fused kernel. current_stream = torch.cuda.current_stream() self.alt_stream.wait_stream(current_stream) - with torch.cuda.stream(self.alt_stream): - q = self.wq_b(q_lora)[0].view(-1, self.n_heads, self.head_dim) - if num_tokens is not None: - q = q[:num_tokens] kw, _ = self.wk_weights_proj(x) key, weights_raw = kw.split([self.head_dim, self.n_heads], dim=-1) @@ -1118,6 +1114,12 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): key = key[:num_tokens] weights_raw = weights_raw[:num_tokens] + # Issued after the main GEMM so the join stays on the main lane at graph replay. + with torch.cuda.stream(self.alt_stream): + q = self.wq_b(q_lora)[0].view(-1, self.n_heads, self.head_dim) + if num_tokens is not None: + q = q[:num_tokens] + current_stream.wait_stream(self.alt_stream) self.alt_stream.wait_stream(current_stream) q_fp8, weights = fused_q_indexer_rope_first_quant( diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py index e5fa8b7d6b..476b91efa7 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py @@ -372,9 +372,7 @@ class DeepseekMLAForwardMixin: ): current_stream = torch.cuda.current_stream() self.alt_stream.wait_stream(current_stream) - with torch.cuda.stream(self.alt_stream): - k_nope = k_nope.unsqueeze(1) - q = self.q_b_proj_forward(q) + k_nope = k_nope.unsqueeze(1) if self.should_run_indexer(prev_topk_indices): topk_indices = self.indexer( x=hidden_states, @@ -389,6 +387,9 @@ class DeepseekMLAForwardMixin: topk_indices = maybe_capture_indexer_topk( self.layer_id, prev_topk_indices ) + # Issued after the indexer so the join stays on the main lane at graph replay. + with torch.cuda.stream(self.alt_stream): + q = self.q_b_proj_forward(q) current_stream.wait_stream(self.alt_stream) else: k_nope = k_nope.unsqueeze(1)