[Fix] glm-5.2: Record side-stream work right before its join to keep CUDA-graph replay on one stream (#45349)

Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
Shu Wang
2026-10-21 15:36:54 +08:00
committed by GitHub
co-authored by Claude Sonnet 5.5 Po-Han Huang
parent 8acfabbd67
commit f7b8c6fe40
2 changed files with 10 additions and 7 deletions
@@ -1107,10 +1107,6 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
# on the GEMMs it consumes, not on the other side's fused kernel.
current_stream = torch.cuda.current_stream()
self.alt_stream.wait_stream(current_stream)
with torch.cuda.stream(self.alt_stream):
q = self.wq_b(q_lora)[0].view(-1, self.n_heads, self.head_dim)
if num_tokens is not None:
q = q[:num_tokens]
kw, _ = self.wk_weights_proj(x)
key, weights_raw = kw.split([self.head_dim, self.n_heads], dim=-1)
@@ -1118,6 +1114,12 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
key = key[:num_tokens]
weights_raw = weights_raw[:num_tokens]
# Issued after the main GEMM so the join stays on the main lane at graph replay.
with torch.cuda.stream(self.alt_stream):
q = self.wq_b(q_lora)[0].view(-1, self.n_heads, self.head_dim)
if num_tokens is not None:
q = q[:num_tokens]
current_stream.wait_stream(self.alt_stream)
self.alt_stream.wait_stream(current_stream)
q_fp8, weights = fused_q_indexer_rope_first_quant(
@@ -372,9 +372,7 @@ class DeepseekMLAForwardMixin:
):
current_stream = torch.cuda.current_stream()
self.alt_stream.wait_stream(current_stream)
with torch.cuda.stream(self.alt_stream):
k_nope = k_nope.unsqueeze(1)
q = self.q_b_proj_forward(q)
k_nope = k_nope.unsqueeze(1)
if self.should_run_indexer(prev_topk_indices):
topk_indices = self.indexer(
x=hidden_states,
@@ -389,6 +387,9 @@ class DeepseekMLAForwardMixin:
topk_indices = maybe_capture_indexer_topk(
self.layer_id, prev_topk_indices
)
# Issued after the indexer so the join stays on the main lane at graph replay.
with torch.cuda.stream(self.alt_stream):
q = self.q_b_proj_forward(q)
current_stream.wait_stream(self.alt_stream)
else:
k_nope = k_nope.unsqueeze(1)