[Fix] glm-5.2: Record side-stream work right before its join to keep CUDA-graph replay on one stream (#45349)
Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
Po-Han Huang
parent
8acfabbd67
commit
f7b8c6fe40
@@ -1107,10 +1107,6 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
# on the GEMMs it consumes, not on the other side's fused kernel.
|
||||
current_stream = torch.cuda.current_stream()
|
||||
self.alt_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(self.alt_stream):
|
||||
q = self.wq_b(q_lora)[0].view(-1, self.n_heads, self.head_dim)
|
||||
if num_tokens is not None:
|
||||
q = q[:num_tokens]
|
||||
|
||||
kw, _ = self.wk_weights_proj(x)
|
||||
key, weights_raw = kw.split([self.head_dim, self.n_heads], dim=-1)
|
||||
@@ -1118,6 +1114,12 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
key = key[:num_tokens]
|
||||
weights_raw = weights_raw[:num_tokens]
|
||||
|
||||
# Issued after the main GEMM so the join stays on the main lane at graph replay.
|
||||
with torch.cuda.stream(self.alt_stream):
|
||||
q = self.wq_b(q_lora)[0].view(-1, self.n_heads, self.head_dim)
|
||||
if num_tokens is not None:
|
||||
q = q[:num_tokens]
|
||||
|
||||
current_stream.wait_stream(self.alt_stream)
|
||||
self.alt_stream.wait_stream(current_stream)
|
||||
q_fp8, weights = fused_q_indexer_rope_first_quant(
|
||||
|
||||
@@ -372,9 +372,7 @@ class DeepseekMLAForwardMixin:
|
||||
):
|
||||
current_stream = torch.cuda.current_stream()
|
||||
self.alt_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(self.alt_stream):
|
||||
k_nope = k_nope.unsqueeze(1)
|
||||
q = self.q_b_proj_forward(q)
|
||||
k_nope = k_nope.unsqueeze(1)
|
||||
if self.should_run_indexer(prev_topk_indices):
|
||||
topk_indices = self.indexer(
|
||||
x=hidden_states,
|
||||
@@ -389,6 +387,9 @@ class DeepseekMLAForwardMixin:
|
||||
topk_indices = maybe_capture_indexer_topk(
|
||||
self.layer_id, prev_topk_indices
|
||||
)
|
||||
# Issued after the indexer so the join stays on the main lane at graph replay.
|
||||
with torch.cuda.stream(self.alt_stream):
|
||||
q = self.q_b_proj_forward(q)
|
||||
current_stream.wait_stream(self.alt_stream)
|
||||
else:
|
||||
k_nope = k_nope.unsqueeze(1)
|
||||
|
||||
Reference in New Issue
Block a user