From 84a6f51bd6749ec2e46b985775c82060cf52be5a Mon Sep 17 00:00:00 2001 From: Aurick Qiao Date: Thu, 27 Aug 2026 19:10:36 -0700 Subject: [PATCH] Fix DeepSeek V4 multistream QKV buffer lifetime (#36547) --- python/sglang/srt/models/deepseek_v4.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 41ea2da41..116291a1f 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -1059,8 +1059,6 @@ class MQALayer(MqaAttentionBase): x_linear, positions, forward_batch, attn_backend, qkv_a=qkv_a ) - del qkv_a - if self.compressor is not None: with torch.cuda.stream(stream_compressor): attn_backend.forward_core_compressor( @@ -1071,6 +1069,7 @@ class MQALayer(MqaAttentionBase): current_stream.wait_stream(stream_kv) current_stream.wait_stream(stream_compressor) current_stream.wait_stream(stream_indexer) + del qkv_a return q @@ -1153,8 +1152,6 @@ class MQALayer(MqaAttentionBase): q_out.copy_(q) q.record_stream(stream_q) - del qkv_a - # Indexer + compressor: serial on current. if self.indexer is not None: self.indexer( @@ -1174,6 +1171,7 @@ class MQALayer(MqaAttentionBase): # Join stream_kv + stream_q before downstream attention. current_stream.wait_stream(stream_kv) current_stream.wait_stream(stream_q) + del qkv_a return q def _forward_prepare_multi_stream_hip(