PR #40919: fix dspark folded result buffer reuse race

Source: https://github.com/sgl-project/sglang/pull/40919 (applied via GitHub .diff, 3way)
This commit is contained in:
2026-10-19 10:12:23 +08:00
parent 5e7190b87a
commit 9247c97919
5 changed files with 226 additions and 11 deletions
+31 -11
View File
@@ -76,6 +76,12 @@ class GenerationBatchResult:
delay_sample_func: Optional[callable] = None
future_indices: Optional[torch.Tensor] = None
speculative_num_draft_tokens: Optional[int] = None
# Folded speculative epilogues return views into graph-owned persistent
# buffers. The owner records a narrow event after those views are copied
# to host, then gates only the next replay that can overwrite them.
persistent_result_copy_owner: Optional[Any] = dataclasses.field(
default=None, repr=False, compare=False
)
# Padded row width in flattened speculative output. Existing algorithms
# default to speculative_num_draft_tokens; linear UNO emits F + 1 columns.
speculative_output_stride: Optional[int] = None
@@ -156,6 +162,16 @@ class GenerationBatchResult:
Only the tensors which are needed for processing results are copied,
e.g., next_token_ids, logits outputs
"""
owner = self.persistent_result_copy_owner
if owner is not None:
# A folded graph may reuse these buffers on the next target replay.
# Copy them before optional outputs so the narrow reuse fence does
# not wait for unrelated D2H traffic.
self._copy_token_results_to_cpu()
owner.record_result_copy_done()
# Avoid retaining the model and graph through the result queue.
self.persistent_result_copy_owner = None
if return_logprob:
if self.logits_output.next_token_logprobs is not None:
self.logits_output.next_token_logprobs = _async_d2h(
@@ -184,17 +200,9 @@ class GenerationBatchResult:
self.logits_output.hidden_states = _async_d2h(
self.logits_output.hidden_states
)
self.next_token_ids = _async_d2h(self.next_token_ids)
if self.accept_lens is not None:
self.accept_lens = _async_d2h(self.accept_lens)
if self.block_accept_lens is not None:
self.block_accept_lens = _async_d2h(self.block_accept_lens)
if self.cap_lens is not None:
self.cap_lens = _async_d2h(self.cap_lens)
if owner is None:
# Preserve the established copy order for non-persistent results.
self._copy_token_results_to_cpu()
# Sub-objects only declare their device fields; the single copy+safety
# primitive (_async_d2h: pinned D2H + record_stream) is injected here so
# all device->host copying and lifetime safety lives in one place.
@@ -216,6 +224,18 @@ class GenerationBatchResult:
self.copy_done.record()
def _copy_token_results_to_cpu(self) -> None:
self.next_token_ids = _async_d2h(self.next_token_ids)
if self.accept_lens is not None:
self.accept_lens = _async_d2h(self.accept_lens)
if self.block_accept_lens is not None:
self.block_accept_lens = _async_d2h(self.block_accept_lens)
if self.cap_lens is not None:
self.cap_lens = _async_d2h(self.cap_lens)
def copy_auxiliary_output_to_cpu(self) -> None:
if self.logits_output is None or self.auxiliary_host_output is not None:
return
@@ -267,6 +267,8 @@ class TargetVerifyExecutor:
verify_forward_batch, _ = verify_input.prepare_for_verify(
batch, self.target_worker
)
if self.verify_epilogue is not None:
self.verify_epilogue.wait_for_result_copy_done()
self.target_worker.forward_batch_generation(
batch=None,
forward_batch=verify_forward_batch,
@@ -340,6 +342,8 @@ class TargetVerifyExecutor:
batch.seq_lens_cpu = seq_lens_cpu_backup
batch.seq_lens_sum = seq_lens_sum_backup
if self.verify_epilogue is not None:
self.verify_epilogue.wait_for_result_copy_done()
target_out = self.target_worker.forward_batch_generation(
batch=None,
forward_batch=verify_forward_batch,
@@ -572,6 +576,31 @@ class DsparkVerifyEpilogue:
self.strided_logits: Optional[torch.Tensor] = None
self.strided_hidden: Optional[torch.Tensor] = None
self._static_step_state: Optional[tuple[int, bool]] = None
self._result_copy_done_event = None
self._result_copy_pending = False
def record_result_copy_done(self) -> None:
"""Fence D2H reads of persistent accept outputs on the copy stream."""
if self._result_copy_pending:
raise RuntimeError(
"DSpark folded result buffer was copied again before its prior "
"reuse dependency was consumed."
)
device_module = torch.get_device_module(self.correct_len_buf.device)
if self._result_copy_done_event is None:
self._result_copy_done_event = device_module.Event()
self._result_copy_done_event.record()
self._result_copy_pending = True
def wait_for_result_copy_done(self) -> None:
"""Gate the next target replay, leaving the preceding draft overlapped."""
if not self._result_copy_pending:
return
device_module = torch.get_device_module(self.correct_len_buf.device)
device_module.current_stream(self.correct_len_buf.device).wait_event(
self._result_copy_done_event
)
self._result_copy_pending = False
def capture_hook(self, runner, out, forward_batch, num_tokens) -> None:
if (
@@ -981,6 +981,7 @@ class DSparkWorkerV2(BaseSpecWorker):
next_draft_input=next_draft_input,
speculative_num_draft_tokens=int(self.verify_num_draft_tokens),
new_seq_lens=accept.new_seq_lens,
persistent_result_copy_owner=epilogue if folded_accept else None,
)
def _commit_target_mamba_states_after_verify(