PR #40919: fix dspark folded result buffer reuse race
Source: https://github.com/sgl-project/sglang/pull/40919 (applied via GitHub .diff, 3way)
This commit is contained in:
@@ -76,6 +76,12 @@ class GenerationBatchResult:
|
||||
delay_sample_func: Optional[callable] = None
|
||||
future_indices: Optional[torch.Tensor] = None
|
||||
speculative_num_draft_tokens: Optional[int] = None
|
||||
# Folded speculative epilogues return views into graph-owned persistent
|
||||
# buffers. The owner records a narrow event after those views are copied
|
||||
# to host, then gates only the next replay that can overwrite them.
|
||||
persistent_result_copy_owner: Optional[Any] = dataclasses.field(
|
||||
default=None, repr=False, compare=False
|
||||
)
|
||||
# Padded row width in flattened speculative output. Existing algorithms
|
||||
# default to speculative_num_draft_tokens; linear UNO emits F + 1 columns.
|
||||
speculative_output_stride: Optional[int] = None
|
||||
@@ -156,6 +162,16 @@ class GenerationBatchResult:
|
||||
Only the tensors which are needed for processing results are copied,
|
||||
e.g., next_token_ids, logits outputs
|
||||
"""
|
||||
owner = self.persistent_result_copy_owner
|
||||
if owner is not None:
|
||||
# A folded graph may reuse these buffers on the next target replay.
|
||||
# Copy them before optional outputs so the narrow reuse fence does
|
||||
# not wait for unrelated D2H traffic.
|
||||
self._copy_token_results_to_cpu()
|
||||
owner.record_result_copy_done()
|
||||
# Avoid retaining the model and graph through the result queue.
|
||||
self.persistent_result_copy_owner = None
|
||||
|
||||
if return_logprob:
|
||||
if self.logits_output.next_token_logprobs is not None:
|
||||
self.logits_output.next_token_logprobs = _async_d2h(
|
||||
@@ -184,17 +200,9 @@ class GenerationBatchResult:
|
||||
self.logits_output.hidden_states = _async_d2h(
|
||||
self.logits_output.hidden_states
|
||||
)
|
||||
self.next_token_ids = _async_d2h(self.next_token_ids)
|
||||
|
||||
if self.accept_lens is not None:
|
||||
self.accept_lens = _async_d2h(self.accept_lens)
|
||||
|
||||
if self.block_accept_lens is not None:
|
||||
self.block_accept_lens = _async_d2h(self.block_accept_lens)
|
||||
|
||||
if self.cap_lens is not None:
|
||||
self.cap_lens = _async_d2h(self.cap_lens)
|
||||
|
||||
if owner is None:
|
||||
# Preserve the established copy order for non-persistent results.
|
||||
self._copy_token_results_to_cpu()
|
||||
# Sub-objects only declare their device fields; the single copy+safety
|
||||
# primitive (_async_d2h: pinned D2H + record_stream) is injected here so
|
||||
# all device->host copying and lifetime safety lives in one place.
|
||||
@@ -216,6 +224,18 @@ class GenerationBatchResult:
|
||||
|
||||
self.copy_done.record()
|
||||
|
||||
def _copy_token_results_to_cpu(self) -> None:
|
||||
self.next_token_ids = _async_d2h(self.next_token_ids)
|
||||
|
||||
if self.accept_lens is not None:
|
||||
self.accept_lens = _async_d2h(self.accept_lens)
|
||||
|
||||
if self.block_accept_lens is not None:
|
||||
self.block_accept_lens = _async_d2h(self.block_accept_lens)
|
||||
|
||||
if self.cap_lens is not None:
|
||||
self.cap_lens = _async_d2h(self.cap_lens)
|
||||
|
||||
def copy_auxiliary_output_to_cpu(self) -> None:
|
||||
if self.logits_output is None or self.auxiliary_host_output is not None:
|
||||
return
|
||||
|
||||
@@ -267,6 +267,8 @@ class TargetVerifyExecutor:
|
||||
verify_forward_batch, _ = verify_input.prepare_for_verify(
|
||||
batch, self.target_worker
|
||||
)
|
||||
if self.verify_epilogue is not None:
|
||||
self.verify_epilogue.wait_for_result_copy_done()
|
||||
self.target_worker.forward_batch_generation(
|
||||
batch=None,
|
||||
forward_batch=verify_forward_batch,
|
||||
@@ -340,6 +342,8 @@ class TargetVerifyExecutor:
|
||||
batch.seq_lens_cpu = seq_lens_cpu_backup
|
||||
batch.seq_lens_sum = seq_lens_sum_backup
|
||||
|
||||
if self.verify_epilogue is not None:
|
||||
self.verify_epilogue.wait_for_result_copy_done()
|
||||
target_out = self.target_worker.forward_batch_generation(
|
||||
batch=None,
|
||||
forward_batch=verify_forward_batch,
|
||||
@@ -572,6 +576,31 @@ class DsparkVerifyEpilogue:
|
||||
self.strided_logits: Optional[torch.Tensor] = None
|
||||
self.strided_hidden: Optional[torch.Tensor] = None
|
||||
self._static_step_state: Optional[tuple[int, bool]] = None
|
||||
self._result_copy_done_event = None
|
||||
self._result_copy_pending = False
|
||||
|
||||
def record_result_copy_done(self) -> None:
|
||||
"""Fence D2H reads of persistent accept outputs on the copy stream."""
|
||||
if self._result_copy_pending:
|
||||
raise RuntimeError(
|
||||
"DSpark folded result buffer was copied again before its prior "
|
||||
"reuse dependency was consumed."
|
||||
)
|
||||
device_module = torch.get_device_module(self.correct_len_buf.device)
|
||||
if self._result_copy_done_event is None:
|
||||
self._result_copy_done_event = device_module.Event()
|
||||
self._result_copy_done_event.record()
|
||||
self._result_copy_pending = True
|
||||
|
||||
def wait_for_result_copy_done(self) -> None:
|
||||
"""Gate the next target replay, leaving the preceding draft overlapped."""
|
||||
if not self._result_copy_pending:
|
||||
return
|
||||
device_module = torch.get_device_module(self.correct_len_buf.device)
|
||||
device_module.current_stream(self.correct_len_buf.device).wait_event(
|
||||
self._result_copy_done_event
|
||||
)
|
||||
self._result_copy_pending = False
|
||||
|
||||
def capture_hook(self, runner, out, forward_batch, num_tokens) -> None:
|
||||
if (
|
||||
|
||||
@@ -981,6 +981,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
next_draft_input=next_draft_input,
|
||||
speculative_num_draft_tokens=int(self.verify_num_draft_tokens),
|
||||
new_seq_lens=accept.new_seq_lens,
|
||||
persistent_result_copy_owner=epilogue if folded_accept else None,
|
||||
)
|
||||
|
||||
def _commit_target_mamba_states_after_verify(
|
||||
|
||||
Reference in New Issue
Block a user