diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index ed40ce3694..07f31d1331 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -76,6 +76,12 @@ class GenerationBatchResult: delay_sample_func: Optional[callable] = None future_indices: Optional[torch.Tensor] = None speculative_num_draft_tokens: Optional[int] = None + # Folded speculative epilogues return views into graph-owned persistent + # buffers. The owner records a narrow event after those views are copied + # to host, then gates only the next replay that can overwrite them. + persistent_result_copy_owner: Optional[Any] = dataclasses.field( + default=None, repr=False, compare=False + ) # Padded row width in flattened speculative output. Existing algorithms # default to speculative_num_draft_tokens; linear UNO emits F + 1 columns. speculative_output_stride: Optional[int] = None @@ -156,6 +162,16 @@ class GenerationBatchResult: Only the tensors which are needed for processing results are copied, e.g., next_token_ids, logits outputs """ + owner = self.persistent_result_copy_owner + if owner is not None: + # A folded graph may reuse these buffers on the next target replay. + # Copy them before optional outputs so the narrow reuse fence does + # not wait for unrelated D2H traffic. + self._copy_token_results_to_cpu() + owner.record_result_copy_done() + # Avoid retaining the model and graph through the result queue. + self.persistent_result_copy_owner = None + if return_logprob: if self.logits_output.next_token_logprobs is not None: self.logits_output.next_token_logprobs = _async_d2h( @@ -184,17 +200,9 @@ class GenerationBatchResult: self.logits_output.hidden_states = _async_d2h( self.logits_output.hidden_states ) - self.next_token_ids = _async_d2h(self.next_token_ids) - - if self.accept_lens is not None: - self.accept_lens = _async_d2h(self.accept_lens) - - if self.block_accept_lens is not None: - self.block_accept_lens = _async_d2h(self.block_accept_lens) - - if self.cap_lens is not None: - self.cap_lens = _async_d2h(self.cap_lens) - + if owner is None: + # Preserve the established copy order for non-persistent results. + self._copy_token_results_to_cpu() # Sub-objects only declare their device fields; the single copy+safety # primitive (_async_d2h: pinned D2H + record_stream) is injected here so # all device->host copying and lifetime safety lives in one place. @@ -216,6 +224,18 @@ class GenerationBatchResult: self.copy_done.record() + def _copy_token_results_to_cpu(self) -> None: + self.next_token_ids = _async_d2h(self.next_token_ids) + + if self.accept_lens is not None: + self.accept_lens = _async_d2h(self.accept_lens) + + if self.block_accept_lens is not None: + self.block_accept_lens = _async_d2h(self.block_accept_lens) + + if self.cap_lens is not None: + self.cap_lens = _async_d2h(self.cap_lens) + def copy_auxiliary_output_to_cpu(self) -> None: if self.logits_output is None or self.auxiliary_host_output is not None: return diff --git a/python/sglang/srt/speculative/dspark_components/dspark_verify.py b/python/sglang/srt/speculative/dspark_components/dspark_verify.py index dd45c0d90c..0ce3e9b7dc 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_verify.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_verify.py @@ -267,6 +267,8 @@ class TargetVerifyExecutor: verify_forward_batch, _ = verify_input.prepare_for_verify( batch, self.target_worker ) + if self.verify_epilogue is not None: + self.verify_epilogue.wait_for_result_copy_done() self.target_worker.forward_batch_generation( batch=None, forward_batch=verify_forward_batch, @@ -340,6 +342,8 @@ class TargetVerifyExecutor: batch.seq_lens_cpu = seq_lens_cpu_backup batch.seq_lens_sum = seq_lens_sum_backup + if self.verify_epilogue is not None: + self.verify_epilogue.wait_for_result_copy_done() target_out = self.target_worker.forward_batch_generation( batch=None, forward_batch=verify_forward_batch, @@ -572,6 +576,31 @@ class DsparkVerifyEpilogue: self.strided_logits: Optional[torch.Tensor] = None self.strided_hidden: Optional[torch.Tensor] = None self._static_step_state: Optional[tuple[int, bool]] = None + self._result_copy_done_event = None + self._result_copy_pending = False + + def record_result_copy_done(self) -> None: + """Fence D2H reads of persistent accept outputs on the copy stream.""" + if self._result_copy_pending: + raise RuntimeError( + "DSpark folded result buffer was copied again before its prior " + "reuse dependency was consumed." + ) + device_module = torch.get_device_module(self.correct_len_buf.device) + if self._result_copy_done_event is None: + self._result_copy_done_event = device_module.Event() + self._result_copy_done_event.record() + self._result_copy_pending = True + + def wait_for_result_copy_done(self) -> None: + """Gate the next target replay, leaving the preceding draft overlapped.""" + if not self._result_copy_pending: + return + device_module = torch.get_device_module(self.correct_len_buf.device) + device_module.current_stream(self.correct_len_buf.device).wait_event( + self._result_copy_done_event + ) + self._result_copy_pending = False def capture_hook(self, runner, out, forward_batch, num_tokens) -> None: if ( diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index e40b4afaf1..66110cbddd 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -981,6 +981,7 @@ class DSparkWorkerV2(BaseSpecWorker): next_draft_input=next_draft_input, speculative_num_draft_tokens=int(self.verify_num_draft_tokens), new_seq_lens=accept.new_seq_lens, + persistent_result_copy_owner=epilogue if folded_accept else None, ) def _commit_target_mamba_states_after_verify( diff --git a/test/registered/spec/dspark/test_dspark_result_buffer_reuse.py b/test/registered/spec/dspark/test_dspark_result_buffer_reuse.py new file mode 100644 index 0000000000..06bb6e049a --- /dev/null +++ b/test/registered/spec/dspark/test_dspark_result_buffer_reuse.py @@ -0,0 +1,119 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.managers.utils import GenerationBatchResult +from sglang.srt.speculative.dspark_components.dspark_verify import ( + DsparkVerifyEpilogue, + TargetVerifyExecutor, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=5, stage="base-b", runner_config="1-gpu-small") + + +def _make_epilogue(device): + epilogue = DsparkVerifyEpilogue.__new__(DsparkVerifyEpilogue) + epilogue.correct_len_buf = torch.zeros((1,), dtype=torch.int32, device=device) + epilogue._result_copy_done_event = None + epilogue._result_copy_pending = False + return epilogue + + +class TestDsparkResultBufferReuse(CustomTestCase): + def test_result_copy_event_is_consumed_once_at_reuse(self): + epilogue = _make_epilogue("cpu") + event = Mock() + stream = Mock() + device_module = Mock() + device_module.Event.return_value = event + device_module.current_stream.return_value = stream + + with patch("torch.get_device_module", return_value=device_module): + epilogue.record_result_copy_done() + event.record.assert_called_once_with() + + with self.assertRaisesRegex(RuntimeError, "copied again"): + epilogue.record_result_copy_done() + + epilogue.wait_for_result_copy_done() + stream.wait_event.assert_called_once_with(event) + self.assertFalse(epilogue._result_copy_pending) + + epilogue.wait_for_result_copy_done() + self.assertEqual(stream.wait_event.call_count, 1) + + def test_target_verify_waits_at_the_buffer_reuse_point(self): + order = [] + executor = TargetVerifyExecutor.__new__(TargetVerifyExecutor) + executor._target_is_dsv41 = False + executor.verify_num_draft_tokens = 3 + executor.verify_epilogue = SimpleNamespace( + wait_for_result_copy_done=lambda: order.append("wait") + ) + executor.target_worker = SimpleNamespace( + forward_batch_generation=lambda **kwargs: ( + order.append("forward") + or SimpleNamespace(logits_output=Mock(), can_run_cuda_graph=True) + ) + ) + verify_input = SimpleNamespace( + live_seq_lens_cpu=torch.tensor([8]), + prepare_for_verify=lambda batch, worker: ( + order.append("prepare") or Mock(), + None, + ), + ) + batch = SimpleNamespace(reqs=[], seq_lens_cpu=torch.tensor([8]), seq_lens_sum=8) + + executor._forward_prepared_verify( + batch=batch, + verify_input=verify_input, + seq_lens_cpu_backup=batch.seq_lens_cpu, + seq_lens_sum_backup=batch.seq_lens_sum, + ) + + self.assertEqual(order, ["prepare", "wait", "forward"]) + + @unittest.skipUnless(torch.cuda.is_available(), "CUDA is required") + def test_delayed_d2h_finishes_before_persistent_buffer_reuse(self): + device = torch.device("cuda") + epilogue = _make_epilogue(device) + epilogue.out_tokens_buf = torch.tensor( + [[11, 12, 13], [21, 22, 23]], dtype=torch.int64, device=device + ) + epilogue.commit_lens_buf = torch.tensor( + [2, 3], dtype=torch.int32, device=device + ) + expected_tokens = epilogue.out_tokens_buf.cpu().reshape(-1) + expected_lens = epilogue.commit_lens_buf.cpu() + + result = GenerationBatchResult( + logits_output=LogitsProcessorOutput(next_token_logits=None), + next_token_ids=epilogue.out_tokens_buf.reshape(-1), + accept_lens=epilogue.commit_lens_buf, + copy_done=torch.cuda.Event(), + persistent_result_copy_owner=epilogue, + ) + forward_stream = torch.cuda.current_stream(device) + copy_stream = torch.cuda.Stream(device=device) + copy_stream.wait_stream(forward_stream) + with torch.cuda.stream(copy_stream): + torch.cuda._sleep(20_000_000) + result.copy_to_cpu(return_logprob=False, return_hidden_states=False) + + epilogue.wait_for_result_copy_done() + epilogue.out_tokens_buf.fill_(99) + epilogue.commit_lens_buf.fill_(1) + torch.cuda.synchronize(device) + + torch.testing.assert_close(result.next_token_ids, expected_tokens) + torch.testing.assert_close(result.accept_lens, expected_lens) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/test_auxiliary_output.py b/test/registered/unit/managers/test_auxiliary_output.py index c6c12a272e..ec6d509f4b 100644 --- a/test/registered/unit/managers/test_auxiliary_output.py +++ b/test/registered/unit/managers/test_auxiliary_output.py @@ -80,6 +80,52 @@ class CopyDone: self.record_count += 1 +def test_persistent_result_copy_event_precedes_optional_tensor_copies(): + next_token_ids = torch.tensor([1, 2]) + accept_lens = torch.tensor([2]) + block_accept_lens = torch.tensor([2]) + cap_lens = torch.tensor([3]) + hidden_states = torch.tensor([[4.0]]) + names = { + id(next_token_ids): "next_token_ids", + id(accept_lens): "accept_lens", + id(block_accept_lens): "block_accept_lens", + id(cap_lens): "cap_lens", + id(hidden_states): "hidden_states", + } + order = [] + owner = Mock() + owner.record_result_copy_done.side_effect = lambda: order.append("reuse_event") + result = GenerationBatchResult( + logits_output=LogitsProcessorOutput( + next_token_logits=None, hidden_states=hidden_states + ), + next_token_ids=next_token_ids, + accept_lens=accept_lens, + block_accept_lens=block_accept_lens, + cap_lens=cap_lens, + copy_done=CopyDone(), + persistent_result_copy_owner=owner, + ) + + def record_copy(tensor): + order.append(names[id(tensor)]) + return tensor + + with patch("sglang.srt.managers.utils._async_d2h", side_effect=record_copy): + result.copy_to_cpu(return_logprob=False, return_hidden_states=True) + + assert order == [ + "next_token_ids", + "accept_lens", + "block_accept_lens", + "cap_lens", + "reuse_event", + "hidden_states", + ] + assert result.persistent_result_copy_owner is None + + def _model_runner_for_sampling_path( *, spec_algorithm=SpeculativeAlgorithm.NONE,