From bda6ef7e01f6c2132e29bdf4be7be908128536e1 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Thu, 15 Oct 2026 17:48:50 -0700 Subject: [PATCH] [Perf] [NVIDIA] scheduler: Pageable host staging and inline hidden-state/DSpark readback under Confidential Computing (#44568) Co-authored-by: Ye (Charlotte) Qi <7269575+yeqcharlotte@users.noreply.github.com> --- python/sglang/srt/managers/scheduler.py | 58 +++++---- python/sglang/srt/utils/common.py | 4 + .../test_prefill_result_polling.py | 1 + .../managers/test_async_d2h_copy_worker.py | 111 +++++++++++++++++- 4 files changed, 142 insertions(+), 32 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 3fe1415815..2a582af940 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4645,22 +4645,7 @@ class Scheduler( return_hidden_states=batch.return_hidden_states, ) else: - # Result D2H on copy_stream overlaps the next forward - # instead of serializing on forward_stream; it's a leaf - # gated by copy_done, so nothing on forward_stream waits. - self.copy_stream.wait_stream(self.forward_stream) - with self.copy_stream_ctx: - copy_fn = partial( - batch_result.copy_to_cpu, - return_logprob=batch.return_logprob, - return_hidden_states=batch.return_hidden_states, - ) - if self.enable_async_d2h_copy: - batch_result.copy_done = ( - self.async_d2h_worker.submit(copy_fn) - ) - else: - copy_fn() + self._copy_result_to_cpu(batch_result, batch) else: batch_result.future_indices = future_indices @@ -4934,6 +4919,32 @@ class Scheduler( return_hidden_states=batch.return_hidden_states, ) + def _copy_result_to_cpu( + self, batch_result: GenerationBatchResult, batch: ScheduleBatch + ) -> None: + """Issue the result readback on copy_stream so it overlaps the next forward. + + copy_stream is a leaf gated by copy_done that forward_stream never waits + for. Under Confidential Computing the copy would block this thread until + the forward is done, so the worker issues it from its own thread instead. + """ + copy_fn = partial( + batch_result.copy_to_cpu, + return_logprob=batch.return_logprob, + return_hidden_states=batch.return_hidden_states, + ) + self.copy_stream.wait_stream(self.forward_stream) + with self.copy_stream_ctx: + if not self.enable_async_d2h_copy: + copy_fn() + elif batch.return_hidden_states or batch.spec_algorithm.is_dspark(): + # Hidden states and the DSpark accept lengths are views of CUDA + # graph buffers that the next replay rewrites; only a copy queued + # right behind this forward reads them in time. + copy_fn() + else: + batch_result.copy_done = self.async_d2h_worker.submit(copy_fn) + def launch_batch_sample_if_needed( self, batch_result: GenerationBatchResult, cur_batch: ScheduleBatch ) -> Union[GenerationBatchResult]: @@ -4951,20 +4962,7 @@ class Scheduler( cur_batch, batch_result.future_indices, batch_result ) - # Run device-to-host copy on a separate stream to avoid blocking the - # forward stream. The copy waits for the sampled result and can overlap - # with subsequent forward computation. - self.copy_stream.wait_stream(self.forward_stream) - with self.copy_stream_ctx: - copy_fn = partial( - batch_result.copy_to_cpu, - return_logprob=cur_batch.return_logprob, - return_hidden_states=cur_batch.return_hidden_states, - ) - if self.enable_async_d2h_copy: - batch_result.copy_done = self.async_d2h_worker.submit(copy_fn) - else: - copy_fn() + self._copy_result_to_cpu(batch_result, cur_batch) # Release the closure and large GPU tensors that are no longer needed. # The delay_sample_func closure captures forward_batch (which holds diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 11fa646841..04fc1026a4 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -583,6 +583,10 @@ def get_available_gpu_memory( def is_pin_memory_available(device=None) -> bool: + # Under NVIDIA Confidential Computing host<->device copies go through a + # bounce buffer, so host staging is pageable there. + if is_confidential_compute(): + return False return current_platform.is_pin_memory_available(device) diff --git a/test/registered/unit/disaggregation/test_prefill_result_polling.py b/test/registered/unit/disaggregation/test_prefill_result_polling.py index 96873ff6b9..0898c485d0 100644 --- a/test/registered/unit/disaggregation/test_prefill_result_polling.py +++ b/test/registered/unit/disaggregation/test_prefill_result_polling.py @@ -118,6 +118,7 @@ class _Scheduler(SchedulerDisaggregationPrefillMixin): """Run real scheduling and result handling with controlled intake and GPU events.""" launch_batch_sample_if_needed = Scheduler.launch_batch_sample_if_needed + _copy_result_to_cpu = Scheduler._copy_result_to_cpu pause_generation = Scheduler.pause_generation continue_generation = Scheduler.continue_generation diff --git a/test/registered/unit/managers/test_async_d2h_copy_worker.py b/test/registered/unit/managers/test_async_d2h_copy_worker.py index 9fe1536d07..a1980512de 100644 --- a/test/registered/unit/managers/test_async_d2h_copy_worker.py +++ b/test/registered/unit/managers/test_async_d2h_copy_worker.py @@ -12,12 +12,17 @@ ordinary GPU, run inside the sglang container: """ import unittest +from types import SimpleNamespace import torch from sglang.srt.environ import envs -from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker -from sglang.srt.utils.common import is_confidential_compute +from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker, HostCopyDone +from sglang.srt.utils.common import ( + async_d2h, + is_confidential_compute, + is_pin_memory_available, +) from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -113,5 +118,107 @@ class TestConfidentialComputeDetection(CustomTestCase): is_confidential_compute.cache_clear() +SLEEP_CYCLES = 200_000_000 # ~100ms at 2GHz; outlives the handoff to the worker + + +def _late_source(value: int, numel: int = 4) -> torch.Tensor: + """A device tensor that holds ``value`` only after a long kernel on this stream.""" + src = torch.zeros(numel, dtype=torch.int64, device="cuda") + torch.cuda._sleep(SLEEP_CYCLES) + return src.fill_(value) + + +@unittest.skipUnless(torch.cuda.is_available(), "pinned host memory requires CUDA") +class TestConfidentialComputeStaging(CustomTestCase): + """Host staging is pageable under CC and pinned otherwise.""" + + def setUp(self): + is_confidential_compute.cache_clear() + self.addCleanup(is_confidential_compute.cache_clear) + + def test_async_d2h_stages_pageable_under_cc(self): + src = torch.arange(4, device="cuda") + for cc in (False, True): + with ( + self.subTest(cc=cc), + envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(cc), + ): + is_confidential_compute.cache_clear() + self.assertEqual(is_pin_memory_available(src.device), not cc) + host = async_d2h(src) + torch.cuda.current_stream().synchronize() + self.assertEqual(host.is_pinned(), not cc) + self.assertEqual(host.tolist(), [0, 1, 2, 3]) + + +@unittest.skipUnless(torch.cuda.is_available(), "the scheduler readback requires CUDA") +class TestSchedulerReadbackRouting(CustomTestCase): + """Every readback path must wait for the forward: a copy issued too early reads + the zeros the source held before it. Under CC only plain batches take the + worker; hidden-state and DSpark batches stay inline, since they read CUDA + graph buffers that the next replay rewrites.""" + + def setUp(self): + is_confidential_compute.cache_clear() + self.addCleanup(is_confidential_compute.cache_clear) + self.worker = AsyncD2HCopyWorker(torch.cuda) + self.addCleanup(self.worker.shutdown) + + def _readback(self, cc: bool, hidden: bool, dspark: bool): + from sglang.srt.managers.scheduler import Scheduler + from sglang.srt.managers.utils import GenerationBatchResult + from sglang.srt.speculative.spec_info import SpeculativeAlgorithm + + forward_stream, copy_stream = torch.cuda.Stream(), torch.cuda.Stream() + scheduler = SimpleNamespace( + forward_stream=forward_stream, + copy_stream=copy_stream, + copy_stream_ctx=torch.cuda.stream(copy_stream), + enable_async_d2h_copy=cc, + async_d2h_worker=self.worker if cc else None, + ) + # Distinct per case, so a reused host block cannot pass a stale value. + value = 7 + cc + hidden + 2 * dspark + with torch.cuda.stream(forward_stream): + ids = _late_source(value, numel=1 << 24) # 128 MiB: the copy takes a while + logits = SimpleNamespace( + hidden_states=_late_source(value) if hidden else None, + sampling_mask_output=None, + auxiliary_device_output=None, + ) + result = GenerationBatchResult( + logits_output=logits, next_token_ids=ids, copy_done=torch.cuda.Event() + ) + algorithm = SpeculativeAlgorithm.DSPARK if dspark else SpeculativeAlgorithm.NONE + batch = SimpleNamespace( + return_logprob=False, return_hidden_states=hidden, spec_algorithm=algorithm + ) + with envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(cc): + is_confidential_compute.cache_clear() + Scheduler._copy_result_to_cpu(scheduler, result, batch) + result.copy_done.synchronize() + return result, logits.hidden_states, value + + def test_readback_lands_after_the_forward(self): + cases = { + "off-inline": (False, False, False), + "cc-worker": (True, False, False), + "cc-hidden-states-inline": (True, True, False), + "cc-dspark-inline": (True, False, True), + } + for name, (cc, hidden, dspark) in cases.items(): + with self.subTest(name): + result, hidden_states, value = self._readback(cc, hidden, dspark) + offloaded = cc and not (hidden or dspark) + self.assertEqual(isinstance(result.copy_done, HostCopyDone), offloaded) + for host in (result.next_token_ids, hidden_states): + if host is None: + continue + self.assertEqual( + (host.device.type, host.is_pinned()), ("cpu", not cc) + ) + self.assertTrue(torch.equal(host, torch.full_like(host, value))) + + if __name__ == "__main__": unittest.main()