diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 6e88182897..a37a5c09ca 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -664,6 +664,8 @@ class Envs: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False) # Opt-in: keep receiving prefill requests while forward results are pending. SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False) + # Override NVML Confidential Computing autodetection. None = autodetect. + SGLANG_FORCE_CONFIDENTIAL_COMPUTE = EnvBool(None) # =================================================================== # Radix and sparse KV caches diff --git a/python/sglang/srt/managers/async_d2h_copy_worker.py b/python/sglang/srt/managers/async_d2h_copy_worker.py new file mode 100644 index 0000000000..64c4232bd9 --- /dev/null +++ b/python/sglang/srt/managers/async_d2h_copy_worker.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import logging +import queue +import threading +from typing import Callable, Optional + +logger = logging.getLogger(__name__) + + +class HostCopyDone: + """Drop-in for a ``copy_done`` CUDA event, backed by a worker thread. + + Carries the copy's exception as well, so ``synchronize()`` can abort the + consumer instead of letting it read invalid host tensors. + """ + + def __init__(self): + self._done = threading.Event() + self.error: Optional[BaseException] = None + + def record(self, *args, **kwargs) -> None: + # No-op for parity with torch.cuda.Event; set_done is the real signal. + pass + + def set_done(self, error: Optional[BaseException] = None) -> None: + self.error = error + self._done.set() + + def synchronize(self) -> None: + """Block until the copy completes; re-raise if it failed.""" + self._done.wait() + if self.error is not None: + raise RuntimeError( + "Async device->host copy failed; destination tensors are invalid" + ) from self.error + + def query(self) -> bool: + """True once the copy has completed (successfully or not).""" + return self._done.is_set() + + +class AsyncD2HCopyWorker: + """Runs blocking device->host copies on a dedicated daemon thread. + + Under NVIDIA Confidential Computing a D2H ``cudaMemcpyAsync`` is forced + synchronous and blocks at issue, stalling the submitting thread; this moves + the copy off it so the overlap scheduler keeps launching the next step. + + ``d2h_copy_stream`` is private on purpose: on a shared stream the blanket + ``synchronize()`` below would also block on whatever the caller queued after + submitting, re-coupling the caller to the copy. + """ + + def __init__(self, device_module): + self.device_module = device_module + # Private stream, created and owned here so nothing outside this worker + # can enqueue onto it (see class docstring). Created on the caller's + # thread, so it lands on the current device. + self.d2h_copy_stream = device_module.Stream() + self._device_index = device_module.current_device() + self._queue: queue.Queue = queue.Queue() + self._thread = threading.Thread( + target=self._loop, name="sglang-d2h-copy-worker", daemon=True + ) + self._thread.start() + + def submit(self, copy_fn: Callable[[], None]) -> HostCopyDone: + """Record readiness on the CURRENT stream and enqueue the copy. + + Must be called with the stream that produced the copy sources current. + """ + src_ready = self.device_module.Event() + src_ready.record() + done = HostCopyDone() + self._queue.put((src_ready, copy_fn, done)) + return done + + def _loop(self): + self.device_module.set_device(self._device_index) + while True: + item = self._queue.get() + if item is None: + return + src_ready, copy_fn, done = item + error = None + try: + # Event-sync, not a stream wait: a stream wait would re-expose + # the caller's thread to the blocking copy. + src_ready.synchronize() + with self.device_module.stream(self.d2h_copy_stream): + copy_fn() + self.d2h_copy_stream.synchronize() + except Exception as e: + logger.exception("AsyncD2HCopyWorker copy failed") + error = e + finally: + # Signal on failure too, so the caller never hangs. + done.set_done(error=error) + + def shutdown(self, timeout: float = 2.0): + """Signal the worker to stop and join it (best-effort, bounded wait).""" + if not self._thread.is_alive(): + return + self._queue.put(None) + self._thread.join(timeout=timeout) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index a943172161..3fe1415815 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -358,7 +358,7 @@ from sglang.srt.utils import ( suppress_other_loggers, triton_load_watch, ) -from sglang.srt.utils.common import is_npu +from sglang.srt.utils.common import is_confidential_compute, is_npu from sglang.srt.utils.hf_transformers_utils import ( get_processor, get_tokenizer, @@ -1648,6 +1648,11 @@ class Scheduler( self.draft_worker.get_confidence_budget_prepare() ) + # If enable async D2H copy when overlap is enabled + # Currently used by NVIDIA Confidential Computing (CC) only + self.enable_async_d2h_copy = False + self.async_d2h_worker = None + if use_mlx(): # MLX uses its own overlap loop and does not create CUDA streams, # but the normal non-overlap scheduler path still relays decode @@ -1668,6 +1673,19 @@ class Scheduler( if not self.enable_overlap: return + # Under NVIDIA Confidential Computing (CC) the per-step D2H result readback is + # forced synchronous and blocks the scheduler thread at issue, serializing the + # overlap pipeline. Offload it to a worker thread (AsyncD2HCopyWorker). + self.enable_async_d2h_copy = is_confidential_compute() + if self.enable_async_d2h_copy: + from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker + + self.async_d2h_worker = AsyncD2HCopyWorker(self.device_module) + logger.info( + "NVIDIA Confidential Computing (CC) detected: " + "using async D2H copy worker to preserve overlap scheduling." + ) + self.batch_record_buf = [None] * 2 self.batch_record_ct = 0 @@ -4632,10 +4650,17 @@ class Scheduler( # gated by copy_done, so nothing on forward_stream waits. self.copy_stream.wait_stream(self.forward_stream) with self.copy_stream_ctx: - batch_result.copy_to_cpu( + copy_fn = partial( + batch_result.copy_to_cpu, return_logprob=batch.return_logprob, return_hidden_states=batch.return_hidden_states, ) + if self.enable_async_d2h_copy: + batch_result.copy_done = ( + self.async_d2h_worker.submit(copy_fn) + ) + else: + copy_fn() else: batch_result.future_indices = future_indices @@ -4931,10 +4956,15 @@ class Scheduler( # with subsequent forward computation. self.copy_stream.wait_stream(self.forward_stream) with self.copy_stream_ctx: - batch_result.copy_to_cpu( + copy_fn = partial( + batch_result.copy_to_cpu, return_logprob=cur_batch.return_logprob, return_hidden_states=cur_batch.return_hidden_states, ) + if self.enable_async_d2h_copy: + batch_result.copy_done = self.async_d2h_worker.submit(copy_fn) + else: + copy_fn() # Release the closure and large GPU tensors that are no longer needed. # The delay_sample_func closure captures forward_batch (which holds @@ -6191,6 +6221,10 @@ def run_scheduler_process( # FPM has a background ZMQ publisher thread that needs explicit # teardown to flush queued metrics and close the socket cleanly. scheduler.metrics_reporter._shutdown_fpm() + # Stop the async D2H copy worker thread if any. + if scheduler.async_d2h_worker is not None: + scheduler.async_d2h_worker.shutdown() + scheduler.async_d2h_worker = None # Graceful path only: on the exception path the GPU may be wedged # and the synchronize() in destroy() could itself hang. if scheduler.gracefully_exit: diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index 320a3c69ac..1a0b2ad4c1 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -24,6 +24,7 @@ from sglang.srt.state_capturer.base import TopkCaptureOutput from sglang.srt.utils.common import async_d2h as _async_d2h if TYPE_CHECKING: + from sglang.srt.managers.async_d2h_copy_worker import HostCopyDone from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput from sglang.srt.managers.scheduler import GenerationBatchResult from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan @@ -73,7 +74,7 @@ class GenerationBatchResult: extend_logprob_start_len_per_req: Optional[List[int]] = None # For overlap scheduling - copy_done: Optional[torch.cuda.Event] = None + copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None delay_sample_func: Optional[callable] = None future_indices: Optional[torch.Tensor] = None speculative_num_draft_tokens: Optional[int] = None @@ -390,7 +391,7 @@ class EmbeddingBatchResult: embeddings: torch.Tensor pooled_hidden_states: Optional[torch.Tensor] = None - copy_done: Optional[torch.cuda.Event] = None + copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None can_run_cuda_graph: bool = False @torch.profiler.record_function("copy_embedding_to_cpu") diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 6202fd7019..e201e8b5d7 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -51,6 +51,7 @@ import warnings from array import array from collections import OrderedDict, defaultdict from contextlib import contextmanager +from ctypes import byref from dataclasses import dataclass from decimal import Decimal from functools import lru_cache, partial @@ -5134,6 +5135,59 @@ def get_or_create_event_loop(): return loop +@lru_cache(maxsize=1) +def is_confidential_compute() -> bool: + """Whether the GPU is running in NVIDIA Confidential Computing (CC) mode. + + Detected once via NVML and cached. + Overridable with ``SGLANG_FORCE_CONFIDENTIAL_COMPUTE``. + """ + forced = envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.get() + if forced is not None: + return forced + if not is_cuda(): + return False + + try: + import pynvml + except ImportError: + logger.warning("pynvml not available; assuming Confidential Computing is off") + return False + + cc_enabled = False + try: + pynvml.nvmlInit() + + # Hopper and newer supports a more nuanced query of confidential + # compute settings + cc_settings = pynvml.c_nvmlSystemConfComputeSettings_v1_t() + ret = pynvml.nvmlSystemGetConfComputeSettings(byref(cc_settings)) + pynvml._nvmlCheckReturn(ret) + # PPCIE implies CC + cc_enabled = ( + cc_settings.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED + or cc_settings.multiGpuMode == pynvml.NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE + ) + except pynvml.NVMLError_NotSupported: + # Simple query for older GPUs + try: + cc_state = pynvml.nvmlSystemGetConfComputeState() + cc_enabled = cc_state.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED + except pynvml.NVMLError as error: + logger.warning(f"Error querying CC state: {error!s}") + except pynvml.NVMLError as error: + logger.warning(f"Error querying CC state: {error!s}") + finally: + # Shutdown + try: + pynvml.nvmlShutdown() + except pynvml.NVMLError: + # Ignore shutdown errors + pass + + return cc_enabled + + def init_cublas(): """We need to run a small matmul to init cublas. Otherwise, it will raise some errors later.""" dtype = torch.float16 diff --git a/test/registered/scheduler/test_async_d2h_copy_worker_server.py b/test/registered/scheduler/test_async_d2h_copy_worker_server.py new file mode 100644 index 0000000000..54da65fc7b --- /dev/null +++ b/test/registered/scheduler/test_async_d2h_copy_worker_server.py @@ -0,0 +1,119 @@ +"""Server-level coverage for the async D2H result readback (AsyncD2HCopyWorker). + +Under Confidential Computing the overlap scheduler hands its per-step readback +to a worker thread and stores a HostCopyDone in `copy_done`. CI GPUs never run +CC, so without the force override nothing exercises that path end to end: +`run_batch`, `launch_batch_sample_if_needed`, and every `copy_done.synchronize()` +consumer keep taking the inline branch. + +Both scheduler submit sites are covered: the inline-sample path, and the +delay-sample path via SGLANG_ENABLE_DELAY_SAMPLE. +""" + +import unittest +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import ExitStack + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=180, stage="base-b", runner_config="1-gpu-small") + +MAX_NEW_TOKENS = 32 +REPEATS = 2 +# Distinct answers make a readback that returns another row's tokens visible. +# Identical prompts cannot: every row would still carry a plausible answer. +PROMPTS = ( + ("The capital of France is", "Paris"), + ("The capital of Japan is", "Tokyo"), + ("The capital of Italy is", "Rome"), + ("The capital of Germany is", "Berlin"), + ("The capital of Spain is", "Madrid"), + ("The capital of Russia is", "Moscow"), + ("The capital of Egypt is", "Cairo"), + ("The capital of Greece is", "Athens"), +) + + +class TestAsyncD2HCopyWorkerServer(CustomTestCase): + """python -m unittest test_async_d2h_copy_worker_server.TestAsyncD2HCopyWorkerServer""" + + process = None + # (EnvField, value) pairs that subclasses add on top of the force flag. + extra_env_overrides: tuple = () + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + with ExitStack() as stack: + stack.enter_context(envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(True)) + for field, value in cls.extra_env_overrides: + stack.enter_context(field.override(value)) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--mem-fraction-static", "0.7"], + ) + + @classmethod + def tearDownClass(cls): + if cls.process is not None: + kill_process_tree(cls.process.pid) + + def _generate(self, prompt: str) -> dict: + payload = { + "text": prompt, + "sampling_params": { + "max_new_tokens": MAX_NEW_TOKENS, + "temperature": 0.0, + "ignore_eos": True, + }, + "return_logprob": True, + "logprob_start_len": -1, + } + r = requests.post(f"{self.base_url}/generate", json=payload, timeout=600) + r.raise_for_status() + return r.json() + + def test_concurrent_decode_readback(self): + # Not asserting that repeats of one prompt agree: batching is not + # batch-invariant, so two coherent completions can legitimately differ. + cases = list(PROMPTS) * REPEATS + with ThreadPoolExecutor(max_workers=len(cases)) as pool: + futures = { + pool.submit(self._generate, prompt): (prompt, answer) + for prompt, answer in cases + } + for future in as_completed(futures): + prompt, answer = futures[future] + result = future.result() + self.assertIn(answer, result["text"], f"{prompt!r} -> wrong answer") + + meta = result["meta_info"] + self.assertEqual(meta["completion_tokens"], MAX_NEW_TOKENS) + # The logprob readback rides the same copy as next_token_ids. + self.assertEqual( + len(meta["output_token_logprobs"]), meta["completion_tokens"] + ) + + +class TestAsyncD2HCopyWorkerServerDelaySample(TestAsyncD2HCopyWorkerServer): + """Covers the launch_batch_sample_if_needed submit site.""" + + extra_env_overrides = ((envs.SGLANG_ENABLE_DELAY_SAMPLE, True),) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_prefill_result_polling.py b/test/registered/unit/disaggregation/test_prefill_result_polling.py index c2577a9e34..edf9cae7d3 100644 --- a/test/registered/unit/disaggregation/test_prefill_result_polling.py +++ b/test/registered/unit/disaggregation/test_prefill_result_polling.py @@ -169,6 +169,7 @@ class _Scheduler(SchedulerDisaggregationPrefillMixin): self.gracefully_exit = False self.enable_staging = False self.enable_overlap = True + self.enable_async_d2h_copy = False self.enable_overlap_mlx = False self.disagg_prefill_bootstrap_queue = SimpleNamespace( pop_bootstrapped=lambda: [] diff --git a/test/registered/unit/managers/test_async_d2h_copy_worker.py b/test/registered/unit/managers/test_async_d2h_copy_worker.py new file mode 100644 index 0000000000..9fe1536d07 --- /dev/null +++ b/test/registered/unit/managers/test_async_d2h_copy_worker.py @@ -0,0 +1,117 @@ +"""Unit tests for the async device->host copy worker (AsyncD2HCopyWorker). + +These validate that ``AsyncD2HCopyWorker`` performs the device->host copy +correctly off the calling thread, and that the CC detection env override works. + +They need a GPU but NO model and NO actual confidential-compute hardware: the +worker logic is identical regardless of CC (CC only changes whether the +scheduler routes the readback through the worker). To exercise the worker on an +ordinary GPU, run inside the sglang container: + + python -m pytest test/registered/unit/managers/test_async_d2h_copy_worker.py -v +""" + +import unittest + +import torch + +from sglang.srt.environ import envs +from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker +from sglang.srt.utils.common import is_confidential_compute +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small") + + +@unittest.skipUnless(torch.cuda.is_available(), "AsyncD2HCopyWorker requires CUDA") +class TestAsyncD2HCopyWorker(CustomTestCase): + def setUp(self): + self.worker = AsyncD2HCopyWorker(torch.cuda) + + def tearDown(self): + # Idempotent: a no-op if a test already shut the worker down. + self.worker.shutdown() + + def _run_one(self, numel: int): + src = torch.randn(numel, device="cuda") + expected = src.detach().to("cpu") # synchronous reference copy + + out = {} + + def copy_fn(): + out["cpu"] = src.to("cpu", non_blocking=True) + + # submit() records readiness on the current stream (where src was + # produced) before handing the copy to the worker. + done = self.worker.submit(copy_fn) + + self.assertTrue(done._done.wait(timeout=30), "worker did not signal completion") + done.synchronize() # drop-in for copy_done; must not raise on success + self.assertIn("cpu", out) + torch.testing.assert_close(out["cpu"], expected) + + def test_single_readback_matches_synchronous(self): + self._run_one(4) # tiny, like next_token_ids at small batch + self._run_one(151936) # vocab-sized, like a logprob row + + def test_copy_waits_for_the_producing_stream(self): + # Guards submit()'s src_ready.record(): with the fill still queued behind + # slow work, a worker that skipped the event would read zeros. + side_stream = torch.cuda.Stream() + src = torch.zeros(4096, device="cuda") + + out = {} + + def copy_fn(): + out["cpu"] = src.to("cpu", non_blocking=True) + + with torch.cuda.stream(side_stream): + torch.cuda._sleep(200_000_000) # ~100ms at 2GHz; outlives the handoff + src.fill_(7) + done = self.worker.submit(copy_fn) + + self.assertTrue(done._done.wait(timeout=30), "worker did not signal completion") + done.synchronize() + torch.testing.assert_close(out["cpu"], torch.full((4096,), 7.0)) + + def test_shutdown_is_idempotent_and_joins(self): + self.worker.shutdown() + self.worker.shutdown() # second call must be a safe no-op + self.assertFalse(self.worker._thread.is_alive()) + + def test_copy_fn_exception_reports_error(self): + # A failing copy must not hang the scheduler, but synchronize() must + # re-raise so the consumer aborts instead of reading invalid CPU tensors. + def boom(): + raise RuntimeError("injected copy failure") + + done = self.worker.submit(boom) + self.assertTrue( + done._done.wait(timeout=30), "handle must be signaled even on failure" + ) + self.assertIsNotNone(done.error, "failed copy must record its error") + with self.assertRaises(RuntimeError): + done.synchronize() + + +class TestConfidentialComputeDetection(CustomTestCase): + """CC detection env override. Does not require CUDA (override short-circuits).""" + + def test_env_override_true(self): + is_confidential_compute.cache_clear() + with envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(True): + is_confidential_compute.cache_clear() + self.assertTrue(is_confidential_compute()) + is_confidential_compute.cache_clear() + + def test_env_override_false(self): + is_confidential_compute.cache_clear() + with envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(False): + is_confidential_compute.cache_clear() + self.assertFalse(is_confidential_compute()) + is_confidential_compute.cache_clear() + + +if __name__ == "__main__": + unittest.main()