perf: fix overlap scheduling for NVIDIA Confidential Computing(CC) on Blackwell (#36810)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
elvischenv
2026-10-15 12:09:09 -07:00
committed by GitHub
co-authored by Claude Opus 5 Po-Han Huang Po-Han Huang
parent 02a43eeef6
commit 87211088c9
8 changed files with 439 additions and 5 deletions
+2
View File
@@ -664,6 +664,8 @@ class Envs:
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
# Opt-in: keep receiving prefill requests while forward results are pending.
SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False)
# Override NVML Confidential Computing autodetection. None = autodetect.
SGLANG_FORCE_CONFIDENTIAL_COMPUTE = EnvBool(None)
# ===================================================================
# Radix and sparse KV caches
@@ -0,0 +1,106 @@
from __future__ import annotations
import logging
import queue
import threading
from typing import Callable, Optional
logger = logging.getLogger(__name__)
class HostCopyDone:
"""Drop-in for a ``copy_done`` CUDA event, backed by a worker thread.
Carries the copy's exception as well, so ``synchronize()`` can abort the
consumer instead of letting it read invalid host tensors.
"""
def __init__(self):
self._done = threading.Event()
self.error: Optional[BaseException] = None
def record(self, *args, **kwargs) -> None:
# No-op for parity with torch.cuda.Event; set_done is the real signal.
pass
def set_done(self, error: Optional[BaseException] = None) -> None:
self.error = error
self._done.set()
def synchronize(self) -> None:
"""Block until the copy completes; re-raise if it failed."""
self._done.wait()
if self.error is not None:
raise RuntimeError(
"Async device->host copy failed; destination tensors are invalid"
) from self.error
def query(self) -> bool:
"""True once the copy has completed (successfully or not)."""
return self._done.is_set()
class AsyncD2HCopyWorker:
"""Runs blocking device->host copies on a dedicated daemon thread.
Under NVIDIA Confidential Computing a D2H ``cudaMemcpyAsync`` is forced
synchronous and blocks at issue, stalling the submitting thread; this moves
the copy off it so the overlap scheduler keeps launching the next step.
``d2h_copy_stream`` is private on purpose: on a shared stream the blanket
``synchronize()`` below would also block on whatever the caller queued after
submitting, re-coupling the caller to the copy.
"""
def __init__(self, device_module):
self.device_module = device_module
# Private stream, created and owned here so nothing outside this worker
# can enqueue onto it (see class docstring). Created on the caller's
# thread, so it lands on the current device.
self.d2h_copy_stream = device_module.Stream()
self._device_index = device_module.current_device()
self._queue: queue.Queue = queue.Queue()
self._thread = threading.Thread(
target=self._loop, name="sglang-d2h-copy-worker", daemon=True
)
self._thread.start()
def submit(self, copy_fn: Callable[[], None]) -> HostCopyDone:
"""Record readiness on the CURRENT stream and enqueue the copy.
Must be called with the stream that produced the copy sources current.
"""
src_ready = self.device_module.Event()
src_ready.record()
done = HostCopyDone()
self._queue.put((src_ready, copy_fn, done))
return done
def _loop(self):
self.device_module.set_device(self._device_index)
while True:
item = self._queue.get()
if item is None:
return
src_ready, copy_fn, done = item
error = None
try:
# Event-sync, not a stream wait: a stream wait would re-expose
# the caller's thread to the blocking copy.
src_ready.synchronize()
with self.device_module.stream(self.d2h_copy_stream):
copy_fn()
self.d2h_copy_stream.synchronize()
except Exception as e:
logger.exception("AsyncD2HCopyWorker copy failed")
error = e
finally:
# Signal on failure too, so the caller never hangs.
done.set_done(error=error)
def shutdown(self, timeout: float = 2.0):
"""Signal the worker to stop and join it (best-effort, bounded wait)."""
if not self._thread.is_alive():
return
self._queue.put(None)
self._thread.join(timeout=timeout)
+37 -3
View File
@@ -358,7 +358,7 @@ from sglang.srt.utils import (
suppress_other_loggers,
triton_load_watch,
)
from sglang.srt.utils.common import is_npu
from sglang.srt.utils.common import is_confidential_compute, is_npu
from sglang.srt.utils.hf_transformers_utils import (
get_processor,
get_tokenizer,
@@ -1648,6 +1648,11 @@ class Scheduler(
self.draft_worker.get_confidence_budget_prepare()
)
# If enable async D2H copy when overlap is enabled
# Currently used by NVIDIA Confidential Computing (CC) only
self.enable_async_d2h_copy = False
self.async_d2h_worker = None
if use_mlx():
# MLX uses its own overlap loop and does not create CUDA streams,
# but the normal non-overlap scheduler path still relays decode
@@ -1668,6 +1673,19 @@ class Scheduler(
if not self.enable_overlap:
return
# Under NVIDIA Confidential Computing (CC) the per-step D2H result readback is
# forced synchronous and blocks the scheduler thread at issue, serializing the
# overlap pipeline. Offload it to a worker thread (AsyncD2HCopyWorker).
self.enable_async_d2h_copy = is_confidential_compute()
if self.enable_async_d2h_copy:
from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker
self.async_d2h_worker = AsyncD2HCopyWorker(self.device_module)
logger.info(
"NVIDIA Confidential Computing (CC) detected: "
"using async D2H copy worker to preserve overlap scheduling."
)
self.batch_record_buf = [None] * 2
self.batch_record_ct = 0
@@ -4632,10 +4650,17 @@ class Scheduler(
# gated by copy_done, so nothing on forward_stream waits.
self.copy_stream.wait_stream(self.forward_stream)
with self.copy_stream_ctx:
batch_result.copy_to_cpu(
copy_fn = partial(
batch_result.copy_to_cpu,
return_logprob=batch.return_logprob,
return_hidden_states=batch.return_hidden_states,
)
if self.enable_async_d2h_copy:
batch_result.copy_done = (
self.async_d2h_worker.submit(copy_fn)
)
else:
copy_fn()
else:
batch_result.future_indices = future_indices
@@ -4931,10 +4956,15 @@ class Scheduler(
# with subsequent forward computation.
self.copy_stream.wait_stream(self.forward_stream)
with self.copy_stream_ctx:
batch_result.copy_to_cpu(
copy_fn = partial(
batch_result.copy_to_cpu,
return_logprob=cur_batch.return_logprob,
return_hidden_states=cur_batch.return_hidden_states,
)
if self.enable_async_d2h_copy:
batch_result.copy_done = self.async_d2h_worker.submit(copy_fn)
else:
copy_fn()
# Release the closure and large GPU tensors that are no longer needed.
# The delay_sample_func closure captures forward_batch (which holds
@@ -6191,6 +6221,10 @@ def run_scheduler_process(
# FPM has a background ZMQ publisher thread that needs explicit
# teardown to flush queued metrics and close the socket cleanly.
scheduler.metrics_reporter._shutdown_fpm()
# Stop the async D2H copy worker thread if any.
if scheduler.async_d2h_worker is not None:
scheduler.async_d2h_worker.shutdown()
scheduler.async_d2h_worker = None
# Graceful path only: on the exception path the GPU may be wedged
# and the synchronize() in destroy() could itself hang.
if scheduler.gracefully_exit:
+3 -2
View File
@@ -24,6 +24,7 @@ from sglang.srt.state_capturer.base import TopkCaptureOutput
from sglang.srt.utils.common import async_d2h as _async_d2h
if TYPE_CHECKING:
from sglang.srt.managers.async_d2h_copy_worker import HostCopyDone
from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
@@ -73,7 +74,7 @@ class GenerationBatchResult:
extend_logprob_start_len_per_req: Optional[List[int]] = None
# For overlap scheduling
copy_done: Optional[torch.cuda.Event] = None
copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None
delay_sample_func: Optional[callable] = None
future_indices: Optional[torch.Tensor] = None
speculative_num_draft_tokens: Optional[int] = None
@@ -390,7 +391,7 @@ class EmbeddingBatchResult:
embeddings: torch.Tensor
pooled_hidden_states: Optional[torch.Tensor] = None
copy_done: Optional[torch.cuda.Event] = None
copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None
can_run_cuda_graph: bool = False
@torch.profiler.record_function("copy_embedding_to_cpu")
+54
View File
@@ -51,6 +51,7 @@ import warnings
from array import array
from collections import OrderedDict, defaultdict
from contextlib import contextmanager
from ctypes import byref
from dataclasses import dataclass
from decimal import Decimal
from functools import lru_cache, partial
@@ -5134,6 +5135,59 @@ def get_or_create_event_loop():
return loop
@lru_cache(maxsize=1)
def is_confidential_compute() -> bool:
"""Whether the GPU is running in NVIDIA Confidential Computing (CC) mode.
Detected once via NVML and cached.
Overridable with ``SGLANG_FORCE_CONFIDENTIAL_COMPUTE``.
"""
forced = envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.get()
if forced is not None:
return forced
if not is_cuda():
return False
try:
import pynvml
except ImportError:
logger.warning("pynvml not available; assuming Confidential Computing is off")
return False
cc_enabled = False
try:
pynvml.nvmlInit()
# Hopper and newer supports a more nuanced query of confidential
# compute settings
cc_settings = pynvml.c_nvmlSystemConfComputeSettings_v1_t()
ret = pynvml.nvmlSystemGetConfComputeSettings(byref(cc_settings))
pynvml._nvmlCheckReturn(ret)
# PPCIE implies CC
cc_enabled = (
cc_settings.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED
or cc_settings.multiGpuMode == pynvml.NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE
)
except pynvml.NVMLError_NotSupported:
# Simple query for older GPUs
try:
cc_state = pynvml.nvmlSystemGetConfComputeState()
cc_enabled = cc_state.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED
except pynvml.NVMLError as error:
logger.warning(f"Error querying CC state: {error!s}")
except pynvml.NVMLError as error:
logger.warning(f"Error querying CC state: {error!s}")
finally:
# Shutdown
try:
pynvml.nvmlShutdown()
except pynvml.NVMLError:
# Ignore shutdown errors
pass
return cc_enabled
def init_cublas():
"""We need to run a small matmul to init cublas. Otherwise, it will raise some errors later."""
dtype = torch.float16
@@ -0,0 +1,119 @@
"""Server-level coverage for the async D2H result readback (AsyncD2HCopyWorker).
Under Confidential Computing the overlap scheduler hands its per-step readback
to a worker thread and stores a HostCopyDone in `copy_done`. CI GPUs never run
CC, so without the force override nothing exercises that path end to end:
`run_batch`, `launch_batch_sample_if_needed`, and every `copy_done.synchronize()`
consumer keep taking the inline branch.
Both scheduler submit sites are covered: the inline-sample path, and the
delay-sample path via SGLANG_ENABLE_DELAY_SAMPLE.
"""
import unittest
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import ExitStack
import requests
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=180, stage="base-b", runner_config="1-gpu-small")
MAX_NEW_TOKENS = 32
REPEATS = 2
# Distinct answers make a readback that returns another row's tokens visible.
# Identical prompts cannot: every row would still carry a plausible answer.
PROMPTS = (
("The capital of France is", "Paris"),
("The capital of Japan is", "Tokyo"),
("The capital of Italy is", "Rome"),
("The capital of Germany is", "Berlin"),
("The capital of Spain is", "Madrid"),
("The capital of Russia is", "Moscow"),
("The capital of Egypt is", "Cairo"),
("The capital of Greece is", "Athens"),
)
class TestAsyncD2HCopyWorkerServer(CustomTestCase):
"""python -m unittest test_async_d2h_copy_worker_server.TestAsyncD2HCopyWorkerServer"""
process = None
# (EnvField, value) pairs that subclasses add on top of the force flag.
extra_env_overrides: tuple = ()
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
with ExitStack() as stack:
stack.enter_context(envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(True))
for field, value in cls.extra_env_overrides:
stack.enter_context(field.override(value))
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--mem-fraction-static", "0.7"],
)
@classmethod
def tearDownClass(cls):
if cls.process is not None:
kill_process_tree(cls.process.pid)
def _generate(self, prompt: str) -> dict:
payload = {
"text": prompt,
"sampling_params": {
"max_new_tokens": MAX_NEW_TOKENS,
"temperature": 0.0,
"ignore_eos": True,
},
"return_logprob": True,
"logprob_start_len": -1,
}
r = requests.post(f"{self.base_url}/generate", json=payload, timeout=600)
r.raise_for_status()
return r.json()
def test_concurrent_decode_readback(self):
# Not asserting that repeats of one prompt agree: batching is not
# batch-invariant, so two coherent completions can legitimately differ.
cases = list(PROMPTS) * REPEATS
with ThreadPoolExecutor(max_workers=len(cases)) as pool:
futures = {
pool.submit(self._generate, prompt): (prompt, answer)
for prompt, answer in cases
}
for future in as_completed(futures):
prompt, answer = futures[future]
result = future.result()
self.assertIn(answer, result["text"], f"{prompt!r} -> wrong answer")
meta = result["meta_info"]
self.assertEqual(meta["completion_tokens"], MAX_NEW_TOKENS)
# The logprob readback rides the same copy as next_token_ids.
self.assertEqual(
len(meta["output_token_logprobs"]), meta["completion_tokens"]
)
class TestAsyncD2HCopyWorkerServerDelaySample(TestAsyncD2HCopyWorkerServer):
"""Covers the launch_batch_sample_if_needed submit site."""
extra_env_overrides = ((envs.SGLANG_ENABLE_DELAY_SAMPLE, True),)
if __name__ == "__main__":
unittest.main()
@@ -169,6 +169,7 @@ class _Scheduler(SchedulerDisaggregationPrefillMixin):
self.gracefully_exit = False
self.enable_staging = False
self.enable_overlap = True
self.enable_async_d2h_copy = False
self.enable_overlap_mlx = False
self.disagg_prefill_bootstrap_queue = SimpleNamespace(
pop_bootstrapped=lambda: []
@@ -0,0 +1,117 @@
"""Unit tests for the async device->host copy worker (AsyncD2HCopyWorker).
These validate that ``AsyncD2HCopyWorker`` performs the device->host copy
correctly off the calling thread, and that the CC detection env override works.
They need a GPU but NO model and NO actual confidential-compute hardware: the
worker logic is identical regardless of CC (CC only changes whether the
scheduler routes the readback through the worker). To exercise the worker on an
ordinary GPU, run inside the sglang container:
python -m pytest test/registered/unit/managers/test_async_d2h_copy_worker.py -v
"""
import unittest
import torch
from sglang.srt.environ import envs
from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker
from sglang.srt.utils.common import is_confidential_compute
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small")
@unittest.skipUnless(torch.cuda.is_available(), "AsyncD2HCopyWorker requires CUDA")
class TestAsyncD2HCopyWorker(CustomTestCase):
def setUp(self):
self.worker = AsyncD2HCopyWorker(torch.cuda)
def tearDown(self):
# Idempotent: a no-op if a test already shut the worker down.
self.worker.shutdown()
def _run_one(self, numel: int):
src = torch.randn(numel, device="cuda")
expected = src.detach().to("cpu") # synchronous reference copy
out = {}
def copy_fn():
out["cpu"] = src.to("cpu", non_blocking=True)
# submit() records readiness on the current stream (where src was
# produced) before handing the copy to the worker.
done = self.worker.submit(copy_fn)
self.assertTrue(done._done.wait(timeout=30), "worker did not signal completion")
done.synchronize() # drop-in for copy_done; must not raise on success
self.assertIn("cpu", out)
torch.testing.assert_close(out["cpu"], expected)
def test_single_readback_matches_synchronous(self):
self._run_one(4) # tiny, like next_token_ids at small batch
self._run_one(151936) # vocab-sized, like a logprob row
def test_copy_waits_for_the_producing_stream(self):
# Guards submit()'s src_ready.record(): with the fill still queued behind
# slow work, a worker that skipped the event would read zeros.
side_stream = torch.cuda.Stream()
src = torch.zeros(4096, device="cuda")
out = {}
def copy_fn():
out["cpu"] = src.to("cpu", non_blocking=True)
with torch.cuda.stream(side_stream):
torch.cuda._sleep(200_000_000) # ~100ms at 2GHz; outlives the handoff
src.fill_(7)
done = self.worker.submit(copy_fn)
self.assertTrue(done._done.wait(timeout=30), "worker did not signal completion")
done.synchronize()
torch.testing.assert_close(out["cpu"], torch.full((4096,), 7.0))
def test_shutdown_is_idempotent_and_joins(self):
self.worker.shutdown()
self.worker.shutdown() # second call must be a safe no-op
self.assertFalse(self.worker._thread.is_alive())
def test_copy_fn_exception_reports_error(self):
# A failing copy must not hang the scheduler, but synchronize() must
# re-raise so the consumer aborts instead of reading invalid CPU tensors.
def boom():
raise RuntimeError("injected copy failure")
done = self.worker.submit(boom)
self.assertTrue(
done._done.wait(timeout=30), "handle must be signaled even on failure"
)
self.assertIsNotNone(done.error, "failed copy must record its error")
with self.assertRaises(RuntimeError):
done.synchronize()
class TestConfidentialComputeDetection(CustomTestCase):
"""CC detection env override. Does not require CUDA (override short-circuits)."""
def test_env_override_true(self):
is_confidential_compute.cache_clear()
with envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(True):
is_confidential_compute.cache_clear()
self.assertTrue(is_confidential_compute())
is_confidential_compute.cache_clear()
def test_env_override_false(self):
is_confidential_compute.cache_clear()
with envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(False):
is_confidential_compute.cache_clear()
self.assertFalse(is_confidential_compute())
is_confidential_compute.cache_clear()
if __name__ == "__main__":
unittest.main()