perf: fix overlap scheduling for NVIDIA Confidential Computing(CC) on Blackwell (#36810)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
elvischenv
2026-10-15 12:09:09 -07:00
committed by GitHub
co-authored by Claude Opus 5 Po-Han Huang Po-Han Huang
parent 02a43eeef6
commit 87211088c9
8 changed files with 439 additions and 5 deletions
+2
View File
@@ -664,6 +664,8 @@ class Envs:
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
# Opt-in: keep receiving prefill requests while forward results are pending.
SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False)
# Override NVML Confidential Computing autodetection. None = autodetect.
SGLANG_FORCE_CONFIDENTIAL_COMPUTE = EnvBool(None)
# ===================================================================
# Radix and sparse KV caches
@@ -0,0 +1,106 @@
from __future__ import annotations
import logging
import queue
import threading
from typing import Callable, Optional
logger = logging.getLogger(__name__)
class HostCopyDone:
"""Drop-in for a ``copy_done`` CUDA event, backed by a worker thread.
Carries the copy's exception as well, so ``synchronize()`` can abort the
consumer instead of letting it read invalid host tensors.
"""
def __init__(self):
self._done = threading.Event()
self.error: Optional[BaseException] = None
def record(self, *args, **kwargs) -> None:
# No-op for parity with torch.cuda.Event; set_done is the real signal.
pass
def set_done(self, error: Optional[BaseException] = None) -> None:
self.error = error
self._done.set()
def synchronize(self) -> None:
"""Block until the copy completes; re-raise if it failed."""
self._done.wait()
if self.error is not None:
raise RuntimeError(
"Async device->host copy failed; destination tensors are invalid"
) from self.error
def query(self) -> bool:
"""True once the copy has completed (successfully or not)."""
return self._done.is_set()
class AsyncD2HCopyWorker:
"""Runs blocking device->host copies on a dedicated daemon thread.
Under NVIDIA Confidential Computing a D2H ``cudaMemcpyAsync`` is forced
synchronous and blocks at issue, stalling the submitting thread; this moves
the copy off it so the overlap scheduler keeps launching the next step.
``d2h_copy_stream`` is private on purpose: on a shared stream the blanket
``synchronize()`` below would also block on whatever the caller queued after
submitting, re-coupling the caller to the copy.
"""
def __init__(self, device_module):
self.device_module = device_module
# Private stream, created and owned here so nothing outside this worker
# can enqueue onto it (see class docstring). Created on the caller's
# thread, so it lands on the current device.
self.d2h_copy_stream = device_module.Stream()
self._device_index = device_module.current_device()
self._queue: queue.Queue = queue.Queue()
self._thread = threading.Thread(
target=self._loop, name="sglang-d2h-copy-worker", daemon=True
)
self._thread.start()
def submit(self, copy_fn: Callable[[], None]) -> HostCopyDone:
"""Record readiness on the CURRENT stream and enqueue the copy.
Must be called with the stream that produced the copy sources current.
"""
src_ready = self.device_module.Event()
src_ready.record()
done = HostCopyDone()
self._queue.put((src_ready, copy_fn, done))
return done
def _loop(self):
self.device_module.set_device(self._device_index)
while True:
item = self._queue.get()
if item is None:
return
src_ready, copy_fn, done = item
error = None
try:
# Event-sync, not a stream wait: a stream wait would re-expose
# the caller's thread to the blocking copy.
src_ready.synchronize()
with self.device_module.stream(self.d2h_copy_stream):
copy_fn()
self.d2h_copy_stream.synchronize()
except Exception as e:
logger.exception("AsyncD2HCopyWorker copy failed")
error = e
finally:
# Signal on failure too, so the caller never hangs.
done.set_done(error=error)
def shutdown(self, timeout: float = 2.0):
"""Signal the worker to stop and join it (best-effort, bounded wait)."""
if not self._thread.is_alive():
return
self._queue.put(None)
self._thread.join(timeout=timeout)
+37 -3
View File
@@ -358,7 +358,7 @@ from sglang.srt.utils import (
suppress_other_loggers,
triton_load_watch,
)
from sglang.srt.utils.common import is_npu
from sglang.srt.utils.common import is_confidential_compute, is_npu
from sglang.srt.utils.hf_transformers_utils import (
get_processor,
get_tokenizer,
@@ -1648,6 +1648,11 @@ class Scheduler(
self.draft_worker.get_confidence_budget_prepare()
)
# If enable async D2H copy when overlap is enabled
# Currently used by NVIDIA Confidential Computing (CC) only
self.enable_async_d2h_copy = False
self.async_d2h_worker = None
if use_mlx():
# MLX uses its own overlap loop and does not create CUDA streams,
# but the normal non-overlap scheduler path still relays decode
@@ -1668,6 +1673,19 @@ class Scheduler(
if not self.enable_overlap:
return
# Under NVIDIA Confidential Computing (CC) the per-step D2H result readback is
# forced synchronous and blocks the scheduler thread at issue, serializing the
# overlap pipeline. Offload it to a worker thread (AsyncD2HCopyWorker).
self.enable_async_d2h_copy = is_confidential_compute()
if self.enable_async_d2h_copy:
from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker
self.async_d2h_worker = AsyncD2HCopyWorker(self.device_module)
logger.info(
"NVIDIA Confidential Computing (CC) detected: "
"using async D2H copy worker to preserve overlap scheduling."
)
self.batch_record_buf = [None] * 2
self.batch_record_ct = 0
@@ -4632,10 +4650,17 @@ class Scheduler(
# gated by copy_done, so nothing on forward_stream waits.
self.copy_stream.wait_stream(self.forward_stream)
with self.copy_stream_ctx:
batch_result.copy_to_cpu(
copy_fn = partial(
batch_result.copy_to_cpu,
return_logprob=batch.return_logprob,
return_hidden_states=batch.return_hidden_states,
)
if self.enable_async_d2h_copy:
batch_result.copy_done = (
self.async_d2h_worker.submit(copy_fn)
)
else:
copy_fn()
else:
batch_result.future_indices = future_indices
@@ -4931,10 +4956,15 @@ class Scheduler(
# with subsequent forward computation.
self.copy_stream.wait_stream(self.forward_stream)
with self.copy_stream_ctx:
batch_result.copy_to_cpu(
copy_fn = partial(
batch_result.copy_to_cpu,
return_logprob=cur_batch.return_logprob,
return_hidden_states=cur_batch.return_hidden_states,
)
if self.enable_async_d2h_copy:
batch_result.copy_done = self.async_d2h_worker.submit(copy_fn)
else:
copy_fn()
# Release the closure and large GPU tensors that are no longer needed.
# The delay_sample_func closure captures forward_batch (which holds
@@ -6191,6 +6221,10 @@ def run_scheduler_process(
# FPM has a background ZMQ publisher thread that needs explicit
# teardown to flush queued metrics and close the socket cleanly.
scheduler.metrics_reporter._shutdown_fpm()
# Stop the async D2H copy worker thread if any.
if scheduler.async_d2h_worker is not None:
scheduler.async_d2h_worker.shutdown()
scheduler.async_d2h_worker = None
# Graceful path only: on the exception path the GPU may be wedged
# and the synchronize() in destroy() could itself hang.
if scheduler.gracefully_exit:
+3 -2
View File
@@ -24,6 +24,7 @@ from sglang.srt.state_capturer.base import TopkCaptureOutput
from sglang.srt.utils.common import async_d2h as _async_d2h
if TYPE_CHECKING:
from sglang.srt.managers.async_d2h_copy_worker import HostCopyDone
from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
@@ -73,7 +74,7 @@ class GenerationBatchResult:
extend_logprob_start_len_per_req: Optional[List[int]] = None
# For overlap scheduling
copy_done: Optional[torch.cuda.Event] = None
copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None
delay_sample_func: Optional[callable] = None
future_indices: Optional[torch.Tensor] = None
speculative_num_draft_tokens: Optional[int] = None
@@ -390,7 +391,7 @@ class EmbeddingBatchResult:
embeddings: torch.Tensor
pooled_hidden_states: Optional[torch.Tensor] = None
copy_done: Optional[torch.cuda.Event] = None
copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None
can_run_cuda_graph: bool = False
@torch.profiler.record_function("copy_embedding_to_cpu")
+54
View File
@@ -51,6 +51,7 @@ import warnings
from array import array
from collections import OrderedDict, defaultdict
from contextlib import contextmanager
from ctypes import byref
from dataclasses import dataclass
from decimal import Decimal
from functools import lru_cache, partial
@@ -5134,6 +5135,59 @@ def get_or_create_event_loop():
return loop
@lru_cache(maxsize=1)
def is_confidential_compute() -> bool:
"""Whether the GPU is running in NVIDIA Confidential Computing (CC) mode.
Detected once via NVML and cached.
Overridable with ``SGLANG_FORCE_CONFIDENTIAL_COMPUTE``.
"""
forced = envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.get()
if forced is not None:
return forced
if not is_cuda():
return False
try:
import pynvml
except ImportError:
logger.warning("pynvml not available; assuming Confidential Computing is off")
return False
cc_enabled = False
try:
pynvml.nvmlInit()
# Hopper and newer supports a more nuanced query of confidential
# compute settings
cc_settings = pynvml.c_nvmlSystemConfComputeSettings_v1_t()
ret = pynvml.nvmlSystemGetConfComputeSettings(byref(cc_settings))
pynvml._nvmlCheckReturn(ret)
# PPCIE implies CC
cc_enabled = (
cc_settings.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED
or cc_settings.multiGpuMode == pynvml.NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE
)
except pynvml.NVMLError_NotSupported:
# Simple query for older GPUs
try:
cc_state = pynvml.nvmlSystemGetConfComputeState()
cc_enabled = cc_state.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED
except pynvml.NVMLError as error:
logger.warning(f"Error querying CC state: {error!s}")
except pynvml.NVMLError as error:
logger.warning(f"Error querying CC state: {error!s}")
finally:
# Shutdown
try:
pynvml.nvmlShutdown()
except pynvml.NVMLError:
# Ignore shutdown errors
pass
return cc_enabled
def init_cublas():
"""We need to run a small matmul to init cublas. Otherwise, it will raise some errors later."""
dtype = torch.float16