perf: fix overlap scheduling for NVIDIA Confidential Computing(CC) on Blackwell (#36810)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Po-Han Huang
Po-Han Huang
parent
02a43eeef6
commit
87211088c9
@@ -664,6 +664,8 @@ class Envs:
|
||||
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
|
||||
# Opt-in: keep receiving prefill requests while forward results are pending.
|
||||
SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False)
|
||||
# Override NVML Confidential Computing autodetection. None = autodetect.
|
||||
SGLANG_FORCE_CONFIDENTIAL_COMPUTE = EnvBool(None)
|
||||
|
||||
# ===================================================================
|
||||
# Radix and sparse KV caches
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
from typing import Callable, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HostCopyDone:
|
||||
"""Drop-in for a ``copy_done`` CUDA event, backed by a worker thread.
|
||||
|
||||
Carries the copy's exception as well, so ``synchronize()`` can abort the
|
||||
consumer instead of letting it read invalid host tensors.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._done = threading.Event()
|
||||
self.error: Optional[BaseException] = None
|
||||
|
||||
def record(self, *args, **kwargs) -> None:
|
||||
# No-op for parity with torch.cuda.Event; set_done is the real signal.
|
||||
pass
|
||||
|
||||
def set_done(self, error: Optional[BaseException] = None) -> None:
|
||||
self.error = error
|
||||
self._done.set()
|
||||
|
||||
def synchronize(self) -> None:
|
||||
"""Block until the copy completes; re-raise if it failed."""
|
||||
self._done.wait()
|
||||
if self.error is not None:
|
||||
raise RuntimeError(
|
||||
"Async device->host copy failed; destination tensors are invalid"
|
||||
) from self.error
|
||||
|
||||
def query(self) -> bool:
|
||||
"""True once the copy has completed (successfully or not)."""
|
||||
return self._done.is_set()
|
||||
|
||||
|
||||
class AsyncD2HCopyWorker:
|
||||
"""Runs blocking device->host copies on a dedicated daemon thread.
|
||||
|
||||
Under NVIDIA Confidential Computing a D2H ``cudaMemcpyAsync`` is forced
|
||||
synchronous and blocks at issue, stalling the submitting thread; this moves
|
||||
the copy off it so the overlap scheduler keeps launching the next step.
|
||||
|
||||
``d2h_copy_stream`` is private on purpose: on a shared stream the blanket
|
||||
``synchronize()`` below would also block on whatever the caller queued after
|
||||
submitting, re-coupling the caller to the copy.
|
||||
"""
|
||||
|
||||
def __init__(self, device_module):
|
||||
self.device_module = device_module
|
||||
# Private stream, created and owned here so nothing outside this worker
|
||||
# can enqueue onto it (see class docstring). Created on the caller's
|
||||
# thread, so it lands on the current device.
|
||||
self.d2h_copy_stream = device_module.Stream()
|
||||
self._device_index = device_module.current_device()
|
||||
self._queue: queue.Queue = queue.Queue()
|
||||
self._thread = threading.Thread(
|
||||
target=self._loop, name="sglang-d2h-copy-worker", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def submit(self, copy_fn: Callable[[], None]) -> HostCopyDone:
|
||||
"""Record readiness on the CURRENT stream and enqueue the copy.
|
||||
|
||||
Must be called with the stream that produced the copy sources current.
|
||||
"""
|
||||
src_ready = self.device_module.Event()
|
||||
src_ready.record()
|
||||
done = HostCopyDone()
|
||||
self._queue.put((src_ready, copy_fn, done))
|
||||
return done
|
||||
|
||||
def _loop(self):
|
||||
self.device_module.set_device(self._device_index)
|
||||
while True:
|
||||
item = self._queue.get()
|
||||
if item is None:
|
||||
return
|
||||
src_ready, copy_fn, done = item
|
||||
error = None
|
||||
try:
|
||||
# Event-sync, not a stream wait: a stream wait would re-expose
|
||||
# the caller's thread to the blocking copy.
|
||||
src_ready.synchronize()
|
||||
with self.device_module.stream(self.d2h_copy_stream):
|
||||
copy_fn()
|
||||
self.d2h_copy_stream.synchronize()
|
||||
except Exception as e:
|
||||
logger.exception("AsyncD2HCopyWorker copy failed")
|
||||
error = e
|
||||
finally:
|
||||
# Signal on failure too, so the caller never hangs.
|
||||
done.set_done(error=error)
|
||||
|
||||
def shutdown(self, timeout: float = 2.0):
|
||||
"""Signal the worker to stop and join it (best-effort, bounded wait)."""
|
||||
if not self._thread.is_alive():
|
||||
return
|
||||
self._queue.put(None)
|
||||
self._thread.join(timeout=timeout)
|
||||
@@ -358,7 +358,7 @@ from sglang.srt.utils import (
|
||||
suppress_other_loggers,
|
||||
triton_load_watch,
|
||||
)
|
||||
from sglang.srt.utils.common import is_npu
|
||||
from sglang.srt.utils.common import is_confidential_compute, is_npu
|
||||
from sglang.srt.utils.hf_transformers_utils import (
|
||||
get_processor,
|
||||
get_tokenizer,
|
||||
@@ -1648,6 +1648,11 @@ class Scheduler(
|
||||
self.draft_worker.get_confidence_budget_prepare()
|
||||
)
|
||||
|
||||
# If enable async D2H copy when overlap is enabled
|
||||
# Currently used by NVIDIA Confidential Computing (CC) only
|
||||
self.enable_async_d2h_copy = False
|
||||
self.async_d2h_worker = None
|
||||
|
||||
if use_mlx():
|
||||
# MLX uses its own overlap loop and does not create CUDA streams,
|
||||
# but the normal non-overlap scheduler path still relays decode
|
||||
@@ -1668,6 +1673,19 @@ class Scheduler(
|
||||
if not self.enable_overlap:
|
||||
return
|
||||
|
||||
# Under NVIDIA Confidential Computing (CC) the per-step D2H result readback is
|
||||
# forced synchronous and blocks the scheduler thread at issue, serializing the
|
||||
# overlap pipeline. Offload it to a worker thread (AsyncD2HCopyWorker).
|
||||
self.enable_async_d2h_copy = is_confidential_compute()
|
||||
if self.enable_async_d2h_copy:
|
||||
from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker
|
||||
|
||||
self.async_d2h_worker = AsyncD2HCopyWorker(self.device_module)
|
||||
logger.info(
|
||||
"NVIDIA Confidential Computing (CC) detected: "
|
||||
"using async D2H copy worker to preserve overlap scheduling."
|
||||
)
|
||||
|
||||
self.batch_record_buf = [None] * 2
|
||||
self.batch_record_ct = 0
|
||||
|
||||
@@ -4632,10 +4650,17 @@ class Scheduler(
|
||||
# gated by copy_done, so nothing on forward_stream waits.
|
||||
self.copy_stream.wait_stream(self.forward_stream)
|
||||
with self.copy_stream_ctx:
|
||||
batch_result.copy_to_cpu(
|
||||
copy_fn = partial(
|
||||
batch_result.copy_to_cpu,
|
||||
return_logprob=batch.return_logprob,
|
||||
return_hidden_states=batch.return_hidden_states,
|
||||
)
|
||||
if self.enable_async_d2h_copy:
|
||||
batch_result.copy_done = (
|
||||
self.async_d2h_worker.submit(copy_fn)
|
||||
)
|
||||
else:
|
||||
copy_fn()
|
||||
else:
|
||||
batch_result.future_indices = future_indices
|
||||
|
||||
@@ -4931,10 +4956,15 @@ class Scheduler(
|
||||
# with subsequent forward computation.
|
||||
self.copy_stream.wait_stream(self.forward_stream)
|
||||
with self.copy_stream_ctx:
|
||||
batch_result.copy_to_cpu(
|
||||
copy_fn = partial(
|
||||
batch_result.copy_to_cpu,
|
||||
return_logprob=cur_batch.return_logprob,
|
||||
return_hidden_states=cur_batch.return_hidden_states,
|
||||
)
|
||||
if self.enable_async_d2h_copy:
|
||||
batch_result.copy_done = self.async_d2h_worker.submit(copy_fn)
|
||||
else:
|
||||
copy_fn()
|
||||
|
||||
# Release the closure and large GPU tensors that are no longer needed.
|
||||
# The delay_sample_func closure captures forward_batch (which holds
|
||||
@@ -6191,6 +6221,10 @@ def run_scheduler_process(
|
||||
# FPM has a background ZMQ publisher thread that needs explicit
|
||||
# teardown to flush queued metrics and close the socket cleanly.
|
||||
scheduler.metrics_reporter._shutdown_fpm()
|
||||
# Stop the async D2H copy worker thread if any.
|
||||
if scheduler.async_d2h_worker is not None:
|
||||
scheduler.async_d2h_worker.shutdown()
|
||||
scheduler.async_d2h_worker = None
|
||||
# Graceful path only: on the exception path the GPU may be wedged
|
||||
# and the synchronize() in destroy() could itself hang.
|
||||
if scheduler.gracefully_exit:
|
||||
|
||||
@@ -24,6 +24,7 @@ from sglang.srt.state_capturer.base import TopkCaptureOutput
|
||||
from sglang.srt.utils.common import async_d2h as _async_d2h
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.async_d2h_copy_worker import HostCopyDone
|
||||
from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
@@ -73,7 +74,7 @@ class GenerationBatchResult:
|
||||
extend_logprob_start_len_per_req: Optional[List[int]] = None
|
||||
|
||||
# For overlap scheduling
|
||||
copy_done: Optional[torch.cuda.Event] = None
|
||||
copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None
|
||||
delay_sample_func: Optional[callable] = None
|
||||
future_indices: Optional[torch.Tensor] = None
|
||||
speculative_num_draft_tokens: Optional[int] = None
|
||||
@@ -390,7 +391,7 @@ class EmbeddingBatchResult:
|
||||
|
||||
embeddings: torch.Tensor
|
||||
pooled_hidden_states: Optional[torch.Tensor] = None
|
||||
copy_done: Optional[torch.cuda.Event] = None
|
||||
copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None
|
||||
can_run_cuda_graph: bool = False
|
||||
|
||||
@torch.profiler.record_function("copy_embedding_to_cpu")
|
||||
|
||||
@@ -51,6 +51,7 @@ import warnings
|
||||
from array import array
|
||||
from collections import OrderedDict, defaultdict
|
||||
from contextlib import contextmanager
|
||||
from ctypes import byref
|
||||
from dataclasses import dataclass
|
||||
from decimal import Decimal
|
||||
from functools import lru_cache, partial
|
||||
@@ -5134,6 +5135,59 @@ def get_or_create_event_loop():
|
||||
return loop
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def is_confidential_compute() -> bool:
|
||||
"""Whether the GPU is running in NVIDIA Confidential Computing (CC) mode.
|
||||
|
||||
Detected once via NVML and cached.
|
||||
Overridable with ``SGLANG_FORCE_CONFIDENTIAL_COMPUTE``.
|
||||
"""
|
||||
forced = envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.get()
|
||||
if forced is not None:
|
||||
return forced
|
||||
if not is_cuda():
|
||||
return False
|
||||
|
||||
try:
|
||||
import pynvml
|
||||
except ImportError:
|
||||
logger.warning("pynvml not available; assuming Confidential Computing is off")
|
||||
return False
|
||||
|
||||
cc_enabled = False
|
||||
try:
|
||||
pynvml.nvmlInit()
|
||||
|
||||
# Hopper and newer supports a more nuanced query of confidential
|
||||
# compute settings
|
||||
cc_settings = pynvml.c_nvmlSystemConfComputeSettings_v1_t()
|
||||
ret = pynvml.nvmlSystemGetConfComputeSettings(byref(cc_settings))
|
||||
pynvml._nvmlCheckReturn(ret)
|
||||
# PPCIE implies CC
|
||||
cc_enabled = (
|
||||
cc_settings.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED
|
||||
or cc_settings.multiGpuMode == pynvml.NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE
|
||||
)
|
||||
except pynvml.NVMLError_NotSupported:
|
||||
# Simple query for older GPUs
|
||||
try:
|
||||
cc_state = pynvml.nvmlSystemGetConfComputeState()
|
||||
cc_enabled = cc_state.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED
|
||||
except pynvml.NVMLError as error:
|
||||
logger.warning(f"Error querying CC state: {error!s}")
|
||||
except pynvml.NVMLError as error:
|
||||
logger.warning(f"Error querying CC state: {error!s}")
|
||||
finally:
|
||||
# Shutdown
|
||||
try:
|
||||
pynvml.nvmlShutdown()
|
||||
except pynvml.NVMLError:
|
||||
# Ignore shutdown errors
|
||||
pass
|
||||
|
||||
return cc_enabled
|
||||
|
||||
|
||||
def init_cublas():
|
||||
"""We need to run a small matmul to init cublas. Otherwise, it will raise some errors later."""
|
||||
dtype = torch.float16
|
||||
|
||||
Reference in New Issue
Block a user