perf: fix overlap scheduling for NVIDIA Confidential Computing(CC) on Blackwell (#36810)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
Po-Han Huang
Po-Han Huang
parent
02a43eeef6
commit
87211088c9
@@ -664,6 +664,8 @@ class Envs:
|
||||
SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False)
|
||||
# Opt-in: keep receiving prefill requests while forward results are pending.
|
||||
SGLANG_ENABLE_DISAGG_PREFILL_CONTINUOUS_INPUT_POLLING = EnvBool(False)
|
||||
# Override NVML Confidential Computing autodetection. None = autodetect.
|
||||
SGLANG_FORCE_CONFIDENTIAL_COMPUTE = EnvBool(None)
|
||||
|
||||
# ===================================================================
|
||||
# Radix and sparse KV caches
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
from typing import Callable, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HostCopyDone:
|
||||
"""Drop-in for a ``copy_done`` CUDA event, backed by a worker thread.
|
||||
|
||||
Carries the copy's exception as well, so ``synchronize()`` can abort the
|
||||
consumer instead of letting it read invalid host tensors.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._done = threading.Event()
|
||||
self.error: Optional[BaseException] = None
|
||||
|
||||
def record(self, *args, **kwargs) -> None:
|
||||
# No-op for parity with torch.cuda.Event; set_done is the real signal.
|
||||
pass
|
||||
|
||||
def set_done(self, error: Optional[BaseException] = None) -> None:
|
||||
self.error = error
|
||||
self._done.set()
|
||||
|
||||
def synchronize(self) -> None:
|
||||
"""Block until the copy completes; re-raise if it failed."""
|
||||
self._done.wait()
|
||||
if self.error is not None:
|
||||
raise RuntimeError(
|
||||
"Async device->host copy failed; destination tensors are invalid"
|
||||
) from self.error
|
||||
|
||||
def query(self) -> bool:
|
||||
"""True once the copy has completed (successfully or not)."""
|
||||
return self._done.is_set()
|
||||
|
||||
|
||||
class AsyncD2HCopyWorker:
|
||||
"""Runs blocking device->host copies on a dedicated daemon thread.
|
||||
|
||||
Under NVIDIA Confidential Computing a D2H ``cudaMemcpyAsync`` is forced
|
||||
synchronous and blocks at issue, stalling the submitting thread; this moves
|
||||
the copy off it so the overlap scheduler keeps launching the next step.
|
||||
|
||||
``d2h_copy_stream`` is private on purpose: on a shared stream the blanket
|
||||
``synchronize()`` below would also block on whatever the caller queued after
|
||||
submitting, re-coupling the caller to the copy.
|
||||
"""
|
||||
|
||||
def __init__(self, device_module):
|
||||
self.device_module = device_module
|
||||
# Private stream, created and owned here so nothing outside this worker
|
||||
# can enqueue onto it (see class docstring). Created on the caller's
|
||||
# thread, so it lands on the current device.
|
||||
self.d2h_copy_stream = device_module.Stream()
|
||||
self._device_index = device_module.current_device()
|
||||
self._queue: queue.Queue = queue.Queue()
|
||||
self._thread = threading.Thread(
|
||||
target=self._loop, name="sglang-d2h-copy-worker", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def submit(self, copy_fn: Callable[[], None]) -> HostCopyDone:
|
||||
"""Record readiness on the CURRENT stream and enqueue the copy.
|
||||
|
||||
Must be called with the stream that produced the copy sources current.
|
||||
"""
|
||||
src_ready = self.device_module.Event()
|
||||
src_ready.record()
|
||||
done = HostCopyDone()
|
||||
self._queue.put((src_ready, copy_fn, done))
|
||||
return done
|
||||
|
||||
def _loop(self):
|
||||
self.device_module.set_device(self._device_index)
|
||||
while True:
|
||||
item = self._queue.get()
|
||||
if item is None:
|
||||
return
|
||||
src_ready, copy_fn, done = item
|
||||
error = None
|
||||
try:
|
||||
# Event-sync, not a stream wait: a stream wait would re-expose
|
||||
# the caller's thread to the blocking copy.
|
||||
src_ready.synchronize()
|
||||
with self.device_module.stream(self.d2h_copy_stream):
|
||||
copy_fn()
|
||||
self.d2h_copy_stream.synchronize()
|
||||
except Exception as e:
|
||||
logger.exception("AsyncD2HCopyWorker copy failed")
|
||||
error = e
|
||||
finally:
|
||||
# Signal on failure too, so the caller never hangs.
|
||||
done.set_done(error=error)
|
||||
|
||||
def shutdown(self, timeout: float = 2.0):
|
||||
"""Signal the worker to stop and join it (best-effort, bounded wait)."""
|
||||
if not self._thread.is_alive():
|
||||
return
|
||||
self._queue.put(None)
|
||||
self._thread.join(timeout=timeout)
|
||||
@@ -358,7 +358,7 @@ from sglang.srt.utils import (
|
||||
suppress_other_loggers,
|
||||
triton_load_watch,
|
||||
)
|
||||
from sglang.srt.utils.common import is_npu
|
||||
from sglang.srt.utils.common import is_confidential_compute, is_npu
|
||||
from sglang.srt.utils.hf_transformers_utils import (
|
||||
get_processor,
|
||||
get_tokenizer,
|
||||
@@ -1648,6 +1648,11 @@ class Scheduler(
|
||||
self.draft_worker.get_confidence_budget_prepare()
|
||||
)
|
||||
|
||||
# If enable async D2H copy when overlap is enabled
|
||||
# Currently used by NVIDIA Confidential Computing (CC) only
|
||||
self.enable_async_d2h_copy = False
|
||||
self.async_d2h_worker = None
|
||||
|
||||
if use_mlx():
|
||||
# MLX uses its own overlap loop and does not create CUDA streams,
|
||||
# but the normal non-overlap scheduler path still relays decode
|
||||
@@ -1668,6 +1673,19 @@ class Scheduler(
|
||||
if not self.enable_overlap:
|
||||
return
|
||||
|
||||
# Under NVIDIA Confidential Computing (CC) the per-step D2H result readback is
|
||||
# forced synchronous and blocks the scheduler thread at issue, serializing the
|
||||
# overlap pipeline. Offload it to a worker thread (AsyncD2HCopyWorker).
|
||||
self.enable_async_d2h_copy = is_confidential_compute()
|
||||
if self.enable_async_d2h_copy:
|
||||
from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker
|
||||
|
||||
self.async_d2h_worker = AsyncD2HCopyWorker(self.device_module)
|
||||
logger.info(
|
||||
"NVIDIA Confidential Computing (CC) detected: "
|
||||
"using async D2H copy worker to preserve overlap scheduling."
|
||||
)
|
||||
|
||||
self.batch_record_buf = [None] * 2
|
||||
self.batch_record_ct = 0
|
||||
|
||||
@@ -4632,10 +4650,17 @@ class Scheduler(
|
||||
# gated by copy_done, so nothing on forward_stream waits.
|
||||
self.copy_stream.wait_stream(self.forward_stream)
|
||||
with self.copy_stream_ctx:
|
||||
batch_result.copy_to_cpu(
|
||||
copy_fn = partial(
|
||||
batch_result.copy_to_cpu,
|
||||
return_logprob=batch.return_logprob,
|
||||
return_hidden_states=batch.return_hidden_states,
|
||||
)
|
||||
if self.enable_async_d2h_copy:
|
||||
batch_result.copy_done = (
|
||||
self.async_d2h_worker.submit(copy_fn)
|
||||
)
|
||||
else:
|
||||
copy_fn()
|
||||
else:
|
||||
batch_result.future_indices = future_indices
|
||||
|
||||
@@ -4931,10 +4956,15 @@ class Scheduler(
|
||||
# with subsequent forward computation.
|
||||
self.copy_stream.wait_stream(self.forward_stream)
|
||||
with self.copy_stream_ctx:
|
||||
batch_result.copy_to_cpu(
|
||||
copy_fn = partial(
|
||||
batch_result.copy_to_cpu,
|
||||
return_logprob=cur_batch.return_logprob,
|
||||
return_hidden_states=cur_batch.return_hidden_states,
|
||||
)
|
||||
if self.enable_async_d2h_copy:
|
||||
batch_result.copy_done = self.async_d2h_worker.submit(copy_fn)
|
||||
else:
|
||||
copy_fn()
|
||||
|
||||
# Release the closure and large GPU tensors that are no longer needed.
|
||||
# The delay_sample_func closure captures forward_batch (which holds
|
||||
@@ -6191,6 +6221,10 @@ def run_scheduler_process(
|
||||
# FPM has a background ZMQ publisher thread that needs explicit
|
||||
# teardown to flush queued metrics and close the socket cleanly.
|
||||
scheduler.metrics_reporter._shutdown_fpm()
|
||||
# Stop the async D2H copy worker thread if any.
|
||||
if scheduler.async_d2h_worker is not None:
|
||||
scheduler.async_d2h_worker.shutdown()
|
||||
scheduler.async_d2h_worker = None
|
||||
# Graceful path only: on the exception path the GPU may be wedged
|
||||
# and the synchronize() in destroy() could itself hang.
|
||||
if scheduler.gracefully_exit:
|
||||
|
||||
@@ -24,6 +24,7 @@ from sglang.srt.state_capturer.base import TopkCaptureOutput
|
||||
from sglang.srt.utils.common import async_d2h as _async_d2h
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.async_d2h_copy_worker import HostCopyDone
|
||||
from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
|
||||
@@ -73,7 +74,7 @@ class GenerationBatchResult:
|
||||
extend_logprob_start_len_per_req: Optional[List[int]] = None
|
||||
|
||||
# For overlap scheduling
|
||||
copy_done: Optional[torch.cuda.Event] = None
|
||||
copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None
|
||||
delay_sample_func: Optional[callable] = None
|
||||
future_indices: Optional[torch.Tensor] = None
|
||||
speculative_num_draft_tokens: Optional[int] = None
|
||||
@@ -390,7 +391,7 @@ class EmbeddingBatchResult:
|
||||
|
||||
embeddings: torch.Tensor
|
||||
pooled_hidden_states: Optional[torch.Tensor] = None
|
||||
copy_done: Optional[torch.cuda.Event] = None
|
||||
copy_done: Optional[Union[torch.cuda.Event, HostCopyDone]] = None
|
||||
can_run_cuda_graph: bool = False
|
||||
|
||||
@torch.profiler.record_function("copy_embedding_to_cpu")
|
||||
|
||||
@@ -51,6 +51,7 @@ import warnings
|
||||
from array import array
|
||||
from collections import OrderedDict, defaultdict
|
||||
from contextlib import contextmanager
|
||||
from ctypes import byref
|
||||
from dataclasses import dataclass
|
||||
from decimal import Decimal
|
||||
from functools import lru_cache, partial
|
||||
@@ -5134,6 +5135,59 @@ def get_or_create_event_loop():
|
||||
return loop
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def is_confidential_compute() -> bool:
|
||||
"""Whether the GPU is running in NVIDIA Confidential Computing (CC) mode.
|
||||
|
||||
Detected once via NVML and cached.
|
||||
Overridable with ``SGLANG_FORCE_CONFIDENTIAL_COMPUTE``.
|
||||
"""
|
||||
forced = envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.get()
|
||||
if forced is not None:
|
||||
return forced
|
||||
if not is_cuda():
|
||||
return False
|
||||
|
||||
try:
|
||||
import pynvml
|
||||
except ImportError:
|
||||
logger.warning("pynvml not available; assuming Confidential Computing is off")
|
||||
return False
|
||||
|
||||
cc_enabled = False
|
||||
try:
|
||||
pynvml.nvmlInit()
|
||||
|
||||
# Hopper and newer supports a more nuanced query of confidential
|
||||
# compute settings
|
||||
cc_settings = pynvml.c_nvmlSystemConfComputeSettings_v1_t()
|
||||
ret = pynvml.nvmlSystemGetConfComputeSettings(byref(cc_settings))
|
||||
pynvml._nvmlCheckReturn(ret)
|
||||
# PPCIE implies CC
|
||||
cc_enabled = (
|
||||
cc_settings.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED
|
||||
or cc_settings.multiGpuMode == pynvml.NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE
|
||||
)
|
||||
except pynvml.NVMLError_NotSupported:
|
||||
# Simple query for older GPUs
|
||||
try:
|
||||
cc_state = pynvml.nvmlSystemGetConfComputeState()
|
||||
cc_enabled = cc_state.ccFeature == pynvml.NVML_CC_SYSTEM_FEATURE_ENABLED
|
||||
except pynvml.NVMLError as error:
|
||||
logger.warning(f"Error querying CC state: {error!s}")
|
||||
except pynvml.NVMLError as error:
|
||||
logger.warning(f"Error querying CC state: {error!s}")
|
||||
finally:
|
||||
# Shutdown
|
||||
try:
|
||||
pynvml.nvmlShutdown()
|
||||
except pynvml.NVMLError:
|
||||
# Ignore shutdown errors
|
||||
pass
|
||||
|
||||
return cc_enabled
|
||||
|
||||
|
||||
def init_cublas():
|
||||
"""We need to run a small matmul to init cublas. Otherwise, it will raise some errors later."""
|
||||
dtype = torch.float16
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Server-level coverage for the async D2H result readback (AsyncD2HCopyWorker).
|
||||
|
||||
Under Confidential Computing the overlap scheduler hands its per-step readback
|
||||
to a worker thread and stores a HostCopyDone in `copy_done`. CI GPUs never run
|
||||
CC, so without the force override nothing exercises that path end to end:
|
||||
`run_batch`, `launch_batch_sample_if_needed`, and every `copy_done.synchronize()`
|
||||
consumer keep taking the inline branch.
|
||||
|
||||
Both scheduler submit sites are covered: the inline-sample path, and the
|
||||
delay-sample path via SGLANG_ENABLE_DELAY_SAMPLE.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from contextlib import ExitStack
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=180, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
MAX_NEW_TOKENS = 32
|
||||
REPEATS = 2
|
||||
# Distinct answers make a readback that returns another row's tokens visible.
|
||||
# Identical prompts cannot: every row would still carry a plausible answer.
|
||||
PROMPTS = (
|
||||
("The capital of France is", "Paris"),
|
||||
("The capital of Japan is", "Tokyo"),
|
||||
("The capital of Italy is", "Rome"),
|
||||
("The capital of Germany is", "Berlin"),
|
||||
("The capital of Spain is", "Madrid"),
|
||||
("The capital of Russia is", "Moscow"),
|
||||
("The capital of Egypt is", "Cairo"),
|
||||
("The capital of Greece is", "Athens"),
|
||||
)
|
||||
|
||||
|
||||
class TestAsyncD2HCopyWorkerServer(CustomTestCase):
|
||||
"""python -m unittest test_async_d2h_copy_worker_server.TestAsyncD2HCopyWorkerServer"""
|
||||
|
||||
process = None
|
||||
# (EnvField, value) pairs that subclasses add on top of the force flag.
|
||||
extra_env_overrides: tuple = ()
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(True))
|
||||
for field, value in cls.extra_env_overrides:
|
||||
stack.enter_context(field.override(value))
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--mem-fraction-static", "0.7"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
if cls.process is not None:
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def _generate(self, prompt: str) -> dict:
|
||||
payload = {
|
||||
"text": prompt,
|
||||
"sampling_params": {
|
||||
"max_new_tokens": MAX_NEW_TOKENS,
|
||||
"temperature": 0.0,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"return_logprob": True,
|
||||
"logprob_start_len": -1,
|
||||
}
|
||||
r = requests.post(f"{self.base_url}/generate", json=payload, timeout=600)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def test_concurrent_decode_readback(self):
|
||||
# Not asserting that repeats of one prompt agree: batching is not
|
||||
# batch-invariant, so two coherent completions can legitimately differ.
|
||||
cases = list(PROMPTS) * REPEATS
|
||||
with ThreadPoolExecutor(max_workers=len(cases)) as pool:
|
||||
futures = {
|
||||
pool.submit(self._generate, prompt): (prompt, answer)
|
||||
for prompt, answer in cases
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
prompt, answer = futures[future]
|
||||
result = future.result()
|
||||
self.assertIn(answer, result["text"], f"{prompt!r} -> wrong answer")
|
||||
|
||||
meta = result["meta_info"]
|
||||
self.assertEqual(meta["completion_tokens"], MAX_NEW_TOKENS)
|
||||
# The logprob readback rides the same copy as next_token_ids.
|
||||
self.assertEqual(
|
||||
len(meta["output_token_logprobs"]), meta["completion_tokens"]
|
||||
)
|
||||
|
||||
|
||||
class TestAsyncD2HCopyWorkerServerDelaySample(TestAsyncD2HCopyWorkerServer):
|
||||
"""Covers the launch_batch_sample_if_needed submit site."""
|
||||
|
||||
extra_env_overrides = ((envs.SGLANG_ENABLE_DELAY_SAMPLE, True),)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -169,6 +169,7 @@ class _Scheduler(SchedulerDisaggregationPrefillMixin):
|
||||
self.gracefully_exit = False
|
||||
self.enable_staging = False
|
||||
self.enable_overlap = True
|
||||
self.enable_async_d2h_copy = False
|
||||
self.enable_overlap_mlx = False
|
||||
self.disagg_prefill_bootstrap_queue = SimpleNamespace(
|
||||
pop_bootstrapped=lambda: []
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Unit tests for the async device->host copy worker (AsyncD2HCopyWorker).
|
||||
|
||||
These validate that ``AsyncD2HCopyWorker`` performs the device->host copy
|
||||
correctly off the calling thread, and that the CC detection env override works.
|
||||
|
||||
They need a GPU but NO model and NO actual confidential-compute hardware: the
|
||||
worker logic is identical regardless of CC (CC only changes whether the
|
||||
scheduler routes the readback through the worker). To exercise the worker on an
|
||||
ordinary GPU, run inside the sglang container:
|
||||
|
||||
python -m pytest test/registered/unit/managers/test_async_d2h_copy_worker.py -v
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.async_d2h_copy_worker import AsyncD2HCopyWorker
|
||||
from sglang.srt.utils.common import is_confidential_compute
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "AsyncD2HCopyWorker requires CUDA")
|
||||
class TestAsyncD2HCopyWorker(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.worker = AsyncD2HCopyWorker(torch.cuda)
|
||||
|
||||
def tearDown(self):
|
||||
# Idempotent: a no-op if a test already shut the worker down.
|
||||
self.worker.shutdown()
|
||||
|
||||
def _run_one(self, numel: int):
|
||||
src = torch.randn(numel, device="cuda")
|
||||
expected = src.detach().to("cpu") # synchronous reference copy
|
||||
|
||||
out = {}
|
||||
|
||||
def copy_fn():
|
||||
out["cpu"] = src.to("cpu", non_blocking=True)
|
||||
|
||||
# submit() records readiness on the current stream (where src was
|
||||
# produced) before handing the copy to the worker.
|
||||
done = self.worker.submit(copy_fn)
|
||||
|
||||
self.assertTrue(done._done.wait(timeout=30), "worker did not signal completion")
|
||||
done.synchronize() # drop-in for copy_done; must not raise on success
|
||||
self.assertIn("cpu", out)
|
||||
torch.testing.assert_close(out["cpu"], expected)
|
||||
|
||||
def test_single_readback_matches_synchronous(self):
|
||||
self._run_one(4) # tiny, like next_token_ids at small batch
|
||||
self._run_one(151936) # vocab-sized, like a logprob row
|
||||
|
||||
def test_copy_waits_for_the_producing_stream(self):
|
||||
# Guards submit()'s src_ready.record(): with the fill still queued behind
|
||||
# slow work, a worker that skipped the event would read zeros.
|
||||
side_stream = torch.cuda.Stream()
|
||||
src = torch.zeros(4096, device="cuda")
|
||||
|
||||
out = {}
|
||||
|
||||
def copy_fn():
|
||||
out["cpu"] = src.to("cpu", non_blocking=True)
|
||||
|
||||
with torch.cuda.stream(side_stream):
|
||||
torch.cuda._sleep(200_000_000) # ~100ms at 2GHz; outlives the handoff
|
||||
src.fill_(7)
|
||||
done = self.worker.submit(copy_fn)
|
||||
|
||||
self.assertTrue(done._done.wait(timeout=30), "worker did not signal completion")
|
||||
done.synchronize()
|
||||
torch.testing.assert_close(out["cpu"], torch.full((4096,), 7.0))
|
||||
|
||||
def test_shutdown_is_idempotent_and_joins(self):
|
||||
self.worker.shutdown()
|
||||
self.worker.shutdown() # second call must be a safe no-op
|
||||
self.assertFalse(self.worker._thread.is_alive())
|
||||
|
||||
def test_copy_fn_exception_reports_error(self):
|
||||
# A failing copy must not hang the scheduler, but synchronize() must
|
||||
# re-raise so the consumer aborts instead of reading invalid CPU tensors.
|
||||
def boom():
|
||||
raise RuntimeError("injected copy failure")
|
||||
|
||||
done = self.worker.submit(boom)
|
||||
self.assertTrue(
|
||||
done._done.wait(timeout=30), "handle must be signaled even on failure"
|
||||
)
|
||||
self.assertIsNotNone(done.error, "failed copy must record its error")
|
||||
with self.assertRaises(RuntimeError):
|
||||
done.synchronize()
|
||||
|
||||
|
||||
class TestConfidentialComputeDetection(CustomTestCase):
|
||||
"""CC detection env override. Does not require CUDA (override short-circuits)."""
|
||||
|
||||
def test_env_override_true(self):
|
||||
is_confidential_compute.cache_clear()
|
||||
with envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(True):
|
||||
is_confidential_compute.cache_clear()
|
||||
self.assertTrue(is_confidential_compute())
|
||||
is_confidential_compute.cache_clear()
|
||||
|
||||
def test_env_override_false(self):
|
||||
is_confidential_compute.cache_clear()
|
||||
with envs.SGLANG_FORCE_CONFIDENTIAL_COMPUTE.override(False):
|
||||
is_confidential_compute.cache_clear()
|
||||
self.assertFalse(is_confidential_compute())
|
||||
is_confidential_compute.cache_clear()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user