[Fix] cuda_graph: Wait for in-flight layer-wise cache loads before prefill graph replay (#44220)

This commit is contained in:
Shuwen Wang
2026-10-17 19:13:57 +08:00
committed by GitHub
parent 5fcb07da34
commit ae28306a74
5 changed files with 89 additions and 1 deletions
+3
View File
@@ -422,6 +422,9 @@ class Envs:
SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL = EnvBool(False)
# KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits)
SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False)
# HiCache: GPU sleep cycles before each layer's host-to-device load, so a forward
# that skips the per-layer load wait reads unloaded layers
SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES = EnvInt(0)
# ===================================================================
# CI reporting: per-model metrics jsonl for nightly XPU dashboard
@@ -7,6 +7,7 @@ from typing import Any, Callable, NamedTuple, Optional
import torch
from sglang.srt.environ import envs
from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains
from sglang.srt.utils import get_device_module
@@ -153,7 +154,10 @@ class L2TransferEngine:
transfers, self.host_to_device_stream, "host_to_device", start_event
) as (transfers, completion):
primary = transfers[0] if transfers else None
delay_cycles = envs.SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES.get()
for layer_id in range(transfer_layer_id_max):
if delay_cycles:
torch.cuda._sleep(delay_cycles)
for transfer in transfers:
local_layer_id = (
transfer.layer_mapper(layer_id)
@@ -2268,6 +2268,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
f"capture mode ({self.capture_hidden_mode.name})."
)
def _wait_for_layer_loads(self) -> None:
# Full replay skips the per-layer load waits in the pool accessors; breakable
# replay still runs them at its eager breaks.
if not self._is_full_backend:
return
counter = self.model_runner.token_to_kv_pool.layer_transfer_counter
if counter is not None and counter.consumer_index >= 0:
counter.wait_until(counter.num_layers - 1)
def execute(
self, forward_batch: ForwardBatch, **kwargs
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
@@ -2285,6 +2294,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
maybe_publish_prefill_shared_read_done(
self.model_runner, forward_batch, self.device_module
)
self._wait_for_layer_loads()
if self.enable_cp_bcg_capture:
assert self.max_context_size is None, (
@@ -52,6 +52,7 @@ import unittest
import requests
from sglang.srt.environ import envs
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kl_multiturn_utils import (
_extract_output_logprobs,
@@ -84,7 +85,7 @@ from sglang.test.test_utils import (
unified_radix_tree_server_env,
)
register_cuda_ci(est_time=3140, stage="extra-a", runner_config="1-gpu-large")
register_cuda_ci(est_time=3340, stage="extra-a", runner_config="1-gpu-large")
_MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling")
_MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test")
@@ -300,6 +301,7 @@ class TestUnifiedHybridHiCacheBitExact(CustomTestCase):
cls.model = _MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = _base_args() + [
"--enable-metrics",
"--enable-hierarchical-cache",
"--hicache-ratio",
"4",
@@ -367,6 +369,27 @@ class TestUnifiedHybridHiCacheBitExact(CustomTestCase):
)
class TestUnifiedHybridHiCacheSlowLoadBitExact(TestUnifiedHybridHiCacheBitExact):
"""A prefill CUDA graph must not replay over a host-to-device load that has not
landed; the delay makes the load lose that race every time."""
@classmethod
def setUpClass(cls):
with envs.SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES.override(50_000_000):
super().setUpClass()
def test_multiturn_decode_cache_hit_branching(self):
server_info = requests.get(self.base_url + "/server_info", timeout=30).json()
self.assertEqual(server_info["cuda_graph_config"]["prefill"]["backend"], "full")
graph_count = _prefill_graph_count(self.base_url)
super().test_multiturn_decode_cache_hit_branching()
self.assertGreater(
_prefill_graph_count(self.base_url),
graph_count,
"Delayed HiCache test did not replay the configured Full CUDA graph.",
)
class TestUnifiedHybridMTPBitExact(CustomTestCase):
"""Same exactness bar with MTP driving the decode loop.
@@ -1,5 +1,6 @@
"""CPU coverage for chunked-prefix Full prefill CUDA-graph state."""
import threading
import unittest
from types import SimpleNamespace
from unittest.mock import patch
@@ -18,6 +19,9 @@ from sglang.srt.managers.scheduler_components.dp_attn import (
_local_prefill_cuda_graph_vote,
)
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.unified_cache.unified_cache_linker import (
LayerWiseLoadCounter,
)
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
@@ -751,5 +755,49 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
self.assertFalse(runner.can_run_graph(forward_batch))
class TestPrefillCudaGraphRunnerLayerLoads(CustomTestCase):
@staticmethod
def _runner(counter, is_full_backend=True):
runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner)
runner._is_full_backend = is_full_backend
runner.model_runner = SimpleNamespace(
token_to_kv_pool=SimpleNamespace(layer_transfer_counter=counter)
)
return runner
def test_replay_waits_for_every_layer_of_an_inflight_load(self):
"""Replay skips per-layer waits, so it must not start while the batch's
layer-wise load is still writing its last layer."""
counter = LayerWiseLoadCounter(num_layers=3)
counter.set_consumer(counter.update_producer())
counter.complete(0, 0)
counter.complete(0, 1)
waiter = threading.Thread(
target=self._runner(counter)._wait_for_layer_loads,
daemon=True,
)
waiter.start()
waiter.join(timeout=0.2)
self.assertTrue(waiter.is_alive())
counter.complete(0, 2)
waiter.join(timeout=5)
self.assertFalse(waiter.is_alive())
def test_replay_without_a_layer_transfer_counter(self):
self._runner(None)._wait_for_layer_loads()
def test_breakable_replay_keeps_per_layer_load_overlap(self):
counter = LayerWiseLoadCounter(num_layers=3)
counter.set_consumer(counter.update_producer())
waiter = threading.Thread(
target=self._runner(counter, is_full_backend=False)._wait_for_layer_loads,
daemon=True,
)
waiter.start()
waiter.join(timeout=5)
self.assertFalse(waiter.is_alive())
if __name__ == "__main__":
unittest.main()