From ae28306a74fc9ba56daab9ccb06dcf45659f58af Mon Sep 17 00:00:00 2001 From: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Date: Sat, 17 Oct 2026 19:13:57 +0800 Subject: [PATCH] [Fix] cuda_graph: Wait for in-flight layer-wise cache loads before prefill graph replay (#44220) --- python/sglang/srt/environ.py | 3 ++ python/sglang/srt/mem_cache/l2_transfer.py | 4 ++ .../runner/prefill_cuda_graph_runner.py | 10 ++++ ..._unified_radix_cache_kl_hybrid_bitexact.py | 25 +++++++++- .../test_prefill_cuda_graph_runner.py | 48 +++++++++++++++++++ 5 files changed, 89 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index a48e49c46f..80e0077766 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -422,6 +422,9 @@ class Envs: SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL = EnvBool(False) # KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits) SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False) + # HiCache: GPU sleep cycles before each layer's host-to-device load, so a forward + # that skips the per-layer load wait reads unloaded layers + SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES = EnvInt(0) # =================================================================== # CI reporting: per-model metrics jsonl for nightly XPU dashboard diff --git a/python/sglang/srt/mem_cache/l2_transfer.py b/python/sglang/srt/mem_cache/l2_transfer.py index 7c0eae564c..76d85ede6e 100644 --- a/python/sglang/srt/mem_cache/l2_transfer.py +++ b/python/sglang/srt/mem_cache/l2_transfer.py @@ -7,6 +7,7 @@ from typing import Any, Callable, NamedTuple, Optional import torch +from sglang.srt.environ import envs from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains from sglang.srt.utils import get_device_module @@ -153,7 +154,10 @@ class L2TransferEngine: transfers, self.host_to_device_stream, "host_to_device", start_event ) as (transfers, completion): primary = transfers[0] if transfers else None + delay_cycles = envs.SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES.get() for layer_id in range(transfer_layer_id_max): + if delay_cycles: + torch.cuda._sleep(delay_cycles) for transfer in transfers: local_layer_id = ( transfer.layer_mapper(layer_id) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index dba75c283e..b37773ac44 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -2268,6 +2268,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): f"capture mode ({self.capture_hidden_mode.name})." ) + def _wait_for_layer_loads(self) -> None: + # Full replay skips the per-layer load waits in the pool accessors; breakable + # replay still runs them at its eager breaks. + if not self._is_full_backend: + return + counter = self.model_runner.token_to_kv_pool.layer_transfer_counter + if counter is not None and counter.consumer_index >= 0: + counter.wait_until(counter.num_layers - 1) + def execute( self, forward_batch: ForwardBatch, **kwargs ) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]: @@ -2285,6 +2294,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): maybe_publish_prefill_shared_read_done( self.model_runner, forward_batch, self.device_module ) + self._wait_for_layer_loads() if self.enable_cp_bcg_capture: assert self.max_context_size is None, ( diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py index 8ef70f352b..8368f15d5f 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py @@ -52,6 +52,7 @@ import unittest import requests +from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kl_multiturn_utils import ( _extract_output_logprobs, @@ -84,7 +85,7 @@ from sglang.test.test_utils import ( unified_radix_tree_server_env, ) -register_cuda_ci(est_time=3140, stage="extra-a", runner_config="1-gpu-large") +register_cuda_ci(est_time=3340, stage="extra-a", runner_config="1-gpu-large") _MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling") _MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test") @@ -300,6 +301,7 @@ class TestUnifiedHybridHiCacheBitExact(CustomTestCase): cls.model = _MODEL_PATH cls.base_url = DEFAULT_URL_FOR_TEST other_args = _base_args() + [ + "--enable-metrics", "--enable-hierarchical-cache", "--hicache-ratio", "4", @@ -367,6 +369,27 @@ class TestUnifiedHybridHiCacheBitExact(CustomTestCase): ) +class TestUnifiedHybridHiCacheSlowLoadBitExact(TestUnifiedHybridHiCacheBitExact): + """A prefill CUDA graph must not replay over a host-to-device load that has not + landed; the delay makes the load lose that race every time.""" + + @classmethod + def setUpClass(cls): + with envs.SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES.override(50_000_000): + super().setUpClass() + + def test_multiturn_decode_cache_hit_branching(self): + server_info = requests.get(self.base_url + "/server_info", timeout=30).json() + self.assertEqual(server_info["cuda_graph_config"]["prefill"]["backend"], "full") + graph_count = _prefill_graph_count(self.base_url) + super().test_multiturn_decode_cache_hit_branching() + self.assertGreater( + _prefill_graph_count(self.base_url), + graph_count, + "Delayed HiCache test did not replay the configured Full CUDA graph.", + ) + + class TestUnifiedHybridMTPBitExact(CustomTestCase): """Same exactness bar with MTP driving the decode loop. diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py index 929f8dd75c..0e3e0f07f4 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py @@ -1,5 +1,6 @@ """CPU coverage for chunked-prefix Full prefill CUDA-graph state.""" +import threading import unittest from types import SimpleNamespace from unittest.mock import patch @@ -18,6 +19,9 @@ from sglang.srt.managers.scheduler_components.dp_attn import ( _local_prefill_cuda_graph_vote, ) from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.unified_cache.unified_cache_linker import ( + LayerWiseLoadCounter, +) from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, @@ -751,5 +755,49 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): self.assertFalse(runner.can_run_graph(forward_batch)) +class TestPrefillCudaGraphRunnerLayerLoads(CustomTestCase): + @staticmethod + def _runner(counter, is_full_backend=True): + runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) + runner._is_full_backend = is_full_backend + runner.model_runner = SimpleNamespace( + token_to_kv_pool=SimpleNamespace(layer_transfer_counter=counter) + ) + return runner + + def test_replay_waits_for_every_layer_of_an_inflight_load(self): + """Replay skips per-layer waits, so it must not start while the batch's + layer-wise load is still writing its last layer.""" + counter = LayerWiseLoadCounter(num_layers=3) + counter.set_consumer(counter.update_producer()) + counter.complete(0, 0) + counter.complete(0, 1) + waiter = threading.Thread( + target=self._runner(counter)._wait_for_layer_loads, + daemon=True, + ) + waiter.start() + waiter.join(timeout=0.2) + self.assertTrue(waiter.is_alive()) + + counter.complete(0, 2) + waiter.join(timeout=5) + self.assertFalse(waiter.is_alive()) + + def test_replay_without_a_layer_transfer_counter(self): + self._runner(None)._wait_for_layer_loads() + + def test_breakable_replay_keeps_per_layer_load_overlap(self): + counter = LayerWiseLoadCounter(num_layers=3) + counter.set_consumer(counter.update_producer()) + waiter = threading.Thread( + target=self._runner(counter, is_full_backend=False)._wait_for_layer_loads, + daemon=True, + ) + waiter.start() + waiter.join(timeout=5) + self.assertFalse(waiter.is_alive()) + + if __name__ == "__main__": unittest.main()