[Fix] cuda_graph: Wait for in-flight layer-wise cache loads before prefill graph replay (#44220)

This commit is contained in:
Shuwen Wang
2026-10-17 19:13:57 +08:00
committed by GitHub
parent 5fcb07da34
commit ae28306a74
5 changed files with 89 additions and 1 deletions
+3
View File
@@ -422,6 +422,9 @@ class Envs:
SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL = EnvBool(False)
# KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits)
SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False)
# HiCache: GPU sleep cycles before each layer's host-to-device load, so a forward
# that skips the per-layer load wait reads unloaded layers
SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES = EnvInt(0)
# ===================================================================
# CI reporting: per-model metrics jsonl for nightly XPU dashboard
@@ -7,6 +7,7 @@ from typing import Any, Callable, NamedTuple, Optional
import torch
from sglang.srt.environ import envs
from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains
from sglang.srt.utils import get_device_module
@@ -153,7 +154,10 @@ class L2TransferEngine:
transfers, self.host_to_device_stream, "host_to_device", start_event
) as (transfers, completion):
primary = transfers[0] if transfers else None
delay_cycles = envs.SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES.get()
for layer_id in range(transfer_layer_id_max):
if delay_cycles:
torch.cuda._sleep(delay_cycles)
for transfer in transfers:
local_layer_id = (
transfer.layer_mapper(layer_id)
@@ -2268,6 +2268,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
f"capture mode ({self.capture_hidden_mode.name})."
)
def _wait_for_layer_loads(self) -> None:
# Full replay skips the per-layer load waits in the pool accessors; breakable
# replay still runs them at its eager breaks.
if not self._is_full_backend:
return
counter = self.model_runner.token_to_kv_pool.layer_transfer_counter
if counter is not None and counter.consumer_index >= 0:
counter.wait_until(counter.num_layers - 1)
def execute(
self, forward_batch: ForwardBatch, **kwargs
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
@@ -2285,6 +2294,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
maybe_publish_prefill_shared_read_done(
self.model_runner, forward_batch, self.device_module
)
self._wait_for_layer_loads()
if self.enable_cp_bcg_capture:
assert self.max_context_size is None, (