[Fix] cuda_graph: Wait for in-flight layer-wise cache loads before prefill graph replay (#44220)
This commit is contained in:
@@ -422,6 +422,9 @@ class Envs:
|
||||
SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL = EnvBool(False)
|
||||
# KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits)
|
||||
SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False)
|
||||
# HiCache: GPU sleep cycles before each layer's host-to-device load, so a forward
|
||||
# that skips the per-layer load wait reads unloaded layers
|
||||
SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES = EnvInt(0)
|
||||
|
||||
# ===================================================================
|
||||
# CI reporting: per-model metrics jsonl for nightly XPU dashboard
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any, Callable, NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains
|
||||
from sglang.srt.utils import get_device_module
|
||||
|
||||
@@ -153,7 +154,10 @@ class L2TransferEngine:
|
||||
transfers, self.host_to_device_stream, "host_to_device", start_event
|
||||
) as (transfers, completion):
|
||||
primary = transfers[0] if transfers else None
|
||||
delay_cycles = envs.SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES.get()
|
||||
for layer_id in range(transfer_layer_id_max):
|
||||
if delay_cycles:
|
||||
torch.cuda._sleep(delay_cycles)
|
||||
for transfer in transfers:
|
||||
local_layer_id = (
|
||||
transfer.layer_mapper(layer_id)
|
||||
|
||||
@@ -2268,6 +2268,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
f"capture mode ({self.capture_hidden_mode.name})."
|
||||
)
|
||||
|
||||
def _wait_for_layer_loads(self) -> None:
|
||||
# Full replay skips the per-layer load waits in the pool accessors; breakable
|
||||
# replay still runs them at its eager breaks.
|
||||
if not self._is_full_backend:
|
||||
return
|
||||
counter = self.model_runner.token_to_kv_pool.layer_transfer_counter
|
||||
if counter is not None and counter.consumer_index >= 0:
|
||||
counter.wait_until(counter.num_layers - 1)
|
||||
|
||||
def execute(
|
||||
self, forward_batch: ForwardBatch, **kwargs
|
||||
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
|
||||
@@ -2285,6 +2294,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
self.model_runner, forward_batch, self.device_module
|
||||
)
|
||||
self._wait_for_layer_loads()
|
||||
|
||||
if self.enable_cp_bcg_capture:
|
||||
assert self.max_context_size is None, (
|
||||
|
||||
Reference in New Issue
Block a user