[Fix] cuda_graph: Wait for in-flight layer-wise cache loads before prefill graph replay (#44220)
This commit is contained in:
@@ -422,6 +422,9 @@ class Envs:
|
||||
SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL = EnvBool(False)
|
||||
# KL tests: skip the cache-hit count assertion (e.g. when alloc failure reduces hits)
|
||||
SGLANG_TEST_SKIP_CACHE_HIT_ASSERT = EnvBool(False)
|
||||
# HiCache: GPU sleep cycles before each layer's host-to-device load, so a forward
|
||||
# that skips the per-layer load wait reads unloaded layers
|
||||
SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES = EnvInt(0)
|
||||
|
||||
# ===================================================================
|
||||
# CI reporting: per-model metrics jsonl for nightly XPU dashboard
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any, Callable, NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains
|
||||
from sglang.srt.utils import get_device_module
|
||||
|
||||
@@ -153,7 +154,10 @@ class L2TransferEngine:
|
||||
transfers, self.host_to_device_stream, "host_to_device", start_event
|
||||
) as (transfers, completion):
|
||||
primary = transfers[0] if transfers else None
|
||||
delay_cycles = envs.SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES.get()
|
||||
for layer_id in range(transfer_layer_id_max):
|
||||
if delay_cycles:
|
||||
torch.cuda._sleep(delay_cycles)
|
||||
for transfer in transfers:
|
||||
local_layer_id = (
|
||||
transfer.layer_mapper(layer_id)
|
||||
|
||||
@@ -2268,6 +2268,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
f"capture mode ({self.capture_hidden_mode.name})."
|
||||
)
|
||||
|
||||
def _wait_for_layer_loads(self) -> None:
|
||||
# Full replay skips the per-layer load waits in the pool accessors; breakable
|
||||
# replay still runs them at its eager breaks.
|
||||
if not self._is_full_backend:
|
||||
return
|
||||
counter = self.model_runner.token_to_kv_pool.layer_transfer_counter
|
||||
if counter is not None and counter.consumer_index >= 0:
|
||||
counter.wait_until(counter.num_layers - 1)
|
||||
|
||||
def execute(
|
||||
self, forward_batch: ForwardBatch, **kwargs
|
||||
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
|
||||
@@ -2285,6 +2294,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
maybe_publish_prefill_shared_read_done(
|
||||
self.model_runner, forward_batch, self.device_module
|
||||
)
|
||||
self._wait_for_layer_loads()
|
||||
|
||||
if self.enable_cp_bcg_capture:
|
||||
assert self.max_context_size is None, (
|
||||
|
||||
+24
-1
@@ -52,6 +52,7 @@ import unittest
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kl_multiturn_utils import (
|
||||
_extract_output_logprobs,
|
||||
@@ -84,7 +85,7 @@ from sglang.test.test_utils import (
|
||||
unified_radix_tree_server_env,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=3140, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_cuda_ci(est_time=3340, stage="extra-a", runner_config="1-gpu-large")
|
||||
|
||||
_MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling")
|
||||
_MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test")
|
||||
@@ -300,6 +301,7 @@ class TestUnifiedHybridHiCacheBitExact(CustomTestCase):
|
||||
cls.model = _MODEL_PATH
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
other_args = _base_args() + [
|
||||
"--enable-metrics",
|
||||
"--enable-hierarchical-cache",
|
||||
"--hicache-ratio",
|
||||
"4",
|
||||
@@ -367,6 +369,27 @@ class TestUnifiedHybridHiCacheBitExact(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestUnifiedHybridHiCacheSlowLoadBitExact(TestUnifiedHybridHiCacheBitExact):
|
||||
"""A prefill CUDA graph must not replay over a host-to-device load that has not
|
||||
landed; the delay makes the load lose that race every time."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
with envs.SGLANG_TEST_HICACHE_H2D_DELAY_CYCLES.override(50_000_000):
|
||||
super().setUpClass()
|
||||
|
||||
def test_multiturn_decode_cache_hit_branching(self):
|
||||
server_info = requests.get(self.base_url + "/server_info", timeout=30).json()
|
||||
self.assertEqual(server_info["cuda_graph_config"]["prefill"]["backend"], "full")
|
||||
graph_count = _prefill_graph_count(self.base_url)
|
||||
super().test_multiturn_decode_cache_hit_branching()
|
||||
self.assertGreater(
|
||||
_prefill_graph_count(self.base_url),
|
||||
graph_count,
|
||||
"Delayed HiCache test did not replay the configured Full CUDA graph.",
|
||||
)
|
||||
|
||||
|
||||
class TestUnifiedHybridMTPBitExact(CustomTestCase):
|
||||
"""Same exactness bar with MTP driving the decode loop.
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""CPU coverage for chunked-prefix Full prefill CUDA-graph state."""
|
||||
|
||||
import threading
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
@@ -18,6 +19,9 @@ from sglang.srt.managers.scheduler_components.dp_attn import (
|
||||
_local_prefill_cuda_graph_vote,
|
||||
)
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.unified_cache.unified_cache_linker import (
|
||||
LayerWiseLoadCounter,
|
||||
)
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
@@ -751,5 +755,49 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
|
||||
self.assertFalse(runner.can_run_graph(forward_batch))
|
||||
|
||||
|
||||
class TestPrefillCudaGraphRunnerLayerLoads(CustomTestCase):
|
||||
@staticmethod
|
||||
def _runner(counter, is_full_backend=True):
|
||||
runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner)
|
||||
runner._is_full_backend = is_full_backend
|
||||
runner.model_runner = SimpleNamespace(
|
||||
token_to_kv_pool=SimpleNamespace(layer_transfer_counter=counter)
|
||||
)
|
||||
return runner
|
||||
|
||||
def test_replay_waits_for_every_layer_of_an_inflight_load(self):
|
||||
"""Replay skips per-layer waits, so it must not start while the batch's
|
||||
layer-wise load is still writing its last layer."""
|
||||
counter = LayerWiseLoadCounter(num_layers=3)
|
||||
counter.set_consumer(counter.update_producer())
|
||||
counter.complete(0, 0)
|
||||
counter.complete(0, 1)
|
||||
waiter = threading.Thread(
|
||||
target=self._runner(counter)._wait_for_layer_loads,
|
||||
daemon=True,
|
||||
)
|
||||
waiter.start()
|
||||
waiter.join(timeout=0.2)
|
||||
self.assertTrue(waiter.is_alive())
|
||||
|
||||
counter.complete(0, 2)
|
||||
waiter.join(timeout=5)
|
||||
self.assertFalse(waiter.is_alive())
|
||||
|
||||
def test_replay_without_a_layer_transfer_counter(self):
|
||||
self._runner(None)._wait_for_layer_loads()
|
||||
|
||||
def test_breakable_replay_keeps_per_layer_load_overlap(self):
|
||||
counter = LayerWiseLoadCounter(num_layers=3)
|
||||
counter.set_consumer(counter.update_producer())
|
||||
waiter = threading.Thread(
|
||||
target=self._runner(counter, is_full_backend=False)._wait_for_layer_loads,
|
||||
daemon=True,
|
||||
)
|
||||
waiter.start()
|
||||
waiter.join(timeout=5)
|
||||
self.assertFalse(waiter.is_alive())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user