diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index b7d31afaef..4b032138a9 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -2311,8 +2311,7 @@ multimodal_breakable_cuda_graph_supported_model_archs = [ "PaddleOCRVLForConditionalGeneration", "Qwen3_5ForConditionalGeneration", "Qwen3_5MoeForConditionalGeneration", - # Qwen4-Exp is intentionally absent: QSA builds host-side sparse metadata - # per forward and cannot serve the breakable prefill capture. + "Qwen4ExpForConditionalGeneration", "MuseGlimmerForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK25ForConditionalGeneration", diff --git a/python/sglang/srt/layers/attention/qsa/metadata.py b/python/sglang/srt/layers/attention/qsa/metadata.py index 581f4bf776..fe17cf0e34 100644 --- a/python/sglang/srt/layers/attention/qsa/metadata.py +++ b/python/sglang/srt/layers/attention/qsa/metadata.py @@ -91,6 +91,8 @@ class QSAIndexerMetadata(msgspec.Struct, frozen=True): extend_rope_matrix: Optional[torch.Tensor] = None graph_ring_group_locs: Optional[torch.Tensor] = None defer_block_expansion: bool = False + # Scheduler lengths for the breakable prefill indexer. + prefill_sequence_lengths_cpu: Optional[Tuple[int, ...]] = None def get_seqlens_int32(self) -> torch.Tensor: return self.sequence_lengths.to(torch.int32) @@ -139,7 +141,9 @@ class QSAIndexerMetadata(msgspec.Struct, frozen=True): compressed_buffer = pool.get_qsa_compressed_k_buffer(layer_id) parts = [] sequence_lengths = self.sequence_lengths.to(torch.int32) - sequence_lengths_list = sequence_lengths.tolist() + sequence_lengths_list = self.prefill_sequence_lengths_cpu + if sequence_lengths_list is None: + sequence_lengths_list = sequence_lengths.tolist() for sequence_id in range(len(sequence_lengths_list)): complete_blocks = int(sequence_lengths_list[sequence_id]) // ratio if complete_blocks == 0: diff --git a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py index 127fb65950..7acae5284d 100644 --- a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py +++ b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py @@ -51,6 +51,9 @@ from sglang.srt.layers.cp.utils import ( from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( + is_in_breakable_cuda_graph, +) from sglang.srt.runtime_context import get_spec from sglang.srt.utils import is_hip @@ -1511,7 +1514,12 @@ class QwenSparseAttnBackend(AttentionBackend): k_buffer = pool.get_key_buffer(layer.layer_id) v_buffer = pool.get_value_buffer(layer.layer_id) req_to_token = self.req_to_token_pool.req_to_token - req_indices = forward_batch.req_pool_indices.tolist() + if is_in_breakable_cuda_graph(): + # Reuse the prepared slot table to avoid a D2H sync per layer on replay. + req_to_token = self._resolve_metadata(forward_batch).token_slot_table + req_indices = range(len(sequence_lens)) + else: + req_indices = forward_batch.req_pool_indices.tolist() k_parts = [ k_buffer.index_select( 0, req_to_token[req_indices[i], : sequence_lens[i]].long() diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index a7c57c4292..3c650b67a7 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -397,6 +397,10 @@ def _local_prefill_cuda_graph_vote( is_target_verify=mode.is_target_verify(), capture_hidden_mode=None, return_logprob=return_logprob, + contains_mm_inputs=any( + mm_input is not None and mm_input.contains_mm_input() + for mm_input in local_batch.multimodal_inputs or () + ), lora_ineligible=prefill_graph_runner.enable_lora, is_mixed=mode == ForwardMode.MIXED, batch_max_context_len=( diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index f097279272..6edd052207 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -52,6 +52,7 @@ import tqdm from sglang.kernels.ops.kvcache.kv_indices import ( create_chunked_prefix_cache_kv_indices, ) +from sglang.srt.configs.model_config import is_qwen4_exp from sglang.srt.distributed.parallel_state import graph_capture from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp from sglang.srt.layers.cp.bcg import ( @@ -310,6 +311,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # --- prefill graph config ------------------------------------- prefill_config = get_exec().graph.cuda_graph_config.prefill self.prefill_backend_name = prefill_config.backend + qwen_bcg = self.prefill_backend_name == Backend.BREAKABLE and is_qwen4_exp( + model_runner.model_config.hf_config + ) + self._qwen_bcg_hc_sidechannel = qwen_bcg and not model_runner.is_draft_worker + self._qwen_bcg_mtp_draft = qwen_bcg and model_runner.is_draft_worker self.prefer_eager_mixed_prefill = ( self.prefill_backend_name == Backend.BREAKABLE and get_parallel().attn_dp_enabled @@ -802,12 +808,13 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): if embeds_name in kwargs: kwargs[embeds_name] = None break - return self.layer_model.forward( + output = self.layer_model.forward( input_ids, positions, forward_batch, **kwargs, ) + return self._pack_qwen_bcg_hc_output(output) # tc_piecewise: compile/capture the outer model.forward path. pp_kwargs = self.model_runner._pp_kwargs(pp_proxy_tensors) return self.model_runner.model.forward( @@ -1239,6 +1246,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): lora_ineligible: bool = False, is_mixed: bool = False, batch_max_context_len: Optional[int] = None, + contains_mm_inputs: bool = False, ) -> bool: """Rank-local replay eligibility: the single source of truth for ``can_run_graph`` (ForwardBatch, forward time) and the dp mlp-sync @@ -1247,6 +1255,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): ``capture_hidden_mode=None`` when unknown at the call site (it is rank-uniform; forward-time-only checking cannot split the group). """ + if contains_mm_inputs and ( + self._qwen_bcg_hc_sidechannel or self._qwen_bcg_mtp_draft + ): + return False if self._is_full_backend and batch_size > self._capture_req_slots: return False # LoRA replays need prepare_lora_batch's static metadata. lora_manager @@ -1329,6 +1341,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): is_target_verify=forward_batch.forward_mode.is_target_verify(), capture_hidden_mode=forward_batch.capture_hidden_mode, return_logprob=forward_batch.return_logprob, + contains_mm_inputs=forward_batch.contains_mm_inputs(), lora_ineligible=self.enable_lora and not ( self._capture_lora @@ -1996,6 +2009,17 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): : ie.shape[0] ].copy_(ie) + def _pack_qwen_bcg_hc_output(self, output): + if self._qwen_bcg_hc_sidechannel: + # Replay cannot repeat the model's Python assignment of HC state. + return output, self.layer_model.last_hc_hidden_states + return output + + def _restore_qwen_bcg_hc_output(self, output): + if self._qwen_bcg_hc_sidechannel: + output, self.layer_model.last_hc_hidden_states = output + return output + def _execute_body_capture( self, forward_batch: ForwardBatch, @@ -2018,12 +2042,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # text-only batches they are get_input_embeddings()(input_ids). # Copy them into the slot before replay so the graph sees the # current request's embeddings (mirrors main's BCG closure). + # The Qwen MTP draft passes its HC stream as inputs_embeds; + # its slot is registered at hidden_size * hc_count width. if ( self.model_runner.pp_group.is_first_rank and self.buffer_registry.has_slot("input_embeds") ): self._fill_input_embeds_slot(args, layer_kwargs, static_num_tokens) hs = self.backend.replay(shape_key, static_forward_batch, **kwargs) + hs = self._restore_qwen_bcg_hc_output(hs) return _slice_output_rows(hs, raw_num_tokens) if full_path else hs original_layer_forward = self.layer_model.forward diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 884806d015..ee3ddccd74 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -62,6 +62,13 @@ from sglang.srt.model_executor.forward_context import ( get_req_to_token_pool, ) from sglang.srt.model_executor.runner import get_is_capture_mode +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( + eager_on_graph, + is_in_breakable_cuda_graph, +) +from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( + get_tc_piecewise_forward_context, +) from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.qwen3_5 import ( Qwen3_5AttentionDecoderLayer, @@ -103,6 +110,28 @@ def _ple_table_is_fp8( return False +@eager_on_graph(True) +def _breakable_qsa_indexer(layer, hidden_states, positions): + # Read the live batch on every replay of this eager break. + forward_batch = get_tc_piecewise_forward_context().forward_batch + topk = layer._compute_qsa_topk_indices_eager( + hidden_states=hidden_states, + positions=positions, + forward_batch=forward_batch, + use_host_prefill_lengths=True, + ) + # Separate buffers cost up to buckets * max_tokens * block_topk * 4B * QSA layers. + # For 8 buckets / 16K tokens / topk512 / 12 layers: 3 GiB -> 384 MiB shared. + bridge = layer._qsa_prefill_topk_bridge + if bridge is None or bridge.shape[0] < hidden_states.shape[0]: + bridge = topk.new_empty((hidden_states.shape[0], topk.shape[1])) + layer._qsa_prefill_topk_bridge = bridge + output = bridge[: hidden_states.shape[0]] + output.fill_(0) + output[: topk.shape[0]].copy_(topk) + return output + + def _get_ple_forward_mode(forward_batch: ForwardBatch) -> ForwardMode: if forward_batch._original_forward_mode is not None: return forward_batch._original_forward_mode @@ -140,6 +169,55 @@ class _PLEBatch(msgspec.Struct, frozen=True): ngram_eos_token_id: Optional[int] +class _BreakablePLEBatch: + # _copy_output needs __dict__ to replace batch on replay; Struct uses slots. + def __init__(self, batch: Optional[_PLEBatch]): + self.batch = batch + + +@eager_on_graph(True) +def _breakable_prepare_ple_batch(input_ids, *, ngram_size, ngram_eos_token_id): + return _BreakablePLEBatch( + batch=_prepare_ple_batch( + input_ids=input_ids, + forward_batch=get_tc_piecewise_forward_context().forward_batch, + ngram_size=ngram_size, + ngram_eos_token_id=ngram_eos_token_id, + ) + ) + + +@eager_on_graph(True) +def _breakable_ple_prefetch(ple, batch): + ple.start_prefetch( + batch=batch.batch, + forward_batch=get_tc_piecewise_forward_context().forward_batch, + ) + + +@eager_on_graph(True) +def _breakable_ple_forward(ple, hidden_states, batch): + forward_batch = get_tc_piecewise_forward_context().forward_batch + if batch.batch is None: + # Idle batch replayed as a dummy extend: join the collective, add nothing. + ple.forward_idle(forward_batch) + return torch.zeros_like(hidden_states) + # PLE pads its output to the token bucket after applying the live layout. + return ple( + hidden_states=hidden_states, + forward_batch=forward_batch, + batch=batch.batch, + ) + + +@eager_on_graph(True) +def _breakable_commit_ple_batch(batch): + _commit_ple_batch( + batch=batch.batch, + forward_batch=get_tc_piecewise_forward_context().forward_batch, + ) + + def _prepare_ple_batch( input_ids: torch.Tensor, forward_batch: ForwardBatch, @@ -1167,7 +1245,9 @@ class Qwen4ExpPLELayer(nn.Module): def _get_prefetch_buffer( self, lookup_tokens: int, lookup_ids: torch.Tensor ) -> torch.Tensor: - if get_is_capture_mode(): + # Breakable replays keep capture mode on but prefetch inside an eager + # break with live token counts, so only real capture uses graph buffers. + if get_is_capture_mode() and not is_in_breakable_cuda_graph(): buffer = self._graph_prefetch_buffers.get(lookup_tokens) if buffer is None: buffer = self._allocate_prefetch_buffer(lookup_tokens, lookup_ids) @@ -1391,7 +1471,7 @@ class Qwen4ExpLayerExtensionMixin: residual: Optional[torch.Tensor], forward_batch: ForwardBatch, *, - ple_batch: Optional[_PLEBatch], + ple_batch: Optional[Union[_PLEBatch, _BreakablePLEBatch]], ): hc_dim = self.hc_count * self.hidden_size if hidden_states.shape[-1] != hc_dim: @@ -1401,7 +1481,14 @@ class Qwen4ExpLayerExtensionMixin: ) if self.ple is not None: - if ple_batch is None: + if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend(): + ple_query = ( + hidden_states if residual is None else hidden_states + residual + ) + hidden_states = hidden_states + _breakable_ple_forward( + ple=self.ple, hidden_states=ple_query, batch=ple_batch + ) + elif ple_batch is None: if not _get_ple_forward_mode(forward_batch).is_idle(): raise RuntimeError( "non-idle Qwen4 PLE forward is missing its batch" @@ -1557,6 +1644,7 @@ class Qwen4ExpAttentionDecoderLayer( from sglang.srt.layers.attention.qsa.glue import build_qsa_indexer self.is_qsa = is_qwen_qsa(config) + self._qsa_prefill_topk_bridge = None if self.is_qsa: self.indexer = build_qsa_indexer( config=config, @@ -1572,6 +1660,24 @@ class Qwen4ExpAttentionDecoderLayer( hidden_states: torch.Tensor, positions: torch.Tensor, forward_batch: ForwardBatch, + ) -> torch.Tensor: + if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend(): + return _breakable_qsa_indexer( + layer=self, hidden_states=hidden_states, positions=positions + ) + return self._compute_qsa_topk_indices_eager( + hidden_states=hidden_states, + positions=positions, + forward_batch=forward_batch, + ) + + def _compute_qsa_topk_indices_eager( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + *, + use_host_prefill_lengths: bool = False, ) -> torch.Tensor: from sglang.srt.layers.attention.qsa.glue import ( get_qsa_indexer_metadata, @@ -1590,6 +1696,13 @@ class Qwen4ExpAttentionDecoderLayer( indexer_metadata = get_qsa_indexer_metadata( backend, self.layer_id, forward_batch ) + if use_host_prefill_lengths: + indexer_metadata = msgspec.structs.replace( + indexer_metadata, + prefill_sequence_lengths_cpu=tuple( + forward_batch.seq_lens_cpu[: forward_batch.batch_size].tolist() + ), + ) topk_indices = self.indexer( hidden_states, positions, @@ -1615,6 +1728,8 @@ class Qwen4ExpAttentionDecoderLayer( self.is_qsa and self.alt_stream is not None and get_is_capture_mode() + # Graph breaks must stay off the decode overlap stream. + and not is_in_breakable_cuda_graph() and hidden_states.shape[0] < _QSA_INDEXER_OVERLAP_TOKEN_THRESHOLD ) attention_kwargs = {} @@ -1766,23 +1881,37 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): # there is no separate residual at a PP boundary (hc_hidden_size contract). residual = None - ple_batch = ( - _prepare_ple_batch( - input_ids, - forward_batch, + breakable_ple = ( + self.has_ple + and is_in_breakable_cuda_graph() + and forward_batch.forward_mode.is_extend() + ) + ple_batch = None + if breakable_ple: + ple_batch = _breakable_prepare_ple_batch( + input_ids=input_ids, + ngram_size=self.ple_ngram_size, + ngram_eos_token_id=self.ple_ngram_eos_token_id, + ) + elif self.has_ple: + ple_batch = _prepare_ple_batch( + input_ids=input_ids, + forward_batch=forward_batch, ngram_size=self.ple_ngram_size, ngram_eos_token_id=self.ple_ngram_eos_token_id, ) - if self.has_ple - else None - ) aux_hidden_states = AuxHiddenStateList() for i in range(self.start_layer, self.end_layer): layer = self.layers[i] if i + 1 < self.end_layer: next_ple = self.layers[i + 1].ple if next_ple is not None: - next_ple.start_prefetch(ple_batch, forward_batch) + if breakable_ple: + _breakable_ple_prefetch(ple=next_ple, batch=ple_batch) + else: + next_ple.start_prefetch( + batch=ple_batch, forward_batch=forward_batch + ) with get_global_expert_distribution_recorder().with_current_layer(i): hidden_states, residual = layer( positions=positions, @@ -1797,7 +1926,10 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): ), ) - _commit_ple_batch(ple_batch, forward_batch) + if breakable_ple: + _breakable_commit_ple_batch(batch=ple_batch) + else: + _commit_ple_batch(batch=ple_batch, forward_batch=forward_batch) if not self.pp_group.is_last_rank: proxy_tensors = {"hidden_states": hidden_states} diff --git a/python/sglang/srt/models/qwen4_exp_mtp.py b/python/sglang/srt/models/qwen4_exp_mtp.py index b32326bd4b..3e4d971571 100644 --- a/python/sglang/srt/models/qwen4_exp_mtp.py +++ b/python/sglang/srt/models/qwen4_exp_mtp.py @@ -51,6 +51,9 @@ class Qwen4ExpForCausalLMMTP(Qwen3_5ForCausalLMMTP): self.pp_group = get_parallel().pp_group self.hidden_size = config.hidden_size self.hc_count = config.hc_count + # The draft passes the fused HC stream in the inputs_embeds position. + # Register the replay slot at HC width, not the token embedding width. + self.input_embeds_hidden_size = self.hidden_size * self.hc_count self._mtp_input_fusion = self._init_mtp_input_fusion(config) self.model = Qwen4ExpModel( diff --git a/test/registered/cp/test_cp_strategy_unit.py b/test/registered/cp/test_cp_strategy_unit.py index 72ecd2d91f..edd8c5753f 100644 --- a/test/registered/cp/test_cp_strategy_unit.py +++ b/test/registered/cp/test_cp_strategy_unit.py @@ -146,6 +146,7 @@ class TestPrefillCPBCGReplay(CustomTestCase): input_embeds=None, replace_embeds=None, mm_inputs=None, + contains_mm_inputs=lambda: False, forward_mode=ForwardMode.EXTEND, capture_hidden_mode=CaptureHiddenMode.NULL, global_num_tokens_cpu=None, diff --git a/test/registered/cuda_graph/breakable/test_breakable_cuda_graph.py b/test/registered/cuda_graph/breakable/test_breakable_cuda_graph.py index 9c9faa1eb0..b44147dc05 100644 --- a/test/registered/cuda_graph/breakable/test_breakable_cuda_graph.py +++ b/test/registered/cuda_graph/breakable/test_breakable_cuda_graph.py @@ -47,6 +47,30 @@ class TestBreakableCUDAGraphBasic(CustomTestCase): cls.eager_on_graph = staticmethod(eager_on_graph) cls.device = torch.device("cuda:0") + def test_qsa_indexer_uses_live_lengths(self): + from sglang.srt.models import qwen4_exp + + context = SimpleNamespace(forward_batch=SimpleNamespace(rows=8)) + layer = SimpleNamespace(_qsa_prefill_topk_bridge=None) + layer._compute_qsa_topk_indices_eager = ( + lambda hidden_states, forward_batch, **kw: hidden_states[ + : forward_batch.rows + ] + ) + x = torch.zeros((8, 2), dtype=torch.int32, device=self.device) + inputs = dict(layer=layer, positions=None) + graph = self.BreakableCUDAGraph() + with patch.object( + qwen4_exp, "get_tc_piecewise_forward_context", return_value=context + ): + with self.BreakableCUDAGraphCapture(graph, stream=torch.cuda.Stream()): + result = qwen4_exp._breakable_qsa_indexer(hidden_states=x + 1, **inputs) + for n in (3, 8): + context.forward_batch = SimpleNamespace(rows=n) + x.fill_(2) + graph.replay() + self.assertEqual(result.tolist(), [[3, 3]] * n + [[0, 0]] * (8 - n)) + def test_no_break_capture_replay(self): """Capture and replay without any graph breaks should work like normal CUDA graph.""" x = torch.zeros(4, device=self.device) diff --git a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py index b76e82bb70..035cd667a0 100644 --- a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py +++ b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py @@ -37,6 +37,8 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): def _make_prefill_runner(self, backend): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner._is_full_backend = False + runner._qwen_bcg_hc_sidechannel = False + runner._qwen_bcg_mtp_draft = False runner.enable_lora = False runner._capture_chunked_prefix = False runner.prefill_backend_name = backend @@ -53,6 +55,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): input_embeds=None, replace_embeds=None, mm_inputs=[object()], + contains_mm_inputs=lambda: True, forward_mode=ForwardMode.EXTEND, capture_hidden_mode=CaptureHiddenMode.NULL, global_num_tokens_cpu=None, diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index 461effc9ce..8d855facbe 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -2,6 +2,8 @@ import unittest from types import SimpleNamespace from unittest.mock import Mock, patch +import torch + from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel @@ -98,6 +100,7 @@ class TestDecodeToExtendConversionVote(CustomTestCase): batch_size=lambda: 2, return_logprob=False, has_grammar=False, + multimodal_inputs=None, reqs=[ SimpleNamespace(beam_group=Mock() if beam else None), SimpleNamespace(beam_group=None), @@ -139,14 +142,20 @@ class TestDecodeToExtendConversionVote(CustomTestCase): class TestPrefillCudaGraphVote(CustomTestCase): - def _vote(self, mode): - runner = Mock(spec=dp_attn.PrefillCudaGraphRunner) + def _vote(self, mode, *, mm=None): + runner_cls = dp_attn.PrefillCudaGraphRunner + runner = Mock(spec=runner_cls) runner.enable_lora = False runner.max_context_size = None - runner.can_replay_locally.return_value = True + runner._qwen_bcg_hc_sidechannel = True + runner.can_replay_locally.side_effect = lambda **kwargs: ( + not kwargs["contains_mm_inputs"] + or runner_cls.can_replay_locally(self=runner, **kwargs) + ) batch = SimpleNamespace( forward_mode=mode, extend_num_tokens=4, + multimodal_inputs=mm, input_embeds=None, replace_embeds=None, prefix_lens=[1, 1], @@ -177,6 +186,32 @@ class TestPrefillCudaGraphVote(CustomTestCase): runner.can_replay_locally.assert_called_once() self.assertTrue(runner.can_replay_locally.call_args.kwargs["is_mixed"]) + @patch.object(dp_attn, "get_parallel") + @patch.object(dp_attn, "all_gather_single") + def test_multimodal_ranks_vote_eager(self, gather, parallel): + infos = [ + dp_attn.MLPSyncBatchInfo( + num_dp_ranks=2, + tp_size=1, + cp_size=1, + num_tokens=4, + num_tokens_for_logprob=1, + can_run_decode_cuda_graph=False, + can_run_draft_cuda_graph=False, + can_run_prefill_cuda_graph=self._vote(ForwardMode.EXTEND, mm=mm)[0], + is_extend_in_batch=True, + local_can_run_tbo=False, + local_forward_mode=ForwardMode.EXTEND.value, + ) + for mm in ([SimpleNamespace(contains_mm_input=lambda: True)], None) + ] + values = torch.cat([i._get_local_tensor(device="cpu") for i in infos]) + gather.side_effect = lambda output, *a, **kw: output.copy_(values) + parallel.return_value.tp_group.active_ranks_cpu = torch.ones(2) + for info in infos: + info.all_gather(device="cpu", group=None) + self.assertFalse(info.can_run_prefill_cuda_graph) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/runner/test_hidden_state_graph_recapture.py b/test/registered/unit/model_executor/runner/test_hidden_state_graph_recapture.py index 7581b8f7a9..19356b12d6 100644 --- a/test/registered/unit/model_executor/runner/test_hidden_state_graph_recapture.py +++ b/test/registered/unit/model_executor/runner/test_hidden_state_graph_recapture.py @@ -83,6 +83,7 @@ class TestHiddenStateGraphRecapture(CustomTestCase): batch_size=1, input_embeds=None, replace_embeds=None, + contains_mm_inputs=lambda: False, forward_mode=ForwardMode.EXTEND, capture_hidden_mode=capture_hidden_mode, spec_info=SimpleNamespace(capture_hidden_mode=spec_capture_hidden_mode), diff --git a/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py b/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py index b6d4187bdb..6293e301a8 100644 --- a/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py +++ b/test/registered/unit/model_executor/runner/test_prefill_cuda_graph_padding.py @@ -45,6 +45,7 @@ class TestPrefillCudaGraphPadding(CustomTestCase): input_embeds=None, replace_embeds=None, mm_inputs=None, + contains_mm_inputs=lambda: False, forward_mode=mode, global_forward_mode=None, _original_forward_mode=None, diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py index 29805fb3f2..ed0906d1b5 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py @@ -572,6 +572,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): capture_hidden_mode=CaptureHiddenMode.NULL, global_num_tokens_cpu=None, dp_prefill_cuda_graph_max_prefix_len=0, + contains_mm_inputs=lambda: False, return_logprob=False, extend_prefix_lens_cpu=[8], ) diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py index 3d4ef6b89d..e9d2a7017e 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py @@ -4,13 +4,14 @@ import unittest from contextlib import nullcontext from functools import partial from types import SimpleNamespace +from unittest.mock import Mock, patch import torch from sglang.srt.model_executor.cuda_graph_buffer_registry import ( build_prefill_registry, ) -from sglang.srt.model_executor.forward_batch_info import PPProxyTensors +from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors from sglang.srt.model_executor.model_runner_components.misc_utils import ( resolve_pp_proxy_dspark_hidden_size, ) @@ -55,6 +56,77 @@ def _make_pp_buffers_and_registry(): class TestPrefillCudaGraphRunnerHelpers(CustomTestCase): + def test_qwen_hc_restore_round_trip(self): + runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) + runner._qwen_bcg_hc_sidechannel = True + hidden = torch.ones((8, 4)) + runner.layer_model = SimpleNamespace(last_hc_hidden_states=hidden) + captured = runner._pack_qwen_bcg_hc_output(hidden) + runner.layer_model.last_hc_hidden_states = None + self.assertIs(runner._restore_qwen_bcg_hc_output(captured), hidden) + self.assertIs(runner.layer_model.last_hc_hidden_states, hidden) + + def test_ple_prefetch_buffer_does_not_grow_under_breakable_replay(self): + from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import ( + enable_breakable_cuda_graph, + ) + from sglang.srt.models import qwen4_exp + + ple = qwen4_exp.Qwen4ExpPLELayer.__new__(qwen4_exp.Qwen4ExpPLELayer) + ple._graph_prefetch_buffers, ple._eager_prefetch_buffer = {}, None + ple._allocate_prefetch_buffer = lambda n, ids: torch.empty((n, 2)) + ids = torch.zeros(1, dtype=torch.int64) + with patch.object(qwen4_exp, "get_is_capture_mode", return_value=True): + with enable_breakable_cuda_graph(): + for n in (1000, 1001, 1234): + self.assertEqual(ple._get_prefetch_buffer(n, ids).shape[0], n) + self.assertEqual(ple._graph_prefetch_buffers, {}) + self.assertEqual(ple._get_prefetch_buffer(16, ids).shape[0], 16) + self.assertEqual(list(ple._graph_prefetch_buffers), [16]) + + def test_ple_replay_uses_live_request_layout(self): + from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( + breakable_cuda_graph as bcg, + ) + from sglang.srt.models import qwen4_exp + + batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + tbo_parent_token_range=None, + spec_algorithm=None, + spec_info=None, + out_cache_loc=None, + _original_forward_mode=None, + global_num_token_non_padded_cpu=None, + ) + context = SimpleNamespace(forward_batch=batch) + capture = Mock(_barrier_fn=None, cuda_graph=SimpleNamespace(_break_fns=[])) + pool = Mock(get_mamba_indices=lambda indices: indices) + cases = [([8], [0] * 8), ([4, 4], [0] * 4 + [1] * 4)] + with ( + patch.object(qwen4_exp, "get_req_to_token_pool", return_value=pool), + patch.object(bcg, "_current_capture_var") as active, + patch.object( + qwen4_exp, "get_tc_piecewise_forward_context", return_value=context + ), + ): + active.get.return_value = capture + for lengths, expected in cases: + context.forward_batch = batch = SimpleNamespace(**vars(batch)) + batch.extend_seq_lens = torch.tensor(lengths) + batch.extend_seq_lens_cpu = lengths + batch.req_pool_indices = torch.arange(1, len(lengths) + 1) + if lengths == [8]: + holder = qwen4_exp._breakable_prepare_ple_batch( + input_ids=torch.arange(8), + ngram_size=None, + ngram_eos_token_id=None, + ) + else: + for replay in capture.cuda_graph._break_fns: + replay() + self.assertEqual(holder.batch.req_indices.tolist(), expected) + def test_dspark_proxy_width_requires_receiving_stage_and_model_support(self): class Model: def get_pp_proxy_dspark_hidden_size(self): @@ -250,6 +322,8 @@ class TestPrefillCudaGraphRunnerHelpers(CustomTestCase): with self.subTest(first_rank=first_rank, positional=positional): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner._is_full_backend = False + runner._qwen_bcg_hc_sidechannel = False + runner._qwen_bcg_mtp_draft = False runner._input_embeds_arg_idx = 3 backing = torch.zeros(4, 8) supplied = torch.full((4, 8), 7.0) if first_rank else None @@ -354,7 +428,10 @@ class TestPrefillCudaGraphRunnerHelpers(CustomTestCase): self.assertEqual(finalized["hidden_states"].shape, (3, 8)) def test_bcg_eager_tail_uses_live_multimodal_embeddings(self): - live_embeds = object() + for live_embeds in (object(), None): + self._check_bcg_eager_tail(live_embeds=live_embeds) + + def _check_bcg_eager_tail(self, live_embeds): live_batch = SimpleNamespace(mm_input_embeds=live_embeds) static_batch = SimpleNamespace( input_ids=None, @@ -364,6 +441,8 @@ class TestPrefillCudaGraphRunnerHelpers(CustomTestCase): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner._is_full_backend = False + runner._qwen_bcg_hc_sidechannel = False + runner._qwen_bcg_mtp_draft = True runner._input_embeds_arg_idx = None runner.buffer_registry = SimpleNamespace(has_slot=lambda _name: False) runner.backend = SimpleNamespace(replay=lambda *_args, **_kwargs: None) @@ -378,13 +457,49 @@ class TestPrefillCudaGraphRunnerHelpers(CustomTestCase): output = runner._execute_body_capture( live_batch, static_batch, - static_num_tokens=1, + static_num_tokens=1 if live_embeds is not None else 8, raw_num_tokens=1, shape_key=object(), ) self.assertIs(output, live_embeds) + def test_bcg_mtp_draft_replay_fills_hc_width_input_embeds_slot(self): + runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) + runner._is_full_backend = False + runner._qwen_bcg_hc_sidechannel = False + runner._qwen_bcg_mtp_draft = True + runner._input_embeds_arg_idx = 3 + runner.buffer_registry = SimpleNamespace(has_slot=lambda _name: True) + slot = torch.empty((8, 2 * 4)) + runner._fill_input_embeds_slot = Mock( + side_effect=lambda args, *_: slot[:1].copy_(args[3]) + ) + runner.backend = SimpleNamespace(replay=lambda *_args, **_kwargs: "replayed") + runner.layer_model = SimpleNamespace(forward=None) + runner.model_runner = SimpleNamespace( + pp_group=SimpleNamespace(is_first_rank=True), + model=SimpleNamespace( + # The draft hands its hc stream to the body in the inputs_embeds slot. + forward=lambda ids, positions, batch, **_kwargs: ( + runner.layer_model.forward( + ids, positions, batch, torch.zeros((1, 8)) + ) + ) + ), + ) + runner._prefill_forward_context = lambda *_args, **_kwargs: nullcontext() + batch = SimpleNamespace(input_ids=None, positions=None, mm_input_embeds=None) + + output = runner._execute_body_capture( + batch, batch, static_num_tokens=8, raw_num_tokens=1, shape_key=object() + ) + self.assertEqual(output, "replayed") + runner._fill_input_embeds_slot.assert_called_once() + self.assertEqual( + runner._fill_input_embeds_slot.call_args.args[0][3].shape[1], slot.shape[1] + ) + if __name__ == "__main__": unittest.main()