[QSA] Enable breakable prefill CUDA graphs for text-only Qwen3.8 Flash-Next (capture-safe metadata, MTP side-channel padding) (#41729)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Qiaolin Yu <liin1211@outlook.com>
This commit is contained in:
YAMY
2026-10-02 17:57:42 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 Qiaolin Yu
parent 65f759144d
commit 8ff02f5820
15 changed files with 381 additions and 23 deletions
+1 -2
View File
@@ -2311,8 +2311,7 @@ multimodal_breakable_cuda_graph_supported_model_archs = [
"PaddleOCRVLForConditionalGeneration",
"Qwen3_5ForConditionalGeneration",
"Qwen3_5MoeForConditionalGeneration",
# Qwen4-Exp is intentionally absent: QSA builds host-side sparse metadata
# per forward and cannot serve the breakable prefill capture.
"Qwen4ExpForConditionalGeneration",
"MuseGlimmerForConditionalGeneration",
"KimiK3ForConditionalGeneration",
"KimiK25ForConditionalGeneration",
@@ -91,6 +91,8 @@ class QSAIndexerMetadata(msgspec.Struct, frozen=True):
extend_rope_matrix: Optional[torch.Tensor] = None
graph_ring_group_locs: Optional[torch.Tensor] = None
defer_block_expansion: bool = False
# Scheduler lengths for the breakable prefill indexer.
prefill_sequence_lengths_cpu: Optional[Tuple[int, ...]] = None
def get_seqlens_int32(self) -> torch.Tensor:
return self.sequence_lengths.to(torch.int32)
@@ -139,7 +141,9 @@ class QSAIndexerMetadata(msgspec.Struct, frozen=True):
compressed_buffer = pool.get_qsa_compressed_k_buffer(layer_id)
parts = []
sequence_lengths = self.sequence_lengths.to(torch.int32)
sequence_lengths_list = sequence_lengths.tolist()
sequence_lengths_list = self.prefill_sequence_lengths_cpu
if sequence_lengths_list is None:
sequence_lengths_list = sequence_lengths.tolist()
for sequence_id in range(len(sequence_lengths_list)):
complete_blocks = int(sequence_lengths_list[sequence_id]) // ratio
if complete_blocks == 0:
@@ -51,6 +51,9 @@ from sglang.srt.layers.cp.utils import (
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
is_in_breakable_cuda_graph,
)
from sglang.srt.runtime_context import get_spec
from sglang.srt.utils import is_hip
@@ -1511,7 +1514,12 @@ class QwenSparseAttnBackend(AttentionBackend):
k_buffer = pool.get_key_buffer(layer.layer_id)
v_buffer = pool.get_value_buffer(layer.layer_id)
req_to_token = self.req_to_token_pool.req_to_token
req_indices = forward_batch.req_pool_indices.tolist()
if is_in_breakable_cuda_graph():
# Reuse the prepared slot table to avoid a D2H sync per layer on replay.
req_to_token = self._resolve_metadata(forward_batch).token_slot_table
req_indices = range(len(sequence_lens))
else:
req_indices = forward_batch.req_pool_indices.tolist()
k_parts = [
k_buffer.index_select(
0, req_to_token[req_indices[i], : sequence_lens[i]].long()
@@ -397,6 +397,10 @@ def _local_prefill_cuda_graph_vote(
is_target_verify=mode.is_target_verify(),
capture_hidden_mode=None,
return_logprob=return_logprob,
contains_mm_inputs=any(
mm_input is not None and mm_input.contains_mm_input()
for mm_input in local_batch.multimodal_inputs or ()
),
lora_ineligible=prefill_graph_runner.enable_lora,
is_mixed=mode == ForwardMode.MIXED,
batch_max_context_len=(
@@ -52,6 +52,7 @@ import tqdm
from sglang.kernels.ops.kvcache.kv_indices import (
create_chunked_prefix_cache_kv_indices,
)
from sglang.srt.configs.model_config import is_qwen4_exp
from sglang.srt.distributed.parallel_state import graph_capture
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.cp.bcg import (
@@ -310,6 +311,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# --- prefill graph config -------------------------------------
prefill_config = get_exec().graph.cuda_graph_config.prefill
self.prefill_backend_name = prefill_config.backend
qwen_bcg = self.prefill_backend_name == Backend.BREAKABLE and is_qwen4_exp(
model_runner.model_config.hf_config
)
self._qwen_bcg_hc_sidechannel = qwen_bcg and not model_runner.is_draft_worker
self._qwen_bcg_mtp_draft = qwen_bcg and model_runner.is_draft_worker
self.prefer_eager_mixed_prefill = (
self.prefill_backend_name == Backend.BREAKABLE
and get_parallel().attn_dp_enabled
@@ -802,12 +808,13 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
if embeds_name in kwargs:
kwargs[embeds_name] = None
break
return self.layer_model.forward(
output = self.layer_model.forward(
input_ids,
positions,
forward_batch,
**kwargs,
)
return self._pack_qwen_bcg_hc_output(output)
# tc_piecewise: compile/capture the outer model.forward path.
pp_kwargs = self.model_runner._pp_kwargs(pp_proxy_tensors)
return self.model_runner.model.forward(
@@ -1239,6 +1246,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
lora_ineligible: bool = False,
is_mixed: bool = False,
batch_max_context_len: Optional[int] = None,
contains_mm_inputs: bool = False,
) -> bool:
"""Rank-local replay eligibility: the single source of truth for
``can_run_graph`` (ForwardBatch, forward time) and the dp mlp-sync
@@ -1247,6 +1255,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
``capture_hidden_mode=None`` when unknown at the call site (it is
rank-uniform; forward-time-only checking cannot split the group).
"""
if contains_mm_inputs and (
self._qwen_bcg_hc_sidechannel or self._qwen_bcg_mtp_draft
):
return False
if self._is_full_backend and batch_size > self._capture_req_slots:
return False
# LoRA replays need prepare_lora_batch's static metadata. lora_manager
@@ -1329,6 +1341,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
is_target_verify=forward_batch.forward_mode.is_target_verify(),
capture_hidden_mode=forward_batch.capture_hidden_mode,
return_logprob=forward_batch.return_logprob,
contains_mm_inputs=forward_batch.contains_mm_inputs(),
lora_ineligible=self.enable_lora
and not (
self._capture_lora
@@ -1996,6 +2009,17 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
: ie.shape[0]
].copy_(ie)
def _pack_qwen_bcg_hc_output(self, output):
if self._qwen_bcg_hc_sidechannel:
# Replay cannot repeat the model's Python assignment of HC state.
return output, self.layer_model.last_hc_hidden_states
return output
def _restore_qwen_bcg_hc_output(self, output):
if self._qwen_bcg_hc_sidechannel:
output, self.layer_model.last_hc_hidden_states = output
return output
def _execute_body_capture(
self,
forward_batch: ForwardBatch,
@@ -2018,12 +2042,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# text-only batches they are get_input_embeddings()(input_ids).
# Copy them into the slot before replay so the graph sees the
# current request's embeddings (mirrors main's BCG closure).
# The Qwen MTP draft passes its HC stream as inputs_embeds;
# its slot is registered at hidden_size * hc_count width.
if (
self.model_runner.pp_group.is_first_rank
and self.buffer_registry.has_slot("input_embeds")
):
self._fill_input_embeds_slot(args, layer_kwargs, static_num_tokens)
hs = self.backend.replay(shape_key, static_forward_batch, **kwargs)
hs = self._restore_qwen_bcg_hc_output(hs)
return _slice_output_rows(hs, raw_num_tokens) if full_path else hs
original_layer_forward = self.layer_model.forward
+144 -12
View File
@@ -62,6 +62,13 @@ from sglang.srt.model_executor.forward_context import (
get_req_to_token_pool,
)
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
eager_on_graph,
is_in_breakable_cuda_graph,
)
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
get_tc_piecewise_forward_context,
)
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.qwen3_5 import (
Qwen3_5AttentionDecoderLayer,
@@ -103,6 +110,28 @@ def _ple_table_is_fp8(
return False
@eager_on_graph(True)
def _breakable_qsa_indexer(layer, hidden_states, positions):
# Read the live batch on every replay of this eager break.
forward_batch = get_tc_piecewise_forward_context().forward_batch
topk = layer._compute_qsa_topk_indices_eager(
hidden_states=hidden_states,
positions=positions,
forward_batch=forward_batch,
use_host_prefill_lengths=True,
)
# Separate buffers cost up to buckets * max_tokens * block_topk * 4B * QSA layers.
# For 8 buckets / 16K tokens / topk512 / 12 layers: 3 GiB -> 384 MiB shared.
bridge = layer._qsa_prefill_topk_bridge
if bridge is None or bridge.shape[0] < hidden_states.shape[0]:
bridge = topk.new_empty((hidden_states.shape[0], topk.shape[1]))
layer._qsa_prefill_topk_bridge = bridge
output = bridge[: hidden_states.shape[0]]
output.fill_(0)
output[: topk.shape[0]].copy_(topk)
return output
def _get_ple_forward_mode(forward_batch: ForwardBatch) -> ForwardMode:
if forward_batch._original_forward_mode is not None:
return forward_batch._original_forward_mode
@@ -140,6 +169,55 @@ class _PLEBatch(msgspec.Struct, frozen=True):
ngram_eos_token_id: Optional[int]
class _BreakablePLEBatch:
# _copy_output needs __dict__ to replace batch on replay; Struct uses slots.
def __init__(self, batch: Optional[_PLEBatch]):
self.batch = batch
@eager_on_graph(True)
def _breakable_prepare_ple_batch(input_ids, *, ngram_size, ngram_eos_token_id):
return _BreakablePLEBatch(
batch=_prepare_ple_batch(
input_ids=input_ids,
forward_batch=get_tc_piecewise_forward_context().forward_batch,
ngram_size=ngram_size,
ngram_eos_token_id=ngram_eos_token_id,
)
)
@eager_on_graph(True)
def _breakable_ple_prefetch(ple, batch):
ple.start_prefetch(
batch=batch.batch,
forward_batch=get_tc_piecewise_forward_context().forward_batch,
)
@eager_on_graph(True)
def _breakable_ple_forward(ple, hidden_states, batch):
forward_batch = get_tc_piecewise_forward_context().forward_batch
if batch.batch is None:
# Idle batch replayed as a dummy extend: join the collective, add nothing.
ple.forward_idle(forward_batch)
return torch.zeros_like(hidden_states)
# PLE pads its output to the token bucket after applying the live layout.
return ple(
hidden_states=hidden_states,
forward_batch=forward_batch,
batch=batch.batch,
)
@eager_on_graph(True)
def _breakable_commit_ple_batch(batch):
_commit_ple_batch(
batch=batch.batch,
forward_batch=get_tc_piecewise_forward_context().forward_batch,
)
def _prepare_ple_batch(
input_ids: torch.Tensor,
forward_batch: ForwardBatch,
@@ -1167,7 +1245,9 @@ class Qwen4ExpPLELayer(nn.Module):
def _get_prefetch_buffer(
self, lookup_tokens: int, lookup_ids: torch.Tensor
) -> torch.Tensor:
if get_is_capture_mode():
# Breakable replays keep capture mode on but prefetch inside an eager
# break with live token counts, so only real capture uses graph buffers.
if get_is_capture_mode() and not is_in_breakable_cuda_graph():
buffer = self._graph_prefetch_buffers.get(lookup_tokens)
if buffer is None:
buffer = self._allocate_prefetch_buffer(lookup_tokens, lookup_ids)
@@ -1391,7 +1471,7 @@ class Qwen4ExpLayerExtensionMixin:
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
*,
ple_batch: Optional[_PLEBatch],
ple_batch: Optional[Union[_PLEBatch, _BreakablePLEBatch]],
):
hc_dim = self.hc_count * self.hidden_size
if hidden_states.shape[-1] != hc_dim:
@@ -1401,7 +1481,14 @@ class Qwen4ExpLayerExtensionMixin:
)
if self.ple is not None:
if ple_batch is None:
if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend():
ple_query = (
hidden_states if residual is None else hidden_states + residual
)
hidden_states = hidden_states + _breakable_ple_forward(
ple=self.ple, hidden_states=ple_query, batch=ple_batch
)
elif ple_batch is None:
if not _get_ple_forward_mode(forward_batch).is_idle():
raise RuntimeError(
"non-idle Qwen4 PLE forward is missing its batch"
@@ -1557,6 +1644,7 @@ class Qwen4ExpAttentionDecoderLayer(
from sglang.srt.layers.attention.qsa.glue import build_qsa_indexer
self.is_qsa = is_qwen_qsa(config)
self._qsa_prefill_topk_bridge = None
if self.is_qsa:
self.indexer = build_qsa_indexer(
config=config,
@@ -1572,6 +1660,24 @@ class Qwen4ExpAttentionDecoderLayer(
hidden_states: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend():
return _breakable_qsa_indexer(
layer=self, hidden_states=hidden_states, positions=positions
)
return self._compute_qsa_topk_indices_eager(
hidden_states=hidden_states,
positions=positions,
forward_batch=forward_batch,
)
def _compute_qsa_topk_indices_eager(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
*,
use_host_prefill_lengths: bool = False,
) -> torch.Tensor:
from sglang.srt.layers.attention.qsa.glue import (
get_qsa_indexer_metadata,
@@ -1590,6 +1696,13 @@ class Qwen4ExpAttentionDecoderLayer(
indexer_metadata = get_qsa_indexer_metadata(
backend, self.layer_id, forward_batch
)
if use_host_prefill_lengths:
indexer_metadata = msgspec.structs.replace(
indexer_metadata,
prefill_sequence_lengths_cpu=tuple(
forward_batch.seq_lens_cpu[: forward_batch.batch_size].tolist()
),
)
topk_indices = self.indexer(
hidden_states,
positions,
@@ -1615,6 +1728,8 @@ class Qwen4ExpAttentionDecoderLayer(
self.is_qsa
and self.alt_stream is not None
and get_is_capture_mode()
# Graph breaks must stay off the decode overlap stream.
and not is_in_breakable_cuda_graph()
and hidden_states.shape[0] < _QSA_INDEXER_OVERLAP_TOKEN_THRESHOLD
)
attention_kwargs = {}
@@ -1766,23 +1881,37 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
# there is no separate residual at a PP boundary (hc_hidden_size contract).
residual = None
ple_batch = (
_prepare_ple_batch(
input_ids,
forward_batch,
breakable_ple = (
self.has_ple
and is_in_breakable_cuda_graph()
and forward_batch.forward_mode.is_extend()
)
ple_batch = None
if breakable_ple:
ple_batch = _breakable_prepare_ple_batch(
input_ids=input_ids,
ngram_size=self.ple_ngram_size,
ngram_eos_token_id=self.ple_ngram_eos_token_id,
)
elif self.has_ple:
ple_batch = _prepare_ple_batch(
input_ids=input_ids,
forward_batch=forward_batch,
ngram_size=self.ple_ngram_size,
ngram_eos_token_id=self.ple_ngram_eos_token_id,
)
if self.has_ple
else None
)
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
if i + 1 < self.end_layer:
next_ple = self.layers[i + 1].ple
if next_ple is not None:
next_ple.start_prefetch(ple_batch, forward_batch)
if breakable_ple:
_breakable_ple_prefetch(ple=next_ple, batch=ple_batch)
else:
next_ple.start_prefetch(
batch=ple_batch, forward_batch=forward_batch
)
with get_global_expert_distribution_recorder().with_current_layer(i):
hidden_states, residual = layer(
positions=positions,
@@ -1797,7 +1926,10 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
),
)
_commit_ple_batch(ple_batch, forward_batch)
if breakable_ple:
_breakable_commit_ple_batch(batch=ple_batch)
else:
_commit_ple_batch(batch=ple_batch, forward_batch=forward_batch)
if not self.pp_group.is_last_rank:
proxy_tensors = {"hidden_states": hidden_states}
@@ -51,6 +51,9 @@ class Qwen4ExpForCausalLMMTP(Qwen3_5ForCausalLMMTP):
self.pp_group = get_parallel().pp_group
self.hidden_size = config.hidden_size
self.hc_count = config.hc_count
# The draft passes the fused HC stream in the inputs_embeds position.
# Register the replay slot at HC width, not the token embedding width.
self.input_embeds_hidden_size = self.hidden_size * self.hc_count
self._mtp_input_fusion = self._init_mtp_input_fusion(config)
self.model = Qwen4ExpModel(