[QSA] Enable breakable prefill CUDA graphs for text-only Qwen3.8 Flash-Next (capture-safe metadata, MTP side-channel padding) (#41729)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Qiaolin Yu <liin1211@outlook.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Qiaolin Yu
parent
65f759144d
commit
8ff02f5820
@@ -2311,8 +2311,7 @@ multimodal_breakable_cuda_graph_supported_model_archs = [
|
||||
"PaddleOCRVLForConditionalGeneration",
|
||||
"Qwen3_5ForConditionalGeneration",
|
||||
"Qwen3_5MoeForConditionalGeneration",
|
||||
# Qwen4-Exp is intentionally absent: QSA builds host-side sparse metadata
|
||||
# per forward and cannot serve the breakable prefill capture.
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
"MuseGlimmerForConditionalGeneration",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK25ForConditionalGeneration",
|
||||
|
||||
@@ -91,6 +91,8 @@ class QSAIndexerMetadata(msgspec.Struct, frozen=True):
|
||||
extend_rope_matrix: Optional[torch.Tensor] = None
|
||||
graph_ring_group_locs: Optional[torch.Tensor] = None
|
||||
defer_block_expansion: bool = False
|
||||
# Scheduler lengths for the breakable prefill indexer.
|
||||
prefill_sequence_lengths_cpu: Optional[Tuple[int, ...]] = None
|
||||
|
||||
def get_seqlens_int32(self) -> torch.Tensor:
|
||||
return self.sequence_lengths.to(torch.int32)
|
||||
@@ -139,7 +141,9 @@ class QSAIndexerMetadata(msgspec.Struct, frozen=True):
|
||||
compressed_buffer = pool.get_qsa_compressed_k_buffer(layer_id)
|
||||
parts = []
|
||||
sequence_lengths = self.sequence_lengths.to(torch.int32)
|
||||
sequence_lengths_list = sequence_lengths.tolist()
|
||||
sequence_lengths_list = self.prefill_sequence_lengths_cpu
|
||||
if sequence_lengths_list is None:
|
||||
sequence_lengths_list = sequence_lengths.tolist()
|
||||
for sequence_id in range(len(sequence_lengths_list)):
|
||||
complete_blocks = int(sequence_lengths_list[sequence_id]) // ratio
|
||||
if complete_blocks == 0:
|
||||
|
||||
@@ -51,6 +51,9 @@ from sglang.srt.layers.cp.utils import (
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
@@ -1511,7 +1514,12 @@ class QwenSparseAttnBackend(AttentionBackend):
|
||||
k_buffer = pool.get_key_buffer(layer.layer_id)
|
||||
v_buffer = pool.get_value_buffer(layer.layer_id)
|
||||
req_to_token = self.req_to_token_pool.req_to_token
|
||||
req_indices = forward_batch.req_pool_indices.tolist()
|
||||
if is_in_breakable_cuda_graph():
|
||||
# Reuse the prepared slot table to avoid a D2H sync per layer on replay.
|
||||
req_to_token = self._resolve_metadata(forward_batch).token_slot_table
|
||||
req_indices = range(len(sequence_lens))
|
||||
else:
|
||||
req_indices = forward_batch.req_pool_indices.tolist()
|
||||
k_parts = [
|
||||
k_buffer.index_select(
|
||||
0, req_to_token[req_indices[i], : sequence_lens[i]].long()
|
||||
|
||||
@@ -397,6 +397,10 @@ def _local_prefill_cuda_graph_vote(
|
||||
is_target_verify=mode.is_target_verify(),
|
||||
capture_hidden_mode=None,
|
||||
return_logprob=return_logprob,
|
||||
contains_mm_inputs=any(
|
||||
mm_input is not None and mm_input.contains_mm_input()
|
||||
for mm_input in local_batch.multimodal_inputs or ()
|
||||
),
|
||||
lora_ineligible=prefill_graph_runner.enable_lora,
|
||||
is_mixed=mode == ForwardMode.MIXED,
|
||||
batch_max_context_len=(
|
||||
|
||||
@@ -52,6 +52,7 @@ import tqdm
|
||||
from sglang.kernels.ops.kvcache.kv_indices import (
|
||||
create_chunked_prefix_cache_kv_indices,
|
||||
)
|
||||
from sglang.srt.configs.model_config import is_qwen4_exp
|
||||
from sglang.srt.distributed.parallel_state import graph_capture
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.cp.bcg import (
|
||||
@@ -310,6 +311,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# --- prefill graph config -------------------------------------
|
||||
prefill_config = get_exec().graph.cuda_graph_config.prefill
|
||||
self.prefill_backend_name = prefill_config.backend
|
||||
qwen_bcg = self.prefill_backend_name == Backend.BREAKABLE and is_qwen4_exp(
|
||||
model_runner.model_config.hf_config
|
||||
)
|
||||
self._qwen_bcg_hc_sidechannel = qwen_bcg and not model_runner.is_draft_worker
|
||||
self._qwen_bcg_mtp_draft = qwen_bcg and model_runner.is_draft_worker
|
||||
self.prefer_eager_mixed_prefill = (
|
||||
self.prefill_backend_name == Backend.BREAKABLE
|
||||
and get_parallel().attn_dp_enabled
|
||||
@@ -802,12 +808,13 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
if embeds_name in kwargs:
|
||||
kwargs[embeds_name] = None
|
||||
break
|
||||
return self.layer_model.forward(
|
||||
output = self.layer_model.forward(
|
||||
input_ids,
|
||||
positions,
|
||||
forward_batch,
|
||||
**kwargs,
|
||||
)
|
||||
return self._pack_qwen_bcg_hc_output(output)
|
||||
# tc_piecewise: compile/capture the outer model.forward path.
|
||||
pp_kwargs = self.model_runner._pp_kwargs(pp_proxy_tensors)
|
||||
return self.model_runner.model.forward(
|
||||
@@ -1239,6 +1246,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
lora_ineligible: bool = False,
|
||||
is_mixed: bool = False,
|
||||
batch_max_context_len: Optional[int] = None,
|
||||
contains_mm_inputs: bool = False,
|
||||
) -> bool:
|
||||
"""Rank-local replay eligibility: the single source of truth for
|
||||
``can_run_graph`` (ForwardBatch, forward time) and the dp mlp-sync
|
||||
@@ -1247,6 +1255,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
``capture_hidden_mode=None`` when unknown at the call site (it is
|
||||
rank-uniform; forward-time-only checking cannot split the group).
|
||||
"""
|
||||
if contains_mm_inputs and (
|
||||
self._qwen_bcg_hc_sidechannel or self._qwen_bcg_mtp_draft
|
||||
):
|
||||
return False
|
||||
if self._is_full_backend and batch_size > self._capture_req_slots:
|
||||
return False
|
||||
# LoRA replays need prepare_lora_batch's static metadata. lora_manager
|
||||
@@ -1329,6 +1341,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
is_target_verify=forward_batch.forward_mode.is_target_verify(),
|
||||
capture_hidden_mode=forward_batch.capture_hidden_mode,
|
||||
return_logprob=forward_batch.return_logprob,
|
||||
contains_mm_inputs=forward_batch.contains_mm_inputs(),
|
||||
lora_ineligible=self.enable_lora
|
||||
and not (
|
||||
self._capture_lora
|
||||
@@ -1996,6 +2009,17 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
: ie.shape[0]
|
||||
].copy_(ie)
|
||||
|
||||
def _pack_qwen_bcg_hc_output(self, output):
|
||||
if self._qwen_bcg_hc_sidechannel:
|
||||
# Replay cannot repeat the model's Python assignment of HC state.
|
||||
return output, self.layer_model.last_hc_hidden_states
|
||||
return output
|
||||
|
||||
def _restore_qwen_bcg_hc_output(self, output):
|
||||
if self._qwen_bcg_hc_sidechannel:
|
||||
output, self.layer_model.last_hc_hidden_states = output
|
||||
return output
|
||||
|
||||
def _execute_body_capture(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
@@ -2018,12 +2042,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# text-only batches they are get_input_embeddings()(input_ids).
|
||||
# Copy them into the slot before replay so the graph sees the
|
||||
# current request's embeddings (mirrors main's BCG closure).
|
||||
# The Qwen MTP draft passes its HC stream as inputs_embeds;
|
||||
# its slot is registered at hidden_size * hc_count width.
|
||||
if (
|
||||
self.model_runner.pp_group.is_first_rank
|
||||
and self.buffer_registry.has_slot("input_embeds")
|
||||
):
|
||||
self._fill_input_embeds_slot(args, layer_kwargs, static_num_tokens)
|
||||
hs = self.backend.replay(shape_key, static_forward_batch, **kwargs)
|
||||
hs = self._restore_qwen_bcg_hc_output(hs)
|
||||
return _slice_output_rows(hs, raw_num_tokens) if full_path else hs
|
||||
|
||||
original_layer_forward = self.layer_model.forward
|
||||
|
||||
@@ -62,6 +62,13 @@ from sglang.srt.model_executor.forward_context import (
|
||||
get_req_to_token_pool,
|
||||
)
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
eager_on_graph,
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
|
||||
get_tc_piecewise_forward_context,
|
||||
)
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.qwen3_5 import (
|
||||
Qwen3_5AttentionDecoderLayer,
|
||||
@@ -103,6 +110,28 @@ def _ple_table_is_fp8(
|
||||
return False
|
||||
|
||||
|
||||
@eager_on_graph(True)
|
||||
def _breakable_qsa_indexer(layer, hidden_states, positions):
|
||||
# Read the live batch on every replay of this eager break.
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
topk = layer._compute_qsa_topk_indices_eager(
|
||||
hidden_states=hidden_states,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
use_host_prefill_lengths=True,
|
||||
)
|
||||
# Separate buffers cost up to buckets * max_tokens * block_topk * 4B * QSA layers.
|
||||
# For 8 buckets / 16K tokens / topk512 / 12 layers: 3 GiB -> 384 MiB shared.
|
||||
bridge = layer._qsa_prefill_topk_bridge
|
||||
if bridge is None or bridge.shape[0] < hidden_states.shape[0]:
|
||||
bridge = topk.new_empty((hidden_states.shape[0], topk.shape[1]))
|
||||
layer._qsa_prefill_topk_bridge = bridge
|
||||
output = bridge[: hidden_states.shape[0]]
|
||||
output.fill_(0)
|
||||
output[: topk.shape[0]].copy_(topk)
|
||||
return output
|
||||
|
||||
|
||||
def _get_ple_forward_mode(forward_batch: ForwardBatch) -> ForwardMode:
|
||||
if forward_batch._original_forward_mode is not None:
|
||||
return forward_batch._original_forward_mode
|
||||
@@ -140,6 +169,55 @@ class _PLEBatch(msgspec.Struct, frozen=True):
|
||||
ngram_eos_token_id: Optional[int]
|
||||
|
||||
|
||||
class _BreakablePLEBatch:
|
||||
# _copy_output needs __dict__ to replace batch on replay; Struct uses slots.
|
||||
def __init__(self, batch: Optional[_PLEBatch]):
|
||||
self.batch = batch
|
||||
|
||||
|
||||
@eager_on_graph(True)
|
||||
def _breakable_prepare_ple_batch(input_ids, *, ngram_size, ngram_eos_token_id):
|
||||
return _BreakablePLEBatch(
|
||||
batch=_prepare_ple_batch(
|
||||
input_ids=input_ids,
|
||||
forward_batch=get_tc_piecewise_forward_context().forward_batch,
|
||||
ngram_size=ngram_size,
|
||||
ngram_eos_token_id=ngram_eos_token_id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@eager_on_graph(True)
|
||||
def _breakable_ple_prefetch(ple, batch):
|
||||
ple.start_prefetch(
|
||||
batch=batch.batch,
|
||||
forward_batch=get_tc_piecewise_forward_context().forward_batch,
|
||||
)
|
||||
|
||||
|
||||
@eager_on_graph(True)
|
||||
def _breakable_ple_forward(ple, hidden_states, batch):
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
if batch.batch is None:
|
||||
# Idle batch replayed as a dummy extend: join the collective, add nothing.
|
||||
ple.forward_idle(forward_batch)
|
||||
return torch.zeros_like(hidden_states)
|
||||
# PLE pads its output to the token bucket after applying the live layout.
|
||||
return ple(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
batch=batch.batch,
|
||||
)
|
||||
|
||||
|
||||
@eager_on_graph(True)
|
||||
def _breakable_commit_ple_batch(batch):
|
||||
_commit_ple_batch(
|
||||
batch=batch.batch,
|
||||
forward_batch=get_tc_piecewise_forward_context().forward_batch,
|
||||
)
|
||||
|
||||
|
||||
def _prepare_ple_batch(
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
@@ -1167,7 +1245,9 @@ class Qwen4ExpPLELayer(nn.Module):
|
||||
def _get_prefetch_buffer(
|
||||
self, lookup_tokens: int, lookup_ids: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
if get_is_capture_mode():
|
||||
# Breakable replays keep capture mode on but prefetch inside an eager
|
||||
# break with live token counts, so only real capture uses graph buffers.
|
||||
if get_is_capture_mode() and not is_in_breakable_cuda_graph():
|
||||
buffer = self._graph_prefetch_buffers.get(lookup_tokens)
|
||||
if buffer is None:
|
||||
buffer = self._allocate_prefetch_buffer(lookup_tokens, lookup_ids)
|
||||
@@ -1391,7 +1471,7 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
residual: Optional[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
ple_batch: Optional[_PLEBatch],
|
||||
ple_batch: Optional[Union[_PLEBatch, _BreakablePLEBatch]],
|
||||
):
|
||||
hc_dim = self.hc_count * self.hidden_size
|
||||
if hidden_states.shape[-1] != hc_dim:
|
||||
@@ -1401,7 +1481,14 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
)
|
||||
|
||||
if self.ple is not None:
|
||||
if ple_batch is None:
|
||||
if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend():
|
||||
ple_query = (
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
)
|
||||
hidden_states = hidden_states + _breakable_ple_forward(
|
||||
ple=self.ple, hidden_states=ple_query, batch=ple_batch
|
||||
)
|
||||
elif ple_batch is None:
|
||||
if not _get_ple_forward_mode(forward_batch).is_idle():
|
||||
raise RuntimeError(
|
||||
"non-idle Qwen4 PLE forward is missing its batch"
|
||||
@@ -1557,6 +1644,7 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
from sglang.srt.layers.attention.qsa.glue import build_qsa_indexer
|
||||
|
||||
self.is_qsa = is_qwen_qsa(config)
|
||||
self._qsa_prefill_topk_bridge = None
|
||||
if self.is_qsa:
|
||||
self.indexer = build_qsa_indexer(
|
||||
config=config,
|
||||
@@ -1572,6 +1660,24 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
if is_in_breakable_cuda_graph() and forward_batch.forward_mode.is_extend():
|
||||
return _breakable_qsa_indexer(
|
||||
layer=self, hidden_states=hidden_states, positions=positions
|
||||
)
|
||||
return self._compute_qsa_topk_indices_eager(
|
||||
hidden_states=hidden_states,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
def _compute_qsa_topk_indices_eager(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
use_host_prefill_lengths: bool = False,
|
||||
) -> torch.Tensor:
|
||||
from sglang.srt.layers.attention.qsa.glue import (
|
||||
get_qsa_indexer_metadata,
|
||||
@@ -1590,6 +1696,13 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
indexer_metadata = get_qsa_indexer_metadata(
|
||||
backend, self.layer_id, forward_batch
|
||||
)
|
||||
if use_host_prefill_lengths:
|
||||
indexer_metadata = msgspec.structs.replace(
|
||||
indexer_metadata,
|
||||
prefill_sequence_lengths_cpu=tuple(
|
||||
forward_batch.seq_lens_cpu[: forward_batch.batch_size].tolist()
|
||||
),
|
||||
)
|
||||
topk_indices = self.indexer(
|
||||
hidden_states,
|
||||
positions,
|
||||
@@ -1615,6 +1728,8 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
self.is_qsa
|
||||
and self.alt_stream is not None
|
||||
and get_is_capture_mode()
|
||||
# Graph breaks must stay off the decode overlap stream.
|
||||
and not is_in_breakable_cuda_graph()
|
||||
and hidden_states.shape[0] < _QSA_INDEXER_OVERLAP_TOKEN_THRESHOLD
|
||||
)
|
||||
attention_kwargs = {}
|
||||
@@ -1766,23 +1881,37 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
# there is no separate residual at a PP boundary (hc_hidden_size contract).
|
||||
residual = None
|
||||
|
||||
ple_batch = (
|
||||
_prepare_ple_batch(
|
||||
input_ids,
|
||||
forward_batch,
|
||||
breakable_ple = (
|
||||
self.has_ple
|
||||
and is_in_breakable_cuda_graph()
|
||||
and forward_batch.forward_mode.is_extend()
|
||||
)
|
||||
ple_batch = None
|
||||
if breakable_ple:
|
||||
ple_batch = _breakable_prepare_ple_batch(
|
||||
input_ids=input_ids,
|
||||
ngram_size=self.ple_ngram_size,
|
||||
ngram_eos_token_id=self.ple_ngram_eos_token_id,
|
||||
)
|
||||
elif self.has_ple:
|
||||
ple_batch = _prepare_ple_batch(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
ngram_size=self.ple_ngram_size,
|
||||
ngram_eos_token_id=self.ple_ngram_eos_token_id,
|
||||
)
|
||||
if self.has_ple
|
||||
else None
|
||||
)
|
||||
aux_hidden_states = AuxHiddenStateList()
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
if i + 1 < self.end_layer:
|
||||
next_ple = self.layers[i + 1].ple
|
||||
if next_ple is not None:
|
||||
next_ple.start_prefetch(ple_batch, forward_batch)
|
||||
if breakable_ple:
|
||||
_breakable_ple_prefetch(ple=next_ple, batch=ple_batch)
|
||||
else:
|
||||
next_ple.start_prefetch(
|
||||
batch=ple_batch, forward_batch=forward_batch
|
||||
)
|
||||
with get_global_expert_distribution_recorder().with_current_layer(i):
|
||||
hidden_states, residual = layer(
|
||||
positions=positions,
|
||||
@@ -1797,7 +1926,10 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
),
|
||||
)
|
||||
|
||||
_commit_ple_batch(ple_batch, forward_batch)
|
||||
if breakable_ple:
|
||||
_breakable_commit_ple_batch(batch=ple_batch)
|
||||
else:
|
||||
_commit_ple_batch(batch=ple_batch, forward_batch=forward_batch)
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
proxy_tensors = {"hidden_states": hidden_states}
|
||||
|
||||
@@ -51,6 +51,9 @@ class Qwen4ExpForCausalLMMTP(Qwen3_5ForCausalLMMTP):
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.hidden_size = config.hidden_size
|
||||
self.hc_count = config.hc_count
|
||||
# The draft passes the fused HC stream in the inputs_embeds position.
|
||||
# Register the replay slot at HC width, not the token embedding width.
|
||||
self.input_embeds_hidden_size = self.hidden_size * self.hc_count
|
||||
self._mtp_input_fusion = self._init_mtp_input_fusion(config)
|
||||
|
||||
self.model = Qwen4ExpModel(
|
||||
|
||||
Reference in New Issue
Block a user