[diffusion] fix: replay Qwen-Image-2.1 BCG graphs across prompt lengths (#43633)

Co-authored-by: Mick <mickqian@radixark.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-11 10:55:01 +08:00
committed by GitHub
co-authored by Mick Claude Opus 5.5
parent b237afa547
commit b21826e609
6 changed files with 306 additions and 19 deletions
@@ -0,0 +1,72 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0
# ==============================================================================
"""Qwen-Image 2.1 breakable CUDA graph (BCG) prompt padding.
Only cached steps replay graphs: prefill runs eagerly and stores each layer's
prefix K/V at its exact length, and a cached step reads nothing else of the
prompt. The padder left-pads that K/V to the text bucket, stacks it into one
tensor (one input copy per replay) and drops the prefill-only inputs, so prompts
of different lengths share one signature. The pad rows never reach attention or
RoPE: the DiT attends past them at a graph break, so replays stay bit-exact with
the unpadded eager forward.
"""
from __future__ import annotations
from typing import Any
import torch
from sglang.multimodal_gen.runtime.breakable_cuda_graph import (
prompt_padding as bcg_utils,
)
from sglang.multimodal_gen.runtime.distributed import get_sp_world_size
def is_qwen_image21_transformer(current_model: Any, call_kwargs: dict) -> bool:
return (
bcg_utils.transformer_class_name_matches(current_model, "qwenimage21")
and "prefix_caches" in call_kwargs
)
def pad_qwen_image21_prompt_kwargs(
call_kwargs: dict, current_model: Any, buckets: tuple[int, ...]
) -> dict:
caches = call_kwargs["prefix_caches"]
# only cached single-sample steps replay: prefill runs eagerly, warmup
# captures one sample, and the SP attention path has no pad-aware break
if (
caches is None
or len(caches) != 1
or not caches[0][0]
or get_sp_world_size() != 1
):
return call_kwargs
seq = caches[0][0]["key"].shape[1]
# a longer prefix (e.g. an edit's condition-image tokens) runs eagerly; the
# runner reports that miss once, so skip the per-step bucket warning
bucket = next((bucket for bucket in buckets if seq <= bucket), None)
if bucket is None:
return call_kwargs
rows = torch.stack(
[tensor for cache in caches[0] for tensor in (cache["key"], cache["value"])]
)
prefix_kv = rows.new_zeros(*rows.shape[:2], bucket, *rows.shape[3:])
prefix_kv[:, :, bucket - seq :] = rows
return dict(
call_kwargs,
encoder_hidden_states=None,
encoder_hidden_states_mask=None,
condition_latents=None,
layouts=[{"target_rope": call_kwargs["layouts"][0]["target_rope"]}],
prefix_caches=None,
prefix_kv=prefix_kv,
prefix_pad=torch.tensor(bucket - seq),
)
bcg_utils.register_prompt_padder(
is_qwen_image21_transformer, pad_qwen_image21_prompt_kwargs
)
@@ -299,6 +299,7 @@ def _ensure_model_padders_registered() -> None:
ming_image,
minimax_h3,
qwen_image,
qwen_image21,
sana_video,
zimage,
)
@@ -25,6 +25,9 @@ from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_kv_triton import (
from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_triton import (
qknorm_complex_rope,
)
from sglang.multimodal_gen.runtime.breakable_cuda_graph.replay_token import (
get_current_replay_token,
)
from sglang.multimodal_gen.runtime.distributed import (
get_sp_world_size,
get_tp_world_size,
@@ -42,6 +45,7 @@ from sglang.multimodal_gen.runtime.layers.linear import (
UnquantizedLinearMethod,
)
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
@@ -52,6 +56,9 @@ from sglang.multimodal_gen.runtime.platforms import (
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
eager_on_graph,
)
_is_cuda = current_platform.is_cuda()
@@ -218,6 +225,31 @@ def project_into(layer, x, out):
return out
class PrefixPad:
"""Rows BCG left-padded onto every cached prefix K/V, read on the host once per forward or graph replay."""
def __init__(self, rows):
self.rows = rows
self.token = self.value = None
def get(self):
token = get_current_replay_token()
if self.value is None or token != self.token:
self.value, self.token = int(self.rows.item()), token
return self.value
@eager_on_graph
def attend_past_prefix_pad(attn, q, k, v, prefix_pad):
# a graph break: the pad varies per request, and skipping it hands the
# kernel exactly the eager K/V, so padded replays stay bit-exact. Calls the
# impl like USPAttention does without SP, since its forward is a break too
rows = prefix_pad.get()
return attn.attn_impl.forward(
q, k[:, rows:], v[:, rows:], get_forward_context().attn_metadata
)
def cat_outputs(outputs, dim=0):
# torch.cat on a one-element list degrades to a full copy of the tensor.
return outputs[0] if len(outputs) == 1 else torch.cat(outputs, dim=dim)
@@ -594,6 +626,11 @@ class QwenImage21Attention(nn.Module):
return reference[0], reference[1], None, None
return k, v, k_out, v_out
def attend_target(self, q, k, v, prefix_pad):
if prefix_pad is None:
return self.target_attn(q, k, v)
return attend_past_prefix_pad(self.target_attn, q, k, v, prefix_pad)
def attend_sample(
self,
q,
@@ -601,20 +638,20 @@ class QwenImage21Attention(nn.Module):
v,
rope,
prefix,
prefix_rope,
segments,
layout,
cache,
k_out=None,
v_out=None,
prefix_pad=None,
):
if cache:
kp, vp = cache["key"], cache["value"]
prefix_output = None
else:
qp, kp, vp = self.qkv(prefix, prefix_rope)
qp, kp, vp = self.qkv(prefix, layout["prefix_rope"])
outputs = []
# text runs are causal; image blocks see the entire preceding sequence and themselves
for start, end, is_image in segments:
for start, end, is_image in layout["segments"]:
mask = None
if not is_image:
mask = (
@@ -633,7 +670,7 @@ class QwenImage21Attention(nn.Module):
if get_sp_world_size() == 1 and _KV_PACK_CUDA_FUSION.can_attempt_once():
cuda_packed = self._pack_kv_cuda(q, k, v, rope, kp, vp, k_out, v_out)
if cuda_packed is not None:
return self.target_attn(*cuda_packed), prefix_output
return self.attend_target(*cuda_packed, prefix_pad), prefix_output
q = apply_qk_norm_rope(q, self.norm_q, rope)
packed = None
if (
@@ -659,13 +696,20 @@ class QwenImage21Attention(nn.Module):
logger=logger,
)
if packed is not None:
out = self.target_attn(q, *packed)
out = self.attend_target(q, *packed, prefix_pad)
else:
k = apply_qk_norm_rope(k, self.norm_k, rope)
out = self.target_attn.forward_with_replicated_kv_prefix(q, kp, vp, k, v)
if prefix_pad is None:
out = self.target_attn.forward_with_replicated_kv_prefix(
q, kp, vp, k, v
)
else:
out = self.attend_target(
q, torch.cat([kp, k], 1), torch.cat([vp, v], 1), prefix_pad
)
return out, prefix_output
def forward(self, x, ropes, prefixes, layouts, caches):
def forward(self, x, ropes, prefixes, layouts, caches, prefix_pad):
# batch target projections while retaining each sample's unpadded prefix
packed = self._project_qkv_packed(x, layouts, caches)
if packed is not None:
@@ -686,11 +730,11 @@ class QwenImage21Attention(nn.Module):
v[sample : sample + 1],
ropes[sample],
prefixes[sample],
layout["prefix_rope"],
layout["segments"],
layout,
caches[sample],
k_out=k_out,
v_out=v_out,
prefix_pad=prefix_pad,
)
outputs.append(out)
prefix_outputs.append(prefix_out)
@@ -721,6 +765,7 @@ class QwenImage21TransformerBlock(nn.Module):
layouts,
ropes,
caches,
prefix_pad,
):
# Cache-DiT's UnifiedBlocks forwards the same args to every layer.
# Slice here so prefix KV stays per-layer after that wrap.
@@ -740,6 +785,7 @@ class QwenImage21TransformerBlock(nn.Module):
prefixes,
layouts,
caches,
prefix_pad,
)
hidden_states = residual_gate_add(hidden_states, attention, gate1)
hidden_states = residual_gate_add(
@@ -891,10 +937,20 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
layouts,
condition_latents=None,
prefix_caches=None,
prefix_kv=None,
prefix_pad=None,
**kwargs,
):
if isinstance(encoder_hidden_states, list):
encoder_hidden_states = encoder_hidden_states[0]
if prefix_kv is not None:
# BCG stacks every layer's padded prefix K/V so a replay copies one input
prefix_caches = [
[
{"key": key, "value": value}
for key, value in prefix_kv.unflatten(0, (-1, 2))
]
]
sp = get_sp_world_size()
target_len = hidden_states.shape[1]
if target_len % sp:
@@ -930,6 +986,8 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
)
prefix_states.append({"hidden_states": prefix})
ropes.append(layout["target_rope"][start:end])
if prefix_pad is not None:
prefix_pad = PrefixPad(prefix_pad)
# Same extras for every block so Cache-DiT's UnifiedBlocks wrap is valid.
# Each block slices prefix_caches by _layer_id. Visit once per layer for
# layerwise offload.
@@ -942,6 +1000,7 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin
layouts,
ropes,
prefix_caches,
prefix_pad,
)
output = self.proj_out(self.norm_out(images, temb))
if sp > 1:
@@ -264,13 +264,6 @@ def prepare_qwen21_mu(batch, server_args):
class QwenImage21DenoisingStage(DenoisingStage):
def _bcg_pad_prompt_kwargs(
self, call_kwargs, current_model=None, force_bucket=None
):
# Prefill runs eagerly. Later steps use exact-length prefix KV, so text
# padding only creates duplicate graphs without enabling more replay.
return call_kwargs
def _predict_noise(
self,
current_model,
@@ -4,6 +4,9 @@ from unittest.mock import patch
import torch
from sglang.multimodal_gen.runtime.breakable_cuda_graph.model_padders import (
qwen_image21 as qwen21_padder,
)
from sglang.multimodal_gen.runtime.breakable_cuda_graph.runner import (
DiffusionBreakableCudaGraphRunner,
_CaptureEntry,
@@ -33,6 +36,10 @@ class QwenImageTransformer2DModel(torch.nn.Module):
pass
class QwenImage21Transformer2DModel(torch.nn.Module):
pass
class OtherTransformer2DModel(torch.nn.Module):
pass
@@ -202,6 +209,7 @@ class TestDiffusionBCGPadding(unittest.TestCase):
def setUp(self):
self.stage = DenoisingStage.__new__(DenoisingStage)
self.qwen_model = QwenImageTransformer2DModel()
self.qwen21_model = QwenImage21Transformer2DModel()
self.ideogram_model = Ideogram4Transformer2DModel()
self.longcat_model = LongCatImageTransformer2DModel()
self.minimax_h3_model = MiniMaxH3DiTModel()
@@ -287,6 +295,88 @@ class TestDiffusionBCGPadding(unittest.TestCase):
self.assertEqual(_attn_mask_meta_local_pad({"local_pad": 7}), 7)
self.assertEqual(_attn_mask_meta_local_pad(DynamicVarlenMaskMeta()), 0)
def _qwen21_kwargs(self, prefix_len: int, *, num_layers: int = 2):
torch.manual_seed(prefix_len)
return {
"hidden_states": torch.zeros(1, 16, 4),
"timestep": torch.zeros(1),
"encoder_hidden_states": [torch.ones(1, prefix_len, 8)],
"encoder_hidden_states_mask": [torch.ones(1, prefix_len, dtype=torch.bool)],
"condition_latents": None,
"layouts": [
{
"encoder_seq_len": prefix_len,
"text_indices": torch.arange(prefix_len),
"image_indices": torch.zeros(0, dtype=torch.long),
"prefix_rope": torch.ones(prefix_len, 4, dtype=torch.complex64),
"target_rope": torch.ones(16, 4, dtype=torch.complex64),
"segments": ((0, prefix_len, False),),
}
],
"prefix_caches": [
[
{
"key": torch.randn(1, prefix_len, 2, 8),
"value": torch.randn(1, prefix_len, 2, 8),
}
for _ in range(num_layers)
]
],
}
def _qwen21_pad(self, kwargs, *, sp_world_size: int = 1):
with patch.object(
qwen21_padder, "get_sp_world_size", return_value=sp_world_size
):
return self.stage._bcg_pad_prompt_kwargs(
kwargs, current_model=self.qwen21_model
)
def test_qwen21_prefix_lengths_share_bucket_signature(self):
with self._patch_buckets(16, 32):
short_kwargs, long_kwargs = self._qwen21_kwargs(5), self._qwen21_kwargs(13)
short = self._qwen21_pad(short_kwargs)
longer = self._qwen21_pad(long_kwargs)
self.assertEqual(_signature_kwargs(short), _signature_kwargs(longer))
for out, kwargs, prefix_len in (
(short, short_kwargs, 5),
(longer, long_kwargs, 13),
):
self.assertEqual(out["prefix_pad"].item(), 16 - prefix_len)
self.assertIsNone(out["prefix_caches"])
# layer-major [key, value] pairs, left-padded so the real rows stay
# contiguous ahead of the target rows
prefix_kv = out["prefix_kv"].unflatten(0, (-1, 2))
self.assertEqual(prefix_kv.shape, (2, 2, 1, 16, 2, 8))
self.assertFalse(prefix_kv[:, :, :, : 16 - prefix_len].any())
for pair, original in zip(
prefix_kv, kwargs["prefix_caches"][0], strict=True
):
for padded, name in zip(pair, ("key", "value"), strict=True):
self.assertTrue(
torch.equal(padded[:, 16 - prefix_len :], original[name])
)
# cached steps never read the prompt, prefix RoPE or causal segments
self.assertIsNone(out["encoder_hidden_states"])
self.assertIsNone(out["encoder_hidden_states_mask"])
self.assertEqual(list(out["layouts"][0]), ["target_rope"])
self.assertIs(
out["layouts"][0]["target_rope"], kwargs["layouts"][0]["target_rope"]
)
self.assertEqual(kwargs["prefix_caches"][0][0]["key"].shape[1], prefix_len)
def test_qwen21_calls_without_a_replayable_prefix_are_not_padded(self):
prefill = self._qwen21_kwargs(5)
prefill["prefix_caches"] = [[{}, {}]]
batched = self._qwen21_kwargs(5)
batched["prefix_caches"] = batched["prefix_caches"] * 2
with self._patch_buckets(16, 32):
for kwargs in (prefill, batched, self._qwen21_kwargs(33)):
self.assertIs(self._qwen21_pad(kwargs), kwargs)
kwargs = self._qwen21_kwargs(5)
self.assertIs(self._qwen21_pad(kwargs, sp_world_size=2), kwargs)
def test_longcat_keeps_its_fixed_512_token_prompt_shape(self):
kwargs = {
"hidden_states": torch.zeros(1, 4096, 64),
@@ -36,6 +36,9 @@ from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import
ComposedPipelineBase,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
DenoisingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.qwen_image21 import (
QwenImage21DenoisingStage,
)
@@ -316,7 +319,9 @@ def test_graph_replay_uses_new_request_prefix(model, edit, sample_count):
):
model(**first)
stage._bcg_run(runner, first, model)
assert len(runner.entries) == 1
# warmup captures every text bucket for a single cached prefix
captured = len(stage._bcg_text_buckets()) if sample_count == 1 else 1
assert len(runner.entries) == captured
with (
torch.no_grad(),
set_forward_context(
@@ -328,7 +333,7 @@ def test_graph_replay_uses_new_request_prefix(model, edit, sample_count):
model(**second)
expected = model(**second)
actual = stage._bcg_run(runner, second, model)
assert len(runner.entries) == 1
assert len(runner.entries) == captured
torch.testing.assert_close(actual, expected, atol=1e-6, rtol=1e-6)
finally:
runner.reset()
@@ -523,6 +528,73 @@ def test_cuda_qk_rope_pack_matches_eager_prefill_and_cached_steps(
runner.reset()
def prompt_inputs_hd128(seed, text_tokens):
kwargs = inputs_hd128(seed, False)
kwargs["encoder_hidden_states"] = torch.randn(
1, text_tokens, 16, device="cuda", dtype=torch.bfloat16
)
kwargs["layouts"] = [
build_layout([False] * text_tokens, [(1, 4, 4)], (16, 56, 56), "cuda")
]
return kwargs
@pytest.mark.parametrize("kv_pack", ["cuda", "triton", "eager"])
@torch.no_grad()
def test_padded_prefix_replays_one_graph_across_prompt_lengths(
bf16_model_hd128, monkeypatch, kv_pack
):
# each K/V packing path must skip the pad rows so the replay matches the
# unpadded eager forward bit for bit
disabled = BitExactFusionGate("reference")
disabled.disable()
if kv_pack != "cuda":
monkeypatch.setattr(model_module, "_KV_PACK_CUDA_FUSION", disabled)
if kv_pack == "eager":
monkeypatch.setattr(model_module, "_KV_ROPE_FUSION", disabled)
monkeypatch.setattr(
DenoisingStage, "_bcg_text_buckets", staticmethod(lambda: (8, 16))
)
model = bf16_model_hd128
stage = object.__new__(QwenImage21DenoisingStage)
runner = DiffusionBreakableCudaGraphRunner(model, torch.device("cuda"))
replays = []
replay = runner.replay
monkeypatch.setattr(
runner,
"replay",
lambda entry, kwargs: replays.append(entry) or replay(entry, kwargs),
)
try:
warmup = prompt_inputs_hd128(3, 4)
with set_forward_context(
None,
None,
Req(sampling_params=QwenImage21SamplingParams(), is_warmup=True),
):
model(**warmup)
stage._bcg_run(runner, warmup, model)
assert len(runner.entries) == 2
replays.clear()
lengths = (2, 7, 8, 13, 16)
for seed, text_tokens in enumerate(lengths, start=20):
kwargs = prompt_inputs_hd128(seed, text_tokens)
with set_forward_context(
None, None, Req(sampling_params=QwenImage21SamplingParams())
):
model(**kwargs)
kwargs["timestep"].fill_(300.0)
expected = model(**kwargs)
padded = stage._bcg_pad_prompt_kwargs(kwargs, current_model=model)
torch.testing.assert_close(model(**padded), expected, atol=0, rtol=0)
actual = stage._bcg_run(runner, kwargs, model)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
assert len(replays) == len(lengths)
assert len(runner.entries) == 2
finally:
runner.reset()
@torch.no_grad()
def test_cuda_kv_pack_mismatch_restores_reference(bf16_model_hd128, monkeypatch):
# A kernel that disagrees with the reference must disable itself and leave