diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/qwen_image21.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/qwen_image21.py new file mode 100644 index 0000000000..163de9da4a --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/qwen_image21.py @@ -0,0 +1,72 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 +# ============================================================================== +"""Qwen-Image 2.1 breakable CUDA graph (BCG) prompt padding. + +Only cached steps replay graphs: prefill runs eagerly and stores each layer's +prefix K/V at its exact length, and a cached step reads nothing else of the +prompt. The padder left-pads that K/V to the text bucket, stacks it into one +tensor (one input copy per replay) and drops the prefill-only inputs, so prompts +of different lengths share one signature. The pad rows never reach attention or +RoPE: the DiT attends past them at a graph break, so replays stay bit-exact with +the unpadded eager forward. +""" + +from __future__ import annotations + +from typing import Any + +import torch + +from sglang.multimodal_gen.runtime.breakable_cuda_graph import ( + prompt_padding as bcg_utils, +) +from sglang.multimodal_gen.runtime.distributed import get_sp_world_size + + +def is_qwen_image21_transformer(current_model: Any, call_kwargs: dict) -> bool: + return ( + bcg_utils.transformer_class_name_matches(current_model, "qwenimage21") + and "prefix_caches" in call_kwargs + ) + + +def pad_qwen_image21_prompt_kwargs( + call_kwargs: dict, current_model: Any, buckets: tuple[int, ...] +) -> dict: + caches = call_kwargs["prefix_caches"] + # only cached single-sample steps replay: prefill runs eagerly, warmup + # captures one sample, and the SP attention path has no pad-aware break + if ( + caches is None + or len(caches) != 1 + or not caches[0][0] + or get_sp_world_size() != 1 + ): + return call_kwargs + seq = caches[0][0]["key"].shape[1] + # a longer prefix (e.g. an edit's condition-image tokens) runs eagerly; the + # runner reports that miss once, so skip the per-step bucket warning + bucket = next((bucket for bucket in buckets if seq <= bucket), None) + if bucket is None: + return call_kwargs + rows = torch.stack( + [tensor for cache in caches[0] for tensor in (cache["key"], cache["value"])] + ) + prefix_kv = rows.new_zeros(*rows.shape[:2], bucket, *rows.shape[3:]) + prefix_kv[:, :, bucket - seq :] = rows + return dict( + call_kwargs, + encoder_hidden_states=None, + encoder_hidden_states_mask=None, + condition_latents=None, + layouts=[{"target_rope": call_kwargs["layouts"][0]["target_rope"]}], + prefix_caches=None, + prefix_kv=prefix_kv, + prefix_pad=torch.tensor(bucket - seq), + ) + + +bcg_utils.register_prompt_padder( + is_qwen_image21_transformer, pad_qwen_image21_prompt_kwargs +) diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py index 345437a0e9..121f1992a8 100644 --- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py +++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/prompt_padding.py @@ -299,6 +299,7 @@ def _ensure_model_padders_registered() -> None: ming_image, minimax_h3, qwen_image, + qwen_image21, sana_video, zimage, ) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py index 1c0331c88a..a8f62de631 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py @@ -25,6 +25,9 @@ from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_kv_triton import ( from sglang.kernels.ops.diffusion.rope.qknorm_complex_rope_triton import ( qknorm_complex_rope, ) +from sglang.multimodal_gen.runtime.breakable_cuda_graph.replay_token import ( + get_current_replay_token, +) from sglang.multimodal_gen.runtime.distributed import ( get_sp_world_size, get_tp_world_size, @@ -42,6 +45,7 @@ from sglang.multimodal_gen.runtime.layers.linear import ( UnquantizedLinearMethod, ) from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA +from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, ) @@ -52,6 +56,9 @@ from sglang.multimodal_gen.runtime.platforms import ( ) from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( + eager_on_graph, +) _is_cuda = current_platform.is_cuda() @@ -218,6 +225,31 @@ def project_into(layer, x, out): return out +class PrefixPad: + """Rows BCG left-padded onto every cached prefix K/V, read on the host once per forward or graph replay.""" + + def __init__(self, rows): + self.rows = rows + self.token = self.value = None + + def get(self): + token = get_current_replay_token() + if self.value is None or token != self.token: + self.value, self.token = int(self.rows.item()), token + return self.value + + +@eager_on_graph +def attend_past_prefix_pad(attn, q, k, v, prefix_pad): + # a graph break: the pad varies per request, and skipping it hands the + # kernel exactly the eager K/V, so padded replays stay bit-exact. Calls the + # impl like USPAttention does without SP, since its forward is a break too + rows = prefix_pad.get() + return attn.attn_impl.forward( + q, k[:, rows:], v[:, rows:], get_forward_context().attn_metadata + ) + + def cat_outputs(outputs, dim=0): # torch.cat on a one-element list degrades to a full copy of the tensor. return outputs[0] if len(outputs) == 1 else torch.cat(outputs, dim=dim) @@ -594,6 +626,11 @@ class QwenImage21Attention(nn.Module): return reference[0], reference[1], None, None return k, v, k_out, v_out + def attend_target(self, q, k, v, prefix_pad): + if prefix_pad is None: + return self.target_attn(q, k, v) + return attend_past_prefix_pad(self.target_attn, q, k, v, prefix_pad) + def attend_sample( self, q, @@ -601,20 +638,20 @@ class QwenImage21Attention(nn.Module): v, rope, prefix, - prefix_rope, - segments, + layout, cache, k_out=None, v_out=None, + prefix_pad=None, ): if cache: kp, vp = cache["key"], cache["value"] prefix_output = None else: - qp, kp, vp = self.qkv(prefix, prefix_rope) + qp, kp, vp = self.qkv(prefix, layout["prefix_rope"]) outputs = [] # text runs are causal; image blocks see the entire preceding sequence and themselves - for start, end, is_image in segments: + for start, end, is_image in layout["segments"]: mask = None if not is_image: mask = ( @@ -633,7 +670,7 @@ class QwenImage21Attention(nn.Module): if get_sp_world_size() == 1 and _KV_PACK_CUDA_FUSION.can_attempt_once(): cuda_packed = self._pack_kv_cuda(q, k, v, rope, kp, vp, k_out, v_out) if cuda_packed is not None: - return self.target_attn(*cuda_packed), prefix_output + return self.attend_target(*cuda_packed, prefix_pad), prefix_output q = apply_qk_norm_rope(q, self.norm_q, rope) packed = None if ( @@ -659,13 +696,20 @@ class QwenImage21Attention(nn.Module): logger=logger, ) if packed is not None: - out = self.target_attn(q, *packed) + out = self.attend_target(q, *packed, prefix_pad) else: k = apply_qk_norm_rope(k, self.norm_k, rope) - out = self.target_attn.forward_with_replicated_kv_prefix(q, kp, vp, k, v) + if prefix_pad is None: + out = self.target_attn.forward_with_replicated_kv_prefix( + q, kp, vp, k, v + ) + else: + out = self.attend_target( + q, torch.cat([kp, k], 1), torch.cat([vp, v], 1), prefix_pad + ) return out, prefix_output - def forward(self, x, ropes, prefixes, layouts, caches): + def forward(self, x, ropes, prefixes, layouts, caches, prefix_pad): # batch target projections while retaining each sample's unpadded prefix packed = self._project_qkv_packed(x, layouts, caches) if packed is not None: @@ -686,11 +730,11 @@ class QwenImage21Attention(nn.Module): v[sample : sample + 1], ropes[sample], prefixes[sample], - layout["prefix_rope"], - layout["segments"], + layout, caches[sample], k_out=k_out, v_out=v_out, + prefix_pad=prefix_pad, ) outputs.append(out) prefix_outputs.append(prefix_out) @@ -721,6 +765,7 @@ class QwenImage21TransformerBlock(nn.Module): layouts, ropes, caches, + prefix_pad, ): # Cache-DiT's UnifiedBlocks forwards the same args to every layer. # Slice here so prefix KV stays per-layer after that wrap. @@ -740,6 +785,7 @@ class QwenImage21TransformerBlock(nn.Module): prefixes, layouts, caches, + prefix_pad, ) hidden_states = residual_gate_add(hidden_states, attention, gate1) hidden_states = residual_gate_add( @@ -891,10 +937,20 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin layouts, condition_latents=None, prefix_caches=None, + prefix_kv=None, + prefix_pad=None, **kwargs, ): if isinstance(encoder_hidden_states, list): encoder_hidden_states = encoder_hidden_states[0] + if prefix_kv is not None: + # BCG stacks every layer's padded prefix K/V so a replay copies one input + prefix_caches = [ + [ + {"key": key, "value": value} + for key, value in prefix_kv.unflatten(0, (-1, 2)) + ] + ] sp = get_sp_world_size() target_len = hidden_states.shape[1] if target_len % sp: @@ -930,6 +986,8 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin ) prefix_states.append({"hidden_states": prefix}) ropes.append(layout["target_rope"][start:end]) + if prefix_pad is not None: + prefix_pad = PrefixPad(prefix_pad) # Same extras for every block so Cache-DiT's UnifiedBlocks wrap is valid. # Each block slices prefix_caches by _layer_id. Visit once per layer for # layerwise offload. @@ -942,6 +1000,7 @@ class QwenImage21Transformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin layouts, ropes, prefix_caches, + prefix_pad, ) output = self.proj_out(self.norm_out(images, temb)) if sp > 1: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image21.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image21.py index c491469f13..33881a767c 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image21.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image21.py @@ -264,13 +264,6 @@ def prepare_qwen21_mu(batch, server_args): class QwenImage21DenoisingStage(DenoisingStage): - def _bcg_pad_prompt_kwargs( - self, call_kwargs, current_model=None, force_bucket=None - ): - # Prefill runs eagerly. Later steps use exact-length prefix KV, so text - # padding only creates duplicate graphs without enabling more replay. - return call_kwargs - def _predict_noise( self, current_model, diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py index b803b2d6f5..ed466819f4 100644 --- a/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py @@ -4,6 +4,9 @@ from unittest.mock import patch import torch +from sglang.multimodal_gen.runtime.breakable_cuda_graph.model_padders import ( + qwen_image21 as qwen21_padder, +) from sglang.multimodal_gen.runtime.breakable_cuda_graph.runner import ( DiffusionBreakableCudaGraphRunner, _CaptureEntry, @@ -33,6 +36,10 @@ class QwenImageTransformer2DModel(torch.nn.Module): pass +class QwenImage21Transformer2DModel(torch.nn.Module): + pass + + class OtherTransformer2DModel(torch.nn.Module): pass @@ -202,6 +209,7 @@ class TestDiffusionBCGPadding(unittest.TestCase): def setUp(self): self.stage = DenoisingStage.__new__(DenoisingStage) self.qwen_model = QwenImageTransformer2DModel() + self.qwen21_model = QwenImage21Transformer2DModel() self.ideogram_model = Ideogram4Transformer2DModel() self.longcat_model = LongCatImageTransformer2DModel() self.minimax_h3_model = MiniMaxH3DiTModel() @@ -287,6 +295,88 @@ class TestDiffusionBCGPadding(unittest.TestCase): self.assertEqual(_attn_mask_meta_local_pad({"local_pad": 7}), 7) self.assertEqual(_attn_mask_meta_local_pad(DynamicVarlenMaskMeta()), 0) + def _qwen21_kwargs(self, prefix_len: int, *, num_layers: int = 2): + torch.manual_seed(prefix_len) + return { + "hidden_states": torch.zeros(1, 16, 4), + "timestep": torch.zeros(1), + "encoder_hidden_states": [torch.ones(1, prefix_len, 8)], + "encoder_hidden_states_mask": [torch.ones(1, prefix_len, dtype=torch.bool)], + "condition_latents": None, + "layouts": [ + { + "encoder_seq_len": prefix_len, + "text_indices": torch.arange(prefix_len), + "image_indices": torch.zeros(0, dtype=torch.long), + "prefix_rope": torch.ones(prefix_len, 4, dtype=torch.complex64), + "target_rope": torch.ones(16, 4, dtype=torch.complex64), + "segments": ((0, prefix_len, False),), + } + ], + "prefix_caches": [ + [ + { + "key": torch.randn(1, prefix_len, 2, 8), + "value": torch.randn(1, prefix_len, 2, 8), + } + for _ in range(num_layers) + ] + ], + } + + def _qwen21_pad(self, kwargs, *, sp_world_size: int = 1): + with patch.object( + qwen21_padder, "get_sp_world_size", return_value=sp_world_size + ): + return self.stage._bcg_pad_prompt_kwargs( + kwargs, current_model=self.qwen21_model + ) + + def test_qwen21_prefix_lengths_share_bucket_signature(self): + with self._patch_buckets(16, 32): + short_kwargs, long_kwargs = self._qwen21_kwargs(5), self._qwen21_kwargs(13) + short = self._qwen21_pad(short_kwargs) + longer = self._qwen21_pad(long_kwargs) + + self.assertEqual(_signature_kwargs(short), _signature_kwargs(longer)) + for out, kwargs, prefix_len in ( + (short, short_kwargs, 5), + (longer, long_kwargs, 13), + ): + self.assertEqual(out["prefix_pad"].item(), 16 - prefix_len) + self.assertIsNone(out["prefix_caches"]) + # layer-major [key, value] pairs, left-padded so the real rows stay + # contiguous ahead of the target rows + prefix_kv = out["prefix_kv"].unflatten(0, (-1, 2)) + self.assertEqual(prefix_kv.shape, (2, 2, 1, 16, 2, 8)) + self.assertFalse(prefix_kv[:, :, :, : 16 - prefix_len].any()) + for pair, original in zip( + prefix_kv, kwargs["prefix_caches"][0], strict=True + ): + for padded, name in zip(pair, ("key", "value"), strict=True): + self.assertTrue( + torch.equal(padded[:, 16 - prefix_len :], original[name]) + ) + # cached steps never read the prompt, prefix RoPE or causal segments + self.assertIsNone(out["encoder_hidden_states"]) + self.assertIsNone(out["encoder_hidden_states_mask"]) + self.assertEqual(list(out["layouts"][0]), ["target_rope"]) + self.assertIs( + out["layouts"][0]["target_rope"], kwargs["layouts"][0]["target_rope"] + ) + self.assertEqual(kwargs["prefix_caches"][0][0]["key"].shape[1], prefix_len) + + def test_qwen21_calls_without_a_replayable_prefix_are_not_padded(self): + prefill = self._qwen21_kwargs(5) + prefill["prefix_caches"] = [[{}, {}]] + batched = self._qwen21_kwargs(5) + batched["prefix_caches"] = batched["prefix_caches"] * 2 + with self._patch_buckets(16, 32): + for kwargs in (prefill, batched, self._qwen21_kwargs(33)): + self.assertIs(self._qwen21_pad(kwargs), kwargs) + kwargs = self._qwen21_kwargs(5) + self.assertIs(self._qwen21_pad(kwargs, sp_world_size=2), kwargs) + def test_longcat_keeps_its_fixed_512_token_prompt_shape(self): kwargs = { "hidden_states": torch.zeros(1, 4096, 64), diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py index b2746c2366..dd75cbb28f 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py @@ -36,6 +36,9 @@ from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ComposedPipelineBase, ) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import ( + DenoisingStage, +) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.qwen_image21 import ( QwenImage21DenoisingStage, ) @@ -316,7 +319,9 @@ def test_graph_replay_uses_new_request_prefix(model, edit, sample_count): ): model(**first) stage._bcg_run(runner, first, model) - assert len(runner.entries) == 1 + # warmup captures every text bucket for a single cached prefix + captured = len(stage._bcg_text_buckets()) if sample_count == 1 else 1 + assert len(runner.entries) == captured with ( torch.no_grad(), set_forward_context( @@ -328,7 +333,7 @@ def test_graph_replay_uses_new_request_prefix(model, edit, sample_count): model(**second) expected = model(**second) actual = stage._bcg_run(runner, second, model) - assert len(runner.entries) == 1 + assert len(runner.entries) == captured torch.testing.assert_close(actual, expected, atol=1e-6, rtol=1e-6) finally: runner.reset() @@ -523,6 +528,73 @@ def test_cuda_qk_rope_pack_matches_eager_prefill_and_cached_steps( runner.reset() +def prompt_inputs_hd128(seed, text_tokens): + kwargs = inputs_hd128(seed, False) + kwargs["encoder_hidden_states"] = torch.randn( + 1, text_tokens, 16, device="cuda", dtype=torch.bfloat16 + ) + kwargs["layouts"] = [ + build_layout([False] * text_tokens, [(1, 4, 4)], (16, 56, 56), "cuda") + ] + return kwargs + + +@pytest.mark.parametrize("kv_pack", ["cuda", "triton", "eager"]) +@torch.no_grad() +def test_padded_prefix_replays_one_graph_across_prompt_lengths( + bf16_model_hd128, monkeypatch, kv_pack +): + # each K/V packing path must skip the pad rows so the replay matches the + # unpadded eager forward bit for bit + disabled = BitExactFusionGate("reference") + disabled.disable() + if kv_pack != "cuda": + monkeypatch.setattr(model_module, "_KV_PACK_CUDA_FUSION", disabled) + if kv_pack == "eager": + monkeypatch.setattr(model_module, "_KV_ROPE_FUSION", disabled) + monkeypatch.setattr( + DenoisingStage, "_bcg_text_buckets", staticmethod(lambda: (8, 16)) + ) + model = bf16_model_hd128 + stage = object.__new__(QwenImage21DenoisingStage) + runner = DiffusionBreakableCudaGraphRunner(model, torch.device("cuda")) + replays = [] + replay = runner.replay + monkeypatch.setattr( + runner, + "replay", + lambda entry, kwargs: replays.append(entry) or replay(entry, kwargs), + ) + try: + warmup = prompt_inputs_hd128(3, 4) + with set_forward_context( + None, + None, + Req(sampling_params=QwenImage21SamplingParams(), is_warmup=True), + ): + model(**warmup) + stage._bcg_run(runner, warmup, model) + assert len(runner.entries) == 2 + replays.clear() + lengths = (2, 7, 8, 13, 16) + for seed, text_tokens in enumerate(lengths, start=20): + kwargs = prompt_inputs_hd128(seed, text_tokens) + with set_forward_context( + None, None, Req(sampling_params=QwenImage21SamplingParams()) + ): + model(**kwargs) + kwargs["timestep"].fill_(300.0) + expected = model(**kwargs) + padded = stage._bcg_pad_prompt_kwargs(kwargs, current_model=model) + torch.testing.assert_close(model(**padded), expected, atol=0, rtol=0) + actual = stage._bcg_run(runner, kwargs, model) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + assert len(replays) == len(lengths) + assert len(runner.entries) == 2 + finally: + runner.reset() + + @torch.no_grad() def test_cuda_kv_pack_mismatch_restores_reference(bf16_model_hd128, monkeypatch): # A kernel that disagrees with the reference must disable itself and leave