diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py index d8c1a9b4f0..29fba16aa7 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py @@ -660,7 +660,10 @@ class QwenImage21Attention(nn.Module): prefix = cache["key"].shape[1] if cache else layouts[0]["prefix_rope"].shape[0] seq = x.shape[1] rows = packed.shape[0] // 3 - buffer = x.new_empty(1, prefix + seq, 3 * rows) + # a buffer kept by an image-conditioned prefill still holds the prefix K/V rows + buffer = cache.get("qkv") if cache else None + if buffer is None: + buffer = x.new_empty(1, prefix + seq, 3 * rows) torch.mm(x.view(seq, x.shape[-1]), packed.t(), out=buffer[0, prefix:]) q = buffer[:, prefix:, :rows].view(1, seq, self.heads, self.head_dim) k_out = buffer[:, :, rows : 2 * rows].view( @@ -680,7 +683,7 @@ class QwenImage21Attention(nn.Module): ) if accepted is reference: return None - return q, k, v, k_out, v_out + return q, k, v, k_out, v_out, buffer def _project_kv_into_buffers(self, x, layouts, caches): """Project K/V straight into ``[1, P+S, H, D]`` buffers; ``None`` keeps the plain path.""" @@ -728,7 +731,16 @@ class QwenImage21Attention(nn.Module): k_out=None, v_out=None, prefix_pad=None, + qkv=None, ): + if cache and qkv is not None and qkv is cache.get("qkv"): + # rows [0:P] of the kept buffer are this prefix's K/V: only Q and the target K change + rows = cache["key"].shape[1] + empty = k_out.new_empty(1, 0, *k_out.shape[2:]) + q = self._pack_kv_cuda( + q, k, v, rope, empty, empty, k_out[:, rows:], v_out[:, rows:] + )[0] + return self.attend_target(q, k_out, v_out, prefix_pad), None if cache: kp, vp = cache["key"], cache["value"] prefix_output = None @@ -755,6 +767,18 @@ class QwenImage21Attention(nn.Module): if get_sp_world_size() == 1 and _KV_PACK_CUDA_FUSION.can_attempt_once(): cuda_packed = self._pack_kv_cuda(q, k, v, rope, kp, vp, k_out, v_out) if cuda_packed is not None: + if ( + prefix_output is not None + and cache is not None + and qkv is not None + and prefix_pad is None + and _KV_PACK_CUDA_FUSION.verified + and any(is_image for _, _, is_image in layout["segments"]) + ): + # an image prefix is as long as the target: copying it every step + # costs more than keeping the packed buffer for the request + rows = kp.shape[1] + cache.update(qkv=qkv, key=k_out[:, :rows], value=v_out[:, :rows]) return self.attend_target(*cuda_packed, prefix_pad), prefix_output q = apply_qk_norm_rope(q, self.norm_q, rope) packed = None @@ -797,8 +821,9 @@ class QwenImage21Attention(nn.Module): def forward(self, x, ropes, prefixes, layouts, caches, prefix_pad): # batch target projections while retaining each sample's unpadded prefix packed = self._project_qkv_packed(x, layouts, caches) + qkv = None if packed is not None: - q, k, v, k_out, v_out = packed + q, k, v, k_out, v_out, qkv = packed else: buffers = self._project_kv_into_buffers(x, layouts, caches) if buffers is None: @@ -820,6 +845,7 @@ class QwenImage21Attention(nn.Module): k_out=k_out, v_out=v_out, prefix_pad=prefix_pad, + qkv=qkv, ) outputs.append(out) prefix_outputs.append(prefix_out) diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py index dd75cbb28f..c59523abde 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_cuda.py @@ -512,10 +512,15 @@ def test_cuda_qk_rope_pack_matches_eager_prefill_and_cached_steps( ): for key in ("key", "value"): torch.testing.assert_close(actual[key], reference[key], atol=0, rtol=0) - assert ( - actual[key].untyped_storage().nbytes() - == actual[key].numel() * actual[key].element_size() - ) + # an image prefix lives in the packed buffer it is attended from; + # a text prefix is a compact copy that pins nothing else + storage = actual[key].untyped_storage() + if edit and qkv_gate.verified: + assert storage.data_ptr() == actual["qkv"].untyped_storage().data_ptr() + else: + assert ( + storage.nbytes() == actual[key].numel() * actual[key].element_size() + ) runner = DiffusionBreakableCudaGraphRunner(actual_model, torch.device("cuda")) try: @@ -539,6 +544,42 @@ def prompt_inputs_hd128(seed, text_tokens): return kwargs +@pytest.mark.parametrize("edit", [False, True]) +@torch.no_grad() +def test_cached_steps_copy_the_prefix_only_for_text( + bf16_model_hd128, edit, monkeypatch +): + # Cached steps of an image prefix project into the buffer kept by the prefill and + # write only the target rows; a text prefix is still copied into a fresh buffer. + for name in ("_QK_ROPE_CUDA_FUSION", "_KV_PACK_CUDA_FUSION", "_QKV_PACK_FUSION"): + monkeypatch.setattr(model_module, name, BitExactFusionGate(f"test {name}")) + packs = [] + original = model_module.qknorm_complex_rope_pack_ + + def record(q, k_out, v_out, *args): + packs.append((args[3].shape[1], k_out.untyped_storage().data_ptr())) + return original(q, k_out, v_out, *args) + + monkeypatch.setattr(model_module, "qknorm_complex_rope_pack_", record) + kwargs = inputs_hd128(13, edit) + with set_forward_context(None, None): + for timestep in (700, 300, 10): + kwargs["timestep"].fill_(timestep) + bf16_model_hd128(**kwargs) + if not model_module._QKV_PACK_FUSION.verified: + pytest.skip("cuBLAS picked a different kernel for the packed projection") + caches = kwargs["prefix_caches"][0] + prefill, cached = packs[: len(caches)], packs[len(caches) :] + prefix_rows = caches[0]["key"].shape[1] + assert [rows for rows, _ in prefill] == [prefix_rows] * len(caches) + if edit: + kept = [cache["qkv"].untyped_storage().data_ptr() for cache in caches] + assert cached == [(0, ptr) for ptr in kept] * 2 + else: + assert all("qkv" not in cache for cache in caches) + assert [rows for rows, _ in cached] == [prefix_rows] * 2 * len(caches) + + @pytest.mark.parametrize("kv_pack", ["cuda", "triton", "eager"]) @torch.no_grad() def test_padded_prefix_replays_one_graph_across_prompt_lengths(