[Perf] diffusion/qwen-image-2.1: Keep an image prefix's K/V in the packed buffer across steps (#46101)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-24 09:14:10 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 3712b5944d
commit 738e4535ea
2 changed files with 74 additions and 7 deletions
@@ -660,7 +660,10 @@ class QwenImage21Attention(nn.Module):
prefix = cache["key"].shape[1] if cache else layouts[0]["prefix_rope"].shape[0]
seq = x.shape[1]
rows = packed.shape[0] // 3
buffer = x.new_empty(1, prefix + seq, 3 * rows)
# a buffer kept by an image-conditioned prefill still holds the prefix K/V rows
buffer = cache.get("qkv") if cache else None
if buffer is None:
buffer = x.new_empty(1, prefix + seq, 3 * rows)
torch.mm(x.view(seq, x.shape[-1]), packed.t(), out=buffer[0, prefix:])
q = buffer[:, prefix:, :rows].view(1, seq, self.heads, self.head_dim)
k_out = buffer[:, :, rows : 2 * rows].view(
@@ -680,7 +683,7 @@ class QwenImage21Attention(nn.Module):
)
if accepted is reference:
return None
return q, k, v, k_out, v_out
return q, k, v, k_out, v_out, buffer
def _project_kv_into_buffers(self, x, layouts, caches):
"""Project K/V straight into ``[1, P+S, H, D]`` buffers; ``None`` keeps the plain path."""
@@ -728,7 +731,16 @@ class QwenImage21Attention(nn.Module):
k_out=None,
v_out=None,
prefix_pad=None,
qkv=None,
):
if cache and qkv is not None and qkv is cache.get("qkv"):
# rows [0:P] of the kept buffer are this prefix's K/V: only Q and the target K change
rows = cache["key"].shape[1]
empty = k_out.new_empty(1, 0, *k_out.shape[2:])
q = self._pack_kv_cuda(
q, k, v, rope, empty, empty, k_out[:, rows:], v_out[:, rows:]
)[0]
return self.attend_target(q, k_out, v_out, prefix_pad), None
if cache:
kp, vp = cache["key"], cache["value"]
prefix_output = None
@@ -755,6 +767,18 @@ class QwenImage21Attention(nn.Module):
if get_sp_world_size() == 1 and _KV_PACK_CUDA_FUSION.can_attempt_once():
cuda_packed = self._pack_kv_cuda(q, k, v, rope, kp, vp, k_out, v_out)
if cuda_packed is not None:
if (
prefix_output is not None
and cache is not None
and qkv is not None
and prefix_pad is None
and _KV_PACK_CUDA_FUSION.verified
and any(is_image for _, _, is_image in layout["segments"])
):
# an image prefix is as long as the target: copying it every step
# costs more than keeping the packed buffer for the request
rows = kp.shape[1]
cache.update(qkv=qkv, key=k_out[:, :rows], value=v_out[:, :rows])
return self.attend_target(*cuda_packed, prefix_pad), prefix_output
q = apply_qk_norm_rope(q, self.norm_q, rope)
packed = None
@@ -797,8 +821,9 @@ class QwenImage21Attention(nn.Module):
def forward(self, x, ropes, prefixes, layouts, caches, prefix_pad):
# batch target projections while retaining each sample's unpadded prefix
packed = self._project_qkv_packed(x, layouts, caches)
qkv = None
if packed is not None:
q, k, v, k_out, v_out = packed
q, k, v, k_out, v_out, qkv = packed
else:
buffers = self._project_kv_into_buffers(x, layouts, caches)
if buffers is None:
@@ -820,6 +845,7 @@ class QwenImage21Attention(nn.Module):
k_out=k_out,
v_out=v_out,
prefix_pad=prefix_pad,
qkv=qkv,
)
outputs.append(out)
prefix_outputs.append(prefix_out)
@@ -512,10 +512,15 @@ def test_cuda_qk_rope_pack_matches_eager_prefill_and_cached_steps(
):
for key in ("key", "value"):
torch.testing.assert_close(actual[key], reference[key], atol=0, rtol=0)
assert (
actual[key].untyped_storage().nbytes()
== actual[key].numel() * actual[key].element_size()
)
# an image prefix lives in the packed buffer it is attended from;
# a text prefix is a compact copy that pins nothing else
storage = actual[key].untyped_storage()
if edit and qkv_gate.verified:
assert storage.data_ptr() == actual["qkv"].untyped_storage().data_ptr()
else:
assert (
storage.nbytes() == actual[key].numel() * actual[key].element_size()
)
runner = DiffusionBreakableCudaGraphRunner(actual_model, torch.device("cuda"))
try:
@@ -539,6 +544,42 @@ def prompt_inputs_hd128(seed, text_tokens):
return kwargs
@pytest.mark.parametrize("edit", [False, True])
@torch.no_grad()
def test_cached_steps_copy_the_prefix_only_for_text(
bf16_model_hd128, edit, monkeypatch
):
# Cached steps of an image prefix project into the buffer kept by the prefill and
# write only the target rows; a text prefix is still copied into a fresh buffer.
for name in ("_QK_ROPE_CUDA_FUSION", "_KV_PACK_CUDA_FUSION", "_QKV_PACK_FUSION"):
monkeypatch.setattr(model_module, name, BitExactFusionGate(f"test {name}"))
packs = []
original = model_module.qknorm_complex_rope_pack_
def record(q, k_out, v_out, *args):
packs.append((args[3].shape[1], k_out.untyped_storage().data_ptr()))
return original(q, k_out, v_out, *args)
monkeypatch.setattr(model_module, "qknorm_complex_rope_pack_", record)
kwargs = inputs_hd128(13, edit)
with set_forward_context(None, None):
for timestep in (700, 300, 10):
kwargs["timestep"].fill_(timestep)
bf16_model_hd128(**kwargs)
if not model_module._QKV_PACK_FUSION.verified:
pytest.skip("cuBLAS picked a different kernel for the packed projection")
caches = kwargs["prefix_caches"][0]
prefill, cached = packs[: len(caches)], packs[len(caches) :]
prefix_rows = caches[0]["key"].shape[1]
assert [rows for rows, _ in prefill] == [prefix_rows] * len(caches)
if edit:
kept = [cache["qkv"].untyped_storage().data_ptr() for cache in caches]
assert cached == [(0, ptr) for ptr in kept] * 2
else:
assert all("qkv" not in cache for cache in caches)
assert [rows for rows, _ in cached] == [prefix_rows] * 2 * len(caches)
@pytest.mark.parametrize("kv_pack", ["cuda", "triton", "eager"])
@torch.no_grad()
def test_padded_prefix_replays_one_graph_across_prompt_lengths(