[Perf] diffusion/qwen-image-2.1: Keep an image prefix's K/V in the packed buffer across steps (#46101)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
3712b5944d
commit
738e4535ea
@@ -660,7 +660,10 @@ class QwenImage21Attention(nn.Module):
|
||||
prefix = cache["key"].shape[1] if cache else layouts[0]["prefix_rope"].shape[0]
|
||||
seq = x.shape[1]
|
||||
rows = packed.shape[0] // 3
|
||||
buffer = x.new_empty(1, prefix + seq, 3 * rows)
|
||||
# a buffer kept by an image-conditioned prefill still holds the prefix K/V rows
|
||||
buffer = cache.get("qkv") if cache else None
|
||||
if buffer is None:
|
||||
buffer = x.new_empty(1, prefix + seq, 3 * rows)
|
||||
torch.mm(x.view(seq, x.shape[-1]), packed.t(), out=buffer[0, prefix:])
|
||||
q = buffer[:, prefix:, :rows].view(1, seq, self.heads, self.head_dim)
|
||||
k_out = buffer[:, :, rows : 2 * rows].view(
|
||||
@@ -680,7 +683,7 @@ class QwenImage21Attention(nn.Module):
|
||||
)
|
||||
if accepted is reference:
|
||||
return None
|
||||
return q, k, v, k_out, v_out
|
||||
return q, k, v, k_out, v_out, buffer
|
||||
|
||||
def _project_kv_into_buffers(self, x, layouts, caches):
|
||||
"""Project K/V straight into ``[1, P+S, H, D]`` buffers; ``None`` keeps the plain path."""
|
||||
@@ -728,7 +731,16 @@ class QwenImage21Attention(nn.Module):
|
||||
k_out=None,
|
||||
v_out=None,
|
||||
prefix_pad=None,
|
||||
qkv=None,
|
||||
):
|
||||
if cache and qkv is not None and qkv is cache.get("qkv"):
|
||||
# rows [0:P] of the kept buffer are this prefix's K/V: only Q and the target K change
|
||||
rows = cache["key"].shape[1]
|
||||
empty = k_out.new_empty(1, 0, *k_out.shape[2:])
|
||||
q = self._pack_kv_cuda(
|
||||
q, k, v, rope, empty, empty, k_out[:, rows:], v_out[:, rows:]
|
||||
)[0]
|
||||
return self.attend_target(q, k_out, v_out, prefix_pad), None
|
||||
if cache:
|
||||
kp, vp = cache["key"], cache["value"]
|
||||
prefix_output = None
|
||||
@@ -755,6 +767,18 @@ class QwenImage21Attention(nn.Module):
|
||||
if get_sp_world_size() == 1 and _KV_PACK_CUDA_FUSION.can_attempt_once():
|
||||
cuda_packed = self._pack_kv_cuda(q, k, v, rope, kp, vp, k_out, v_out)
|
||||
if cuda_packed is not None:
|
||||
if (
|
||||
prefix_output is not None
|
||||
and cache is not None
|
||||
and qkv is not None
|
||||
and prefix_pad is None
|
||||
and _KV_PACK_CUDA_FUSION.verified
|
||||
and any(is_image for _, _, is_image in layout["segments"])
|
||||
):
|
||||
# an image prefix is as long as the target: copying it every step
|
||||
# costs more than keeping the packed buffer for the request
|
||||
rows = kp.shape[1]
|
||||
cache.update(qkv=qkv, key=k_out[:, :rows], value=v_out[:, :rows])
|
||||
return self.attend_target(*cuda_packed, prefix_pad), prefix_output
|
||||
q = apply_qk_norm_rope(q, self.norm_q, rope)
|
||||
packed = None
|
||||
@@ -797,8 +821,9 @@ class QwenImage21Attention(nn.Module):
|
||||
def forward(self, x, ropes, prefixes, layouts, caches, prefix_pad):
|
||||
# batch target projections while retaining each sample's unpadded prefix
|
||||
packed = self._project_qkv_packed(x, layouts, caches)
|
||||
qkv = None
|
||||
if packed is not None:
|
||||
q, k, v, k_out, v_out = packed
|
||||
q, k, v, k_out, v_out, qkv = packed
|
||||
else:
|
||||
buffers = self._project_kv_into_buffers(x, layouts, caches)
|
||||
if buffers is None:
|
||||
@@ -820,6 +845,7 @@ class QwenImage21Attention(nn.Module):
|
||||
k_out=k_out,
|
||||
v_out=v_out,
|
||||
prefix_pad=prefix_pad,
|
||||
qkv=qkv,
|
||||
)
|
||||
outputs.append(out)
|
||||
prefix_outputs.append(prefix_out)
|
||||
|
||||
@@ -512,10 +512,15 @@ def test_cuda_qk_rope_pack_matches_eager_prefill_and_cached_steps(
|
||||
):
|
||||
for key in ("key", "value"):
|
||||
torch.testing.assert_close(actual[key], reference[key], atol=0, rtol=0)
|
||||
assert (
|
||||
actual[key].untyped_storage().nbytes()
|
||||
== actual[key].numel() * actual[key].element_size()
|
||||
)
|
||||
# an image prefix lives in the packed buffer it is attended from;
|
||||
# a text prefix is a compact copy that pins nothing else
|
||||
storage = actual[key].untyped_storage()
|
||||
if edit and qkv_gate.verified:
|
||||
assert storage.data_ptr() == actual["qkv"].untyped_storage().data_ptr()
|
||||
else:
|
||||
assert (
|
||||
storage.nbytes() == actual[key].numel() * actual[key].element_size()
|
||||
)
|
||||
|
||||
runner = DiffusionBreakableCudaGraphRunner(actual_model, torch.device("cuda"))
|
||||
try:
|
||||
@@ -539,6 +544,42 @@ def prompt_inputs_hd128(seed, text_tokens):
|
||||
return kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("edit", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_cached_steps_copy_the_prefix_only_for_text(
|
||||
bf16_model_hd128, edit, monkeypatch
|
||||
):
|
||||
# Cached steps of an image prefix project into the buffer kept by the prefill and
|
||||
# write only the target rows; a text prefix is still copied into a fresh buffer.
|
||||
for name in ("_QK_ROPE_CUDA_FUSION", "_KV_PACK_CUDA_FUSION", "_QKV_PACK_FUSION"):
|
||||
monkeypatch.setattr(model_module, name, BitExactFusionGate(f"test {name}"))
|
||||
packs = []
|
||||
original = model_module.qknorm_complex_rope_pack_
|
||||
|
||||
def record(q, k_out, v_out, *args):
|
||||
packs.append((args[3].shape[1], k_out.untyped_storage().data_ptr()))
|
||||
return original(q, k_out, v_out, *args)
|
||||
|
||||
monkeypatch.setattr(model_module, "qknorm_complex_rope_pack_", record)
|
||||
kwargs = inputs_hd128(13, edit)
|
||||
with set_forward_context(None, None):
|
||||
for timestep in (700, 300, 10):
|
||||
kwargs["timestep"].fill_(timestep)
|
||||
bf16_model_hd128(**kwargs)
|
||||
if not model_module._QKV_PACK_FUSION.verified:
|
||||
pytest.skip("cuBLAS picked a different kernel for the packed projection")
|
||||
caches = kwargs["prefix_caches"][0]
|
||||
prefill, cached = packs[: len(caches)], packs[len(caches) :]
|
||||
prefix_rows = caches[0]["key"].shape[1]
|
||||
assert [rows for rows, _ in prefill] == [prefix_rows] * len(caches)
|
||||
if edit:
|
||||
kept = [cache["qkv"].untyped_storage().data_ptr() for cache in caches]
|
||||
assert cached == [(0, ptr) for ptr in kept] * 2
|
||||
else:
|
||||
assert all("qkv" not in cache for cache in caches)
|
||||
assert [rows for rows, _ in cached] == [prefix_rows] * 2 * len(caches)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_pack", ["cuda", "triton", "eager"])
|
||||
@torch.no_grad()
|
||||
def test_padded_prefix_replays_one_graph_across_prompt_lengths(
|
||||
|
||||
Reference in New Issue
Block a user