[Fix] diffusion: size the in-context slot from both input grids for wan-animate-2(#44090)

Co-authored-by: Ankit Dhall <8938083+ankitdhall@users.noreply.github.com>
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
Ankit Dhall
2026-10-19 10:11:58 +08:00
committed by GitHub
co-authored by Ankit Dhall Mick
parent a7fd2e1158
commit 642f43ce59
3 changed files with 177 additions and 3 deletions
@@ -439,7 +439,12 @@ class _InContextLayout:
Three sets of sizes: this clip's token counts; those of a full-length clip, which every
clip is padded up to; and the full-length sizes rounded up to 128 for flex_attention.
The attention pads to the last set and the block mask is built from the same numbers,
so one compiled mask serves every clip of a request."""
so one compiled mask serves every clip of a request.
Both streams are laid into per-frame slots of one width, the larger of the two grids'
tokens per frame: the reference image and the driving video are each fitted to the
pixel budget at their own aspect ratio, so their 16-aligned grids can differ by a
latent row or column in either direction."""
def __init__(
self,
@@ -470,8 +475,12 @@ class _InContextLayout:
)
# A full-length clip. The generation video carries one extra reference-image frame,
# the reference video does not.
self.max_tokens_per_frame = full_clip_h * full_clip_w
# the reference video does not. Each frame slot must hold a frame of either stream;
# when the grids agree, or the reference video's is the smaller, this is the
# generation grid's tokens per frame.
self.max_tokens_per_frame = max(
full_clip_h * full_clip_w, reference_video_h * reference_video_w
)
self.max_generation_video_num_tokens = full_clip_f * self.max_tokens_per_frame
self.max_reference_video_num_tokens = (
full_clip_f - 1
@@ -60,6 +60,7 @@ from sglang.multimodal_gen.runtime.models.dits.wan_animate_2 import (
)
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import (
WanAnimate2TransformerBlock,
_InContextLayout,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
DPMSolverMultistepScheduler,
@@ -881,6 +882,45 @@ def test_resize_by_area_treats_the_request_size_as_a_pixel_area_budget(
assert out.shape[0] % 16 == 0 and out.shape[1] % 16 == 0
def test_attention_slot_covers_both_grids_when_the_inputs_round_differently():
# The reference image and the driving video are each fitted to the budget at their own
# aspect ratio (_build_request_state calls resize_by_area once per input), so their
# 16-aligned canvases can differ by one step per side with the floor going either
# way. At 720x1280 a 1072x592 image lands on 1280x704 and a 538x304 video on 1264x720:
# the video grid is one latent row taller and one column narrower and holds more
# tokens per frame (79x45 = 3555 > 80x44 = 3520). The in-context layout must size its
# per-frame slot for both, or the reference K/V copy overflows.
budget = 720 * 1280
clip_len = 81
image, _ = resize_by_area(np.zeros((592, 1072, 3), np.uint8), budget, divisor=16)
video_frame, _ = resize_by_area(
np.zeros((304, 538, 3), np.uint8), budget, divisor=16
)
assert (image.shape[1], image.shape[0]) == (1280, 704)
assert (video_frame.shape[1], video_frame.shape[0]) == (1264, 720)
# Generation grid: the image canvas through the VAE (//8) and the (2, 2) patch (//2);
# reference-video grid: the same for the video canvas, one frame fewer (no
# reference-image frame), as build_clip_conditioning derives it from the VAE latent.
full_clip_grid = generation_token_grid(
clip_len, image.shape[0] // 8, image.shape[1] // 8
)
reference_video_grid = (
full_clip_grid[0] - 1,
video_frame.shape[0] // 8 // 2,
video_frame.shape[1] // 8 // 2,
)
assert full_clip_grid == (22, 44, 80)
assert reference_video_grid == (21, 45, 79)
layout = _InContextLayout(full_clip_grid, reference_video_grid, full_clip_grid)
assert layout.generation_video_tokens_per_frame == 3520
assert layout.reference_video_tokens_per_frame == 3555
assert layout.max_tokens_per_frame == 3555
assert layout.max_tokens_per_frame >= layout.generation_video_tokens_per_frame
assert layout.max_tokens_per_frame >= layout.reference_video_tokens_per_frame
# build_clip_conditioning: per-clip tensors from the request state with stand-in encoders.
@@ -6,6 +6,7 @@ Single-process and CPU only, in the style of ``test_sp_shard.py`` / ``test_usp_r
shape helpers are called directly. The parallelism rejections live in ``test_wan_animate_2_config.py``.
"""
import math
from collections import OrderedDict
from contextlib import nullcontext
from types import SimpleNamespace
@@ -238,6 +239,93 @@ def test_attention_pads_tokens_into_the_slots_the_block_mask_admits():
assert not any(valid(qi, reference_start + ki) for qi in (0, 1) for ki in (0, 1))
def test_attention_slots_fit_a_reference_frame_wider_than_the_generation_frame():
# resize_by_area fits the reference image and the driving video to the pixel budget
# independently, so the reference video's grid can hold more tokens per frame than the
# generation grid (720x1280 with 16:9-ish inputs: 79x45 against 80x44). The slot must
# then take the reference frame's width, or the reference K/V copy raises "The expanded
# size of the tensor (4) must match the existing size (5)".
layout = _InContextLayout((2, 2, 2), (1, 1, 5), (2, 2, 2))
captured = {}
def fake_flex_attention(q, k, v, *, block_mask, score_mod):
captured.update(q=q, k=k, v=v)
return torch.zeros_like(q)
batch, heads, dim = 1, 2, 8
def tokens(values, scale):
base = torch.tensor(values, dtype=torch.float32).view(1, len(values), 1, 1)
return (base * scale).expand(batch, len(values), heads, dim).clone()
# Generation frames 0 (1..4) and 1 (5..8) of 4 tokens each; one 5-token reference frame.
gen_ids = [1, 2, 3, 4, 5, 6, 7, 8]
q, k, v = tokens(gen_ids, 1.0), tokens(gen_ids, 10.0), tokens(gen_ids, 100.0)
reference_k = tokens([1, 2, 3, 4, 5], 1000.0)
reference_v = tokens([1, 2, 3, 4, 5], 10000.0)
with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention):
out = WanAnimate2TransformerBlock._in_context_attention(
SimpleNamespace(log_scale=0.0),
q,
k,
v,
reference_k,
reference_v,
"SENTINEL_BLOCK_MASK",
layout,
)
assert tuple(out.shape) == (batch, 8, heads, dim)
q_padded = captured["q"].transpose(1, 2)
k_padded = captured["k"].transpose(1, 2)
v_padded = captured["v"].transpose(1, 2)
def slot(t, i):
return t[0, i, 0, 0].item()
# Generation tokens sit at the 5-token stride, slot 4 of each frame empty.
assert [slot(q_padded, i) for i in range(10)] == [1, 2, 3, 4, 0, 5, 6, 7, 8, 0]
assert [slot(k_padded, i) for i in range(10)] == [
10,
20,
30,
40,
0,
50,
60,
70,
80,
0,
]
assert q_padded[:, 10:].abs().sum() == 0
# The five reference tokens fill their slot.
reference_start = layout.padded_generation_video_len
assert [slot(k_padded, reference_start + i) for i in range(5)] == [
1000,
2000,
3000,
4000,
5000,
]
assert [slot(v_padded, reference_start + i) for i in range(5)] == [
10000,
20000,
30000,
40000,
50000,
]
assert k_padded[:, 10:reference_start].abs().sum() == 0
assert k_padded[:, reference_start + 5 :].abs().sum() == 0
# The mask routes frame-1 queries to all five reference slots and frame-0 queries to none.
valid = _mask_probe(layout.block_mask_fn())
assert all(valid(qi, reference_start + ki) for qi in range(5, 9) for ki in range(5))
assert not any(
valid(qi, reference_start + ki) for qi in range(4) for ki in range(5)
)
assert not any(valid(qi, reference_start + 5) for qi in range(5, 9))
def test_in_context_attention_hands_flex_a_score_mod_only_when_log_scale_is_nonzero():
# At log_scale 0 the bias is an exact identity, so flex must run without a score_mod;
# otherwise it must get the cached partial (compiled flex guards on its identity).
@@ -572,6 +660,43 @@ def test_layout_pads_the_full_clip_geometry_to_128_token_blocks(
) == expected
@pytest.mark.parametrize(
"full_clip_grid, reference_video_grid, expected_tokens_per_frame",
[
# 720x1280 budget, 1072x592 image and 538x304 video: the video's canvas (1264x720)
# is one latent row taller and one column narrower than the image's (1280x704),
# 79x45 = 3555 tokens against 80x44 = 3520; the slot takes the larger.
((22, 44, 80), (21, 45, 79), 3555),
# Mirror case, the video's grid is the smaller: the generation grid sizes the slot.
((22, 45, 79), (21, 44, 80), 3555),
# Equal grids: unchanged.
((22, 44, 80), (21, 44, 80), 3520),
],
)
def test_layout_slot_is_the_larger_of_the_two_grids_tokens_per_frame(
full_clip_grid, reference_video_grid, expected_tokens_per_frame
):
layout = _InContextLayout(full_clip_grid, reference_video_grid, full_clip_grid)
full_clip_f = full_clip_grid[0]
assert layout.max_tokens_per_frame == expected_tokens_per_frame
assert layout.max_tokens_per_frame >= layout.generation_video_tokens_per_frame
assert layout.max_tokens_per_frame >= layout.reference_video_tokens_per_frame
assert (
layout.max_generation_video_num_tokens
== full_clip_f * expected_tokens_per_frame
)
assert (
layout.max_reference_video_num_tokens
== (full_clip_f - 1) * expected_tokens_per_frame
)
assert layout.padded_generation_video_len == (
math.ceil(layout.max_generation_video_num_tokens / 128) * 128
)
assert layout.padded_reference_video_len == (
math.ceil(layout.max_reference_video_num_tokens / 128) * 128
)
class _KvTaggingTransformer:
"""Fake DiT: ``build_reference_kv`` tags the K/V with the clip's reference-video latents,
``__call__`` records the (clip_cond, reference_kv) pair each forward was handed."""