From 642f43ce59583cce5a40a22d5a806fe74be9ad68 Mon Sep 17 00:00:00 2001 From: Ankit Dhall Date: Mon, 19 Oct 2026 04:11:58 +0200 Subject: [PATCH] [Fix] diffusion: size the in-context slot from both input grids for wan-animate-2(#44090) Co-authored-by: Ankit Dhall <8938083+ankitdhall@users.noreply.github.com> Co-authored-by: Mick --- .../models/dits/wan_animate_2_block.py | 15 ++- .../test/unit/test_wan_animate_2_config.py | 40 ++++++ .../unit/test_wan_animate_2_parallelism.py | 125 ++++++++++++++++++ 3 files changed, 177 insertions(+), 3 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py index 2c906f3076..5d325a9bd6 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py @@ -439,7 +439,12 @@ class _InContextLayout: Three sets of sizes: this clip's token counts; those of a full-length clip, which every clip is padded up to; and the full-length sizes rounded up to 128 for flex_attention. The attention pads to the last set and the block mask is built from the same numbers, - so one compiled mask serves every clip of a request.""" + so one compiled mask serves every clip of a request. + + Both streams are laid into per-frame slots of one width, the larger of the two grids' + tokens per frame: the reference image and the driving video are each fitted to the + pixel budget at their own aspect ratio, so their 16-aligned grids can differ by a + latent row or column in either direction.""" def __init__( self, @@ -470,8 +475,12 @@ class _InContextLayout: ) # A full-length clip. The generation video carries one extra reference-image frame, - # the reference video does not. - self.max_tokens_per_frame = full_clip_h * full_clip_w + # the reference video does not. Each frame slot must hold a frame of either stream; + # when the grids agree, or the reference video's is the smaller, this is the + # generation grid's tokens per frame. + self.max_tokens_per_frame = max( + full_clip_h * full_clip_w, reference_video_h * reference_video_w + ) self.max_generation_video_num_tokens = full_clip_f * self.max_tokens_per_frame self.max_reference_video_num_tokens = ( full_clip_f - 1 diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py index 906969871c..bd25910d66 100644 --- a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py +++ b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py @@ -60,6 +60,7 @@ from sglang.multimodal_gen.runtime.models.dits.wan_animate_2 import ( ) from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import ( WanAnimate2TransformerBlock, + _InContextLayout, ) from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import ( DPMSolverMultistepScheduler, @@ -881,6 +882,45 @@ def test_resize_by_area_treats_the_request_size_as_a_pixel_area_budget( assert out.shape[0] % 16 == 0 and out.shape[1] % 16 == 0 +def test_attention_slot_covers_both_grids_when_the_inputs_round_differently(): + # The reference image and the driving video are each fitted to the budget at their own + # aspect ratio (_build_request_state calls resize_by_area once per input), so their + # 16-aligned canvases can differ by one step per side with the floor going either + # way. At 720x1280 a 1072x592 image lands on 1280x704 and a 538x304 video on 1264x720: + # the video grid is one latent row taller and one column narrower and holds more + # tokens per frame (79x45 = 3555 > 80x44 = 3520). The in-context layout must size its + # per-frame slot for both, or the reference K/V copy overflows. + budget = 720 * 1280 + clip_len = 81 + image, _ = resize_by_area(np.zeros((592, 1072, 3), np.uint8), budget, divisor=16) + video_frame, _ = resize_by_area( + np.zeros((304, 538, 3), np.uint8), budget, divisor=16 + ) + assert (image.shape[1], image.shape[0]) == (1280, 704) + assert (video_frame.shape[1], video_frame.shape[0]) == (1264, 720) + + # Generation grid: the image canvas through the VAE (//8) and the (2, 2) patch (//2); + # reference-video grid: the same for the video canvas, one frame fewer (no + # reference-image frame), as build_clip_conditioning derives it from the VAE latent. + full_clip_grid = generation_token_grid( + clip_len, image.shape[0] // 8, image.shape[1] // 8 + ) + reference_video_grid = ( + full_clip_grid[0] - 1, + video_frame.shape[0] // 8 // 2, + video_frame.shape[1] // 8 // 2, + ) + assert full_clip_grid == (22, 44, 80) + assert reference_video_grid == (21, 45, 79) + + layout = _InContextLayout(full_clip_grid, reference_video_grid, full_clip_grid) + assert layout.generation_video_tokens_per_frame == 3520 + assert layout.reference_video_tokens_per_frame == 3555 + assert layout.max_tokens_per_frame == 3555 + assert layout.max_tokens_per_frame >= layout.generation_video_tokens_per_frame + assert layout.max_tokens_per_frame >= layout.reference_video_tokens_per_frame + + # build_clip_conditioning: per-clip tensors from the request state with stand-in encoders. diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py index 0bef27689d..b13ad14f94 100644 --- a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py +++ b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py @@ -6,6 +6,7 @@ Single-process and CPU only, in the style of ``test_sp_shard.py`` / ``test_usp_r shape helpers are called directly. The parallelism rejections live in ``test_wan_animate_2_config.py``. """ +import math from collections import OrderedDict from contextlib import nullcontext from types import SimpleNamespace @@ -238,6 +239,93 @@ def test_attention_pads_tokens_into_the_slots_the_block_mask_admits(): assert not any(valid(qi, reference_start + ki) for qi in (0, 1) for ki in (0, 1)) +def test_attention_slots_fit_a_reference_frame_wider_than_the_generation_frame(): + # resize_by_area fits the reference image and the driving video to the pixel budget + # independently, so the reference video's grid can hold more tokens per frame than the + # generation grid (720x1280 with 16:9-ish inputs: 79x45 against 80x44). The slot must + # then take the reference frame's width, or the reference K/V copy raises "The expanded + # size of the tensor (4) must match the existing size (5)". + layout = _InContextLayout((2, 2, 2), (1, 1, 5), (2, 2, 2)) + captured = {} + + def fake_flex_attention(q, k, v, *, block_mask, score_mod): + captured.update(q=q, k=k, v=v) + return torch.zeros_like(q) + + batch, heads, dim = 1, 2, 8 + + def tokens(values, scale): + base = torch.tensor(values, dtype=torch.float32).view(1, len(values), 1, 1) + return (base * scale).expand(batch, len(values), heads, dim).clone() + + # Generation frames 0 (1..4) and 1 (5..8) of 4 tokens each; one 5-token reference frame. + gen_ids = [1, 2, 3, 4, 5, 6, 7, 8] + q, k, v = tokens(gen_ids, 1.0), tokens(gen_ids, 10.0), tokens(gen_ids, 100.0) + reference_k = tokens([1, 2, 3, 4, 5], 1000.0) + reference_v = tokens([1, 2, 3, 4, 5], 10000.0) + with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention): + out = WanAnimate2TransformerBlock._in_context_attention( + SimpleNamespace(log_scale=0.0), + q, + k, + v, + reference_k, + reference_v, + "SENTINEL_BLOCK_MASK", + layout, + ) + + assert tuple(out.shape) == (batch, 8, heads, dim) + q_padded = captured["q"].transpose(1, 2) + k_padded = captured["k"].transpose(1, 2) + v_padded = captured["v"].transpose(1, 2) + + def slot(t, i): + return t[0, i, 0, 0].item() + + # Generation tokens sit at the 5-token stride, slot 4 of each frame empty. + assert [slot(q_padded, i) for i in range(10)] == [1, 2, 3, 4, 0, 5, 6, 7, 8, 0] + assert [slot(k_padded, i) for i in range(10)] == [ + 10, + 20, + 30, + 40, + 0, + 50, + 60, + 70, + 80, + 0, + ] + assert q_padded[:, 10:].abs().sum() == 0 + # The five reference tokens fill their slot. + reference_start = layout.padded_generation_video_len + assert [slot(k_padded, reference_start + i) for i in range(5)] == [ + 1000, + 2000, + 3000, + 4000, + 5000, + ] + assert [slot(v_padded, reference_start + i) for i in range(5)] == [ + 10000, + 20000, + 30000, + 40000, + 50000, + ] + assert k_padded[:, 10:reference_start].abs().sum() == 0 + assert k_padded[:, reference_start + 5 :].abs().sum() == 0 + + # The mask routes frame-1 queries to all five reference slots and frame-0 queries to none. + valid = _mask_probe(layout.block_mask_fn()) + assert all(valid(qi, reference_start + ki) for qi in range(5, 9) for ki in range(5)) + assert not any( + valid(qi, reference_start + ki) for qi in range(4) for ki in range(5) + ) + assert not any(valid(qi, reference_start + 5) for qi in range(5, 9)) + + def test_in_context_attention_hands_flex_a_score_mod_only_when_log_scale_is_nonzero(): # At log_scale 0 the bias is an exact identity, so flex must run without a score_mod; # otherwise it must get the cached partial (compiled flex guards on its identity). @@ -572,6 +660,43 @@ def test_layout_pads_the_full_clip_geometry_to_128_token_blocks( ) == expected +@pytest.mark.parametrize( + "full_clip_grid, reference_video_grid, expected_tokens_per_frame", + [ + # 720x1280 budget, 1072x592 image and 538x304 video: the video's canvas (1264x720) + # is one latent row taller and one column narrower than the image's (1280x704), + # 79x45 = 3555 tokens against 80x44 = 3520; the slot takes the larger. + ((22, 44, 80), (21, 45, 79), 3555), + # Mirror case, the video's grid is the smaller: the generation grid sizes the slot. + ((22, 45, 79), (21, 44, 80), 3555), + # Equal grids: unchanged. + ((22, 44, 80), (21, 44, 80), 3520), + ], +) +def test_layout_slot_is_the_larger_of_the_two_grids_tokens_per_frame( + full_clip_grid, reference_video_grid, expected_tokens_per_frame +): + layout = _InContextLayout(full_clip_grid, reference_video_grid, full_clip_grid) + full_clip_f = full_clip_grid[0] + assert layout.max_tokens_per_frame == expected_tokens_per_frame + assert layout.max_tokens_per_frame >= layout.generation_video_tokens_per_frame + assert layout.max_tokens_per_frame >= layout.reference_video_tokens_per_frame + assert ( + layout.max_generation_video_num_tokens + == full_clip_f * expected_tokens_per_frame + ) + assert ( + layout.max_reference_video_num_tokens + == (full_clip_f - 1) * expected_tokens_per_frame + ) + assert layout.padded_generation_video_len == ( + math.ceil(layout.max_generation_video_num_tokens / 128) * 128 + ) + assert layout.padded_reference_video_len == ( + math.ceil(layout.max_reference_video_num_tokens / 128) * 128 + ) + + class _KvTaggingTransformer: """Fake DiT: ``build_reference_kv`` tags the K/V with the clip's reference-video latents, ``__call__`` records the (clip_cond, reference_kv) pair each forward was handed."""