[Feat] diffusion: support FastH3 Omni (Ref2VA 8-step PDD) (#45530)

Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
WenhaoZhang
2026-10-22 13:07:13 +08:00
committed by GitHub
co-authored by Mick
parent 3472e16f96
commit d9ce511646
14 changed files with 812 additions and 107 deletions
@@ -60,8 +60,12 @@ class MiniMaxH3PipelineConfig(PipelineConfig):
vae_precision: str = "fp32"
vae_decode_precision: str = "fp16"
audio_vae_precision: str = "fp32"
# Default VSA-H3 sparsity; --attention-backend-config VSA_sparsity overrides.
# Default VSA-H3 sparsity, tile size and reference-video keep rate;
# --attention-backend-config VSA_sparsity / vsa_tile_size /
# vsa_ref_keep_rate override them.
vsa_sparsity: float = 0.9
vsa_tile_size: int = 64
vsa_ref_keep_rate: float | None = None
text_encoder_configs: tuple[MiniMaxH3Qwen3VLConfig, ...] = field(
default_factory=lambda: (MiniMaxH3Qwen3VLConfig(),)
)
@@ -396,11 +400,41 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig):
)
__all__ = ["FastH3PipelineConfig", "MiniMaxH3PipelineConfig"]
@dataclass
class FastH3OmniPipelineConfig(FastH3PipelineConfig):
"""FastH3 Omni 8-Step V1: PDD + VSA-distilled MiniMax-H3, ref2va only."""
# The checkpoint's trained VSA (fastvideo_inference.json).
vsa_sparsity: float = 0.9
vsa_tile_size: int = 128
vsa_ref_keep_rate: float | None = 0.1
def __post_init__(self) -> None:
self.dit_config.arch_config.has_gate_compress = True
# Stack tiling batches every VAE tile in one forward; FastH3 only
# decodes, but Omni also encodes reference videos, which runs out of
# memory stacked.
def validate_server_args(self, server_args) -> None:
if server_args.model_variant is not None:
raise ValueError(
"FastH3 Omni ships one ref2va-distilled weight partition; "
"--model-variant does not apply."
)
self._default_to_trained_sparse_attention(server_args)
MiniMaxH3PipelineConfig.validate_server_args(self, server_args)
__all__ = [
"FastH3OmniPipelineConfig",
"FastH3PipelineConfig",
"MiniMaxH3PipelineConfig",
]
def register():
from sglang.multimodal_gen.configs.sample.minimax_h3 import (
FastH3OmniSamplingParams,
FastH3SamplingParams,
MiniMaxH3SamplingParams,
)
@@ -429,6 +463,19 @@ def register():
model_detectors=[
lambda model_id: (
"fasth3" in model_id.lower().replace("-", "").replace("_", "")
and "omni" not in model_id.lower()
)
],
)
register_configs(
sampling_param_cls=FastH3OmniSamplingParams,
pipeline_config_cls=FastH3OmniPipelineConfig,
hf_model_paths=[
"FastVideo/FastVideo-FastH3-Omni-8-Step-V1",
],
model_detectors=[
lambda model_id: (
"fasth3omni" in model_id.lower().replace("-", "").replace("_", "")
)
],
)
@@ -29,6 +29,9 @@ def _optional_unit_float(value: Any, field_name: str) -> float | None:
@dataclass
class MiniMaxH3SamplingParams(SamplingParams):
min_num_inference_steps: ClassVar[int] = 1
# Single-partition releases name their partition; None infers it from
# --model-variant or the model path.
release_partition: ClassVar[str | None] = None
height: int = 512
width: int = 896
@@ -110,7 +113,9 @@ class MiniMaxH3SamplingParams(SamplingParams):
file delivery, while the public H3 contract correctly requires it.
"""
selected_variant = getattr(server_args, "model_variant", None)
if selected_variant is not None:
if self.release_partition is not None:
selected_partition = self.release_partition
elif selected_variant is not None:
selected_partition = str(selected_variant).strip().lower()
else:
selected_path = server_args.model_subfolder or server_args.model_path
@@ -378,4 +383,33 @@ class FastH3SamplingParams(MiniMaxH3SamplingParams):
)
__all__ = ["FastH3SamplingParams", "MiniMaxH3SamplingParams"]
@dataclass
class FastH3OmniSamplingParams(MiniMaxH3SamplingParams):
"""FastH3 Omni 8-Step V1: eight PDD forwards over ordered references."""
release_partition: ClassVar[str | None] = "ref2va"
num_inference_steps: int = 8
task: str | None = "ref2va"
quality: str = "lossless"
# Trained with target-only flow: references stay clean.
imgvid_cond_noise_aug_for_inference: float | None = 1.0
audio_cond_noise_aug_for_inference: float | None = 1.0
def _validate(self) -> None:
super()._validate()
if self.num_inference_steps != 8:
raise ValueError(
"FastH3 Omni is distilled for exactly eight PDD forwards; got "
f"num_inference_steps={self.num_inference_steps}."
)
if self.task is not None and self.task.strip().lower() != "ref2va":
raise ValueError(
f"FastH3 Omni is distilled for ref2va only (got task={self.task!r})."
)
__all__ = [
"FastH3OmniSamplingParams",
"FastH3SamplingParams",
"MiniMaxH3SamplingParams",
]
+1
View File
@@ -189,6 +189,7 @@ KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
"minimaxai/minimax-h3": "MiniMaxH3Pipeline",
"minimax/minimax-h3": "MiniMaxH3Pipeline",
"fastvideo/fastvideo-fasth3-8-step-v2": "FastH3Pipeline",
"fastvideo/fastvideo-fasth3-omni-8-step-v1": "FastH3Pipeline",
"openvdn/vdn-minimax-h3": "VDNH3Pipeline",
"lerobot/pi05": "Pi05Pipeline",
"pi05": "Pi05Pipeline",
@@ -66,6 +66,7 @@ def construct_variable_block_sizes(
dit_seq_shape: tuple[int, int, int],
num_tiles: tuple[int, int, int],
device: torch.device,
tile_size: tuple[int, int, int] = VSA_TILE_SIZE,
) -> torch.LongTensor:
"""
Compute the number of valid (non‑padded) tokens inside every
@@ -78,7 +79,7 @@ def construct_variable_block_sizes(
"""
# unpack
t, h, w = dit_seq_shape
ts_t, ts_h, ts_w = VSA_TILE_SIZE
ts_t, ts_h, ts_w = tile_size
n_t, n_h, n_w = num_tiles
def _sizes(dim_len: int, tile: int, n_tiles: int) -> torch.LongTensor:
@@ -19,6 +19,14 @@ under a FLOP-matched budget ("compete"). The compression branch is gated by
``to_gate_compress``: base H3 has no such weights and the gate loads as
zeros (pure sparse); VSA-distilled students (FastH3) ship trained gates
that activate the branch.
Ref2VA interleaves reference videos with dense rows. Each reference video is
its own 3-D tiled region; every video query keeps its own top-k of EACH
region (references at ``ref_keep_rate``, the generated video at the step
sparsity), FastVideo's "p2_multi_region" policy. Checkpoints trained on
128-token (4, 4, 8) tiles select and compress at that logical granularity;
each logical tile runs as two contiguous 64-token kernel tiles, so the
selected key set is unchanged.
"""
import functools
@@ -50,83 +58,121 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels impo
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
# The FastH3 checkpoints are trained and served at the 64-token (4, 4, 4)
# tile geometry; the kernel block size matches it exactly.
# The kernels run 64-token blocks. FastH3 is trained on (4, 4, 4) tiles;
# FastH3 Omni on 128-token (4, 4, 8) tiles, each served as two kernel blocks.
VSA_H3_TILE_SHAPE = (4, 4, 4)
VSA_H3_TILE_ELEMS = math.prod(VSA_H3_TILE_SHAPE)
VSA_H3_TILE_SHAPES = {64: VSA_H3_TILE_SHAPE, 128: (4, 4, 8)}
_DIT_BLOCK_PREFIX = re.compile(r"^blocks\.(\d+)\.")
PackedSegment = int | tuple[int, int, int]
@functools.lru_cache(maxsize=8)
def _h3_segment_tile_geometry(
segments: tuple[PackedSegment, ...],
tile_shape: tuple[int, int, int],
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, tuple[tuple[int, int], ...]]:
"""Tile a packed sequence of interleaved dense segments and video regions.
``segments`` lists the packed order: an ``int`` is a dense segment's row
count, a ``(t, h, w)`` triple one video region's token grid. Dense
segments are chunked into the leading prefix tiles (never straddling a
segment boundary); each region follows as a contiguous run of 3-D tiles.
Returns, at kernel (64-token block) granularity, (variable_block_sizes
int32 [n_blocks], pack_index int32 [S_pad]: padded position -> packed row
or -1, unpack_index int32 [used]: packed row -> padded position), then
num_prefix_tiles and the per-region [start, end) spans in logical tiles.
A 128-token logical tile is two consecutive kernel blocks; its upper half
is empty when the tile holds at most 64 rows.
"""
tile_elems = math.prod(tile_shape)
split = tile_elems // VSA_H3_TILE_ELEMS
prefix_sizes: list[int] = []
prefix_rows: list[torch.Tensor] = []
video_entries: list[tuple[int, tuple[int, int, int]]] = []
cursor = 0
for segment in segments:
if isinstance(segment, tuple):
video_entries.append((cursor, segment))
cursor += math.prod(segment)
continue
full, remainder = divmod(segment, tile_elems)
prefix_sizes.extend([tile_elems] * full)
if remainder:
prefix_sizes.append(remainder)
prefix_rows.append(
torch.arange(cursor, cursor + segment, device=device, dtype=torch.long)
)
cursor += segment
num_prefix_tiles = len(prefix_sizes)
video_rows: list[torch.Tensor] = []
size_parts = [torch.tensor(prefix_sizes, dtype=torch.long, device=device)]
spans: list[tuple[int, int]] = []
tile_cursor = num_prefix_tiles
for start, grid in video_entries:
num_tiles = tuple(math.ceil(n / t) for n, t in zip(grid, tile_shape))
sizes = construct_variable_block_sizes(grid, num_tiles, device, tile_shape)
video_rows.append(get_tile_partition_indices(grid, tile_shape, device) + start)
size_parts.append(sizes.to(torch.long))
spans.append((tile_cursor, tile_cursor + int(sizes.numel())))
tile_cursor += int(sizes.numel())
tile_partition_indices = torch.cat(prefix_rows + video_rows)
logical_sizes = torch.cat(size_parts)
non_pad_index = get_non_pad_index(logical_sizes, tile_elems)
if (
int(logical_sizes.sum()) != cursor
or non_pad_index.numel() != cursor
or tile_partition_indices.numel() != cursor
):
raise ValueError(
f"VSA-H3 tile geometry mismatch for segments={segments}, "
f"tile={tile_shape}: sizes sum {int(logical_sizes.sum())}, non-pad "
f"{non_pad_index.numel()}, expected {cursor}."
)
# A logical tile's padded slots are its kernel blocks' slots in order,
# so the pack/unpack maps are shared; only the per-block sizes split.
seq_pad = logical_sizes.numel() * tile_elems
pack_index = torch.full((seq_pad,), -1, dtype=torch.int32, device=device)
pack_index[non_pad_index] = tile_partition_indices.to(torch.int32)
unpack_index = torch.empty(cursor, dtype=torch.int32, device=device)
unpack_index[tile_partition_indices] = non_pad_index.to(torch.int32)
offsets = torch.arange(split, device=device) * VSA_H3_TILE_ELEMS
block_sizes = (
(logical_sizes[:, None] - offsets).clamp(0, VSA_H3_TILE_ELEMS).flatten()
)
return (
block_sizes.to(torch.int32),
pack_index,
unpack_index,
num_prefix_tiles,
tuple(spans),
)
def _h3_tile_geometry(
prefix_segments: tuple[int, ...],
dit_seq_shape: tuple[int, int, int],
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
"""Segment-pure prefix chunks, then video tiles.
"""Segment-pure prefix chunks, then the generated video's 64-token tiles.
Returns (variable_block_sizes int32 [n_tiles], pack_index int32 [S_pad]:
padded position -> packed row or -1, unpack_index int32 [used]: packed row
-> padded position, num_prefix_tiles, num_video_tiles).
Returns (variable_block_sizes, pack_index, unpack_index,
num_prefix_tiles, num_video_tiles).
"""
prefix_len = sum(prefix_segments)
prefix_sizes: list[int] = []
for segment in prefix_segments:
full, remainder = divmod(segment, VSA_H3_TILE_ELEMS)
prefix_sizes.extend([VSA_H3_TILE_ELEMS] * full)
if remainder:
prefix_sizes.append(remainder)
num_prefix_tiles = len(prefix_sizes)
num_tiles = (
math.ceil(dit_seq_shape[0] / VSA_H3_TILE_SHAPE[0]),
math.ceil(dit_seq_shape[1] / VSA_H3_TILE_SHAPE[1]),
math.ceil(dit_seq_shape[2] / VSA_H3_TILE_SHAPE[2]),
)
video_sizes = construct_variable_block_sizes(dit_seq_shape, num_tiles, device)
num_video_tiles = int(video_sizes.numel())
video_indices = (
get_tile_partition_indices(dit_seq_shape, VSA_H3_TILE_SHAPE, device)
+ prefix_len
)
tile_partition_indices = torch.cat(
[
torch.arange(prefix_len, device=device, dtype=torch.long),
video_indices,
]
)
variable_block_sizes = torch.cat(
[
torch.tensor(prefix_sizes, dtype=torch.long, device=device),
video_sizes.to(torch.long),
]
)
non_pad_index = get_non_pad_index(variable_block_sizes, VSA_H3_TILE_ELEMS)
total = prefix_len + math.prod(dit_seq_shape)
sizes_sum = int(variable_block_sizes.sum())
if sizes_sum != total or non_pad_index.numel() != total:
raise ValueError(
f"VSA-H3 tile geometry mismatch for prefix={prefix_segments}, "
f"video={dit_seq_shape}: sizes sum {sizes_sum}, non-pad "
f"{non_pad_index.numel()}, expected {total}."
sizes, pack_index, unpack_index, num_prefix_tiles, spans = (
_h3_segment_tile_geometry(
(*prefix_segments, tuple(dit_seq_shape)), VSA_H3_TILE_SHAPE, device
)
seq_pad = variable_block_sizes.numel() * VSA_H3_TILE_ELEMS
pack_index = torch.full((seq_pad,), -1, dtype=torch.int32, device=device)
pack_index[non_pad_index] = tile_partition_indices.to(torch.int32)
unpack_index = torch.empty(total, dtype=torch.int32, device=device)
unpack_index[tile_partition_indices] = non_pad_index.to(torch.int32)
return (
variable_block_sizes.to(torch.int32),
pack_index,
unpack_index,
num_prefix_tiles,
num_video_tiles,
)
return sizes, pack_index, unpack_index, num_prefix_tiles, spans[0][1] - spans[0][0]
class VideoSparseAttentionH3Backend(AttentionBackend):
@@ -160,16 +206,32 @@ class VideoSparseAttentionH3Metadata(AttentionMetadata):
num_prefix_tiles: int
num_video_tiles: int
exempt: bool
# Kernel (64-token block) granularity; a 128-token logical tile is two
# consecutive blocks. Tile counts above are logical.
variable_block_sizes: torch.Tensor
pack_index: torch.Tensor
unpack_index: torch.Tensor
dense_layers: tuple[int, ...] = ()
workspace_cache: dict = field(default_factory=dict)
tile_elems: int = VSA_H3_TILE_ELEMS
# Logical-tile [start, end) per video region, the generated video last,
# with each region's sparsity. None: the generated video alone, at
# VSA_sparsity.
video_tile_spans: tuple[tuple[int, int], ...] | None = None
span_sparsities: tuple[float, ...] | None = None
@property
def num_tiles(self) -> int:
return self.num_prefix_tiles + self.num_video_tiles
@property
def blocks_per_tile(self) -> int:
return self.tile_elems // VSA_H3_TILE_ELEMS
@property
def uses_regions(self) -> bool:
return self.video_tile_spans is not None
class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder):
def __init__(self) -> None:
@@ -186,14 +248,50 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder):
exempt: bool = True,
dense_layers: tuple[int, ...] = (),
dense_first_n_steps: int = 0,
packed_segments: tuple[PackedSegment, ...] | None = None,
tile_size: int = VSA_H3_TILE_ELEMS,
ref_keep_rate: float | None = None,
**kwargs: dict[str, Any],
) -> VideoSparseAttentionH3Metadata:
"""``packed_segments`` replaces ``prefix_segments`` for layouts with
reference videos: the packed order of dense row counts and video
token grids, ending with the generated video's grid. Every earlier
grid is a reference region kept at ``ref_keep_rate``."""
dit_seq_shape = (
raw_latent_shape[0] // patch_size[0],
raw_latent_shape[1] // patch_size[1],
raw_latent_shape[2] // patch_size[2],
)
prefix_segments = tuple(int(s) for s in prefix_segments if s > 0)
tile_shape = VSA_H3_TILE_SHAPES.get(int(tile_size))
if tile_shape is None:
raise ValueError(
f"VSA-H3 tile_size must be one of {sorted(VSA_H3_TILE_SHAPES)}, "
f"got {tile_size!r}"
)
if packed_segments is None:
packed_segments = (*prefix_segments, dit_seq_shape)
segments = tuple(
tuple(int(v) for v in s) if isinstance(s, tuple) else int(s)
for s in packed_segments
if isinstance(s, tuple) or s > 0
)
if not segments or segments[-1] != dit_seq_shape:
raise ValueError(
f"VSA-H3 packed segments must end with the generated video "
f"{dit_seq_shape}, got {segments}"
)
num_references = sum(isinstance(s, tuple) for s in segments) - 1
if num_references:
if ref_keep_rate is None or not 0.0 < float(ref_keep_rate) <= 1.0:
raise ValueError(
f"VSA-H3 with {num_references} reference-video region(s) "
f"needs vsa_ref_keep_rate in (0, 1], got {ref_keep_rate!r}"
)
if not exempt:
raise ValueError(
"vsa_mode='compete' supports only the generated-video "
"region; use vsa_mode='exempt' with reference videos"
)
if current_timestep < dense_first_n_steps:
VSA_sparsity = 0.0
@@ -202,21 +300,31 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder):
pack_index,
unpack_index,
num_prefix_tiles,
num_video_tiles,
) = _h3_tile_geometry(prefix_segments, dit_seq_shape, device)
spans,
) = _h3_segment_tile_geometry(segments, tile_shape, device)
regional = num_references > 0 or tile_shape != VSA_H3_TILE_SHAPE
span_sparsities = None
if regional:
span_sparsities = (1.0 - float(ref_keep_rate or 1.0),) * num_references
span_sparsities += (float(VSA_sparsity),)
return VideoSparseAttentionH3Metadata(
current_timestep=current_timestep,
VSA_sparsity=float(VSA_sparsity),
total_seq_length=sum(prefix_segments) + math.prod(dit_seq_shape),
total_seq_length=sum(
math.prod(s) if isinstance(s, tuple) else s for s in segments
),
num_prefix_tiles=num_prefix_tiles,
num_video_tiles=num_video_tiles,
num_video_tiles=spans[-1][1] - num_prefix_tiles,
exempt=exempt,
variable_block_sizes=variable_block_sizes,
pack_index=pack_index,
unpack_index=unpack_index,
dense_layers=tuple(int(layer) for layer in dense_layers),
workspace_cache=self._workspace_cache,
tile_elems=math.prod(tile_shape),
video_tile_spans=spans if regional else None,
span_sparsities=span_sparsities,
)
@@ -225,6 +333,50 @@ def _compute_topk(sparsity: float, num_video_tiles: int) -> int:
return max(1, min(keep, num_video_tiles))
def _region_tile_lists(
scores: torch.Tensor,
num_prefix_tiles: int,
spans: tuple[tuple[int, int], ...],
sparsities: tuple[float, ...],
) -> torch.Tensor:
"""scores [H, n_tiles, n_tiles] -> ascending int32 logical kv-tile lists
[H, num_video_tiles, width]: every prefix tile, then each region's own
top-k (the whole region when its budget covers it)."""
prefix = num_prefix_tiles
video_rows = scores[:, prefix:, :]
parts = [
torch.arange(prefix, device=scores.device).expand(
*video_rows.shape[:-1], prefix
)
]
for (start, end), sparsity in zip(spans, sparsities):
keep = _compute_topk(sparsity, end - start)
if keep == end - start:
parts.append(
torch.arange(start, end, device=scores.device).expand(
*video_rows.shape[:-1], keep
)
)
continue
picked = video_rows[..., start:end].topk(keep, dim=-1).indices
parts.append(picked.sort(dim=-1).values + start)
return torch.cat(parts, dim=-1).to(torch.int32)
def _pool_logical_tiles(
pooled: torch.Tensor, meta: VideoSparseAttentionH3Metadata
) -> torch.Tensor:
"""Kernel-block means [3, H, n_blocks, D] -> logical-tile means."""
split = meta.blocks_per_tile
if split == 1:
return pooled
sizes = meta.variable_block_sizes.view(-1, split).to(torch.float32)
blocks = pooled.unflatten(2, (-1, split))
# Empty upper halves pool to 0/0; they carry no weight.
weighted = torch.where(sizes[..., None] > 0, blocks * sizes[..., None], 0.0)
return weighted.sum(3) / sizes.sum(-1)[:, None]
def _topk_tile_lists(
scores: torch.Tensor,
num_prefix_tiles: int,
@@ -266,6 +418,8 @@ def _workspace_key(
meta.num_tiles,
meta.num_prefix_tiles,
meta.exempt,
meta.tile_elems,
meta.uses_regions,
heads,
head_dim,
has_gate,
@@ -276,10 +430,11 @@ def _workspace_key(
class _Workspace:
"""Per-geometry scratch: tiled q/k/v(/gate) [3|4, H, S_pad, D], pooled fp32
tile means [3, H, n_tiles, D], and the kernel index lists (prefix rows and
prefix columns are static; only the top-k video columns change per layer).
The native kernel pairs query tiles, so the tile axis is padded to an even
count with one empty tile.
block means [3, H, n_blocks, D], and the kernel index lists (prefix rows
and prefix columns are static; only the top-k video columns change per
layer). All of it is at kernel-block granularity. The native kernel pairs
query blocks, so the block axis is padded to an even count with one empty
block.
"""
def __init__(
@@ -291,7 +446,7 @@ class _Workspace:
dtype: torch.dtype,
device: torch.device,
) -> None:
n_tiles = meta.num_tiles
n_tiles = int(meta.variable_block_sizes.numel())
self.n_tiles = n_tiles
self.use_sm100_kernel = can_use_vsa_block_sparse_sm100(
device.index, dtype, head_dim
@@ -321,7 +476,7 @@ class _Workspace:
self.dense_num[:, :n_tiles] = n_tiles
self.q2k_index = self.dense_index.clone()
self.q2k_num = self.dense_num.clone()
num_video_tiles = meta.num_video_tiles
num_video_tiles = 0 if meta.uses_regions else meta.num_video_tiles
# Zeroed scratch for the exempt top-k compaction; kept zero between uses.
self.topk_mask = torch.zeros(
(heads, num_video_tiles, num_video_tiles), dtype=torch.int8, device=device
@@ -339,7 +494,7 @@ class _Workspace:
def _gate_tile_index(
meta: VideoSparseAttentionH3Metadata, row_start: int, num_rows: int
) -> torch.Tensor:
"""Tile of each packed row in the shard, -1 for pad rows."""
"""Kernel block of each packed row in the shard, -1 for pad rows."""
key = ("gate_tile_index", row_start, num_rows)
tile_index = meta.workspace_cache.get(key)
if tile_index is None:
@@ -390,6 +545,8 @@ def _select_kv_lists(
sparsity: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Kernel index lists for this layer: dense, or top-k video columns."""
if meta.uses_regions:
return _select_region_kv_lists(ws, meta, scores, sparsity)
keep = _compute_topk(sparsity, meta.num_video_tiles)
if sparsity <= 0.0 or keep >= meta.num_video_tiles:
return ws.dense_index, ws.dense_num
@@ -414,6 +571,36 @@ def _select_kv_lists(
)
def _select_region_kv_lists(
ws: _Workspace,
meta: VideoSparseAttentionH3Metadata,
scores: torch.Tensor | None,
sparsity: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-region top-k over logical tiles, expanded to kernel blocks."""
spans = meta.video_tile_spans
sparsities = meta.span_sparsities
# Dense steps and layers keep every region dense.
if sparsity <= 0.0:
return ws.dense_index, ws.dense_num
if all(
_compute_topk(s, end - start) == end - start
for (start, end), s in zip(spans, sparsities)
):
return ws.dense_index, ws.dense_num
lists = _region_tile_lists(scores, meta.num_prefix_tiles, spans, sparsities)
split = meta.blocks_per_tile
if split > 1:
offsets = torch.arange(split, device=lists.device, dtype=lists.dtype)
lists = (lists[..., None] * split + offsets).flatten(-2)
lists = lists.repeat_interleave(split, dim=1)
prefix_blocks = meta.num_prefix_tiles * split
width = lists.shape[-1]
ws.q2k_index[:, prefix_blocks : ws.n_tiles, :width] = lists
ws.q2k_num[:, prefix_blocks : ws.n_tiles] = width
return ws.q2k_index, ws.q2k_num
class VideoSparseAttentionH3Impl(AttentionImpl):
def __init__(
self,
@@ -524,7 +711,7 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
ws.tiled,
ws.pooled,
)
q_pooled, k_pooled, v_pooled = ws.pooled
q_pooled, k_pooled, v_pooled = _pool_logical_tiles(ws.pooled, meta)
scores = None
if sparsity > 0.0 or needs_compress:
@@ -547,6 +734,12 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
out_compress = None
if needs_compress:
out_compress = torch.matmul(torch.softmax(scores, dim=-1), v_pooled)
if meta.blocks_per_tile > 1:
# The gate folds per kernel block; both halves of a logical
# tile share its compressed row.
out_compress = out_compress.repeat_interleave(
meta.blocks_per_tile, dim=1
)
result = torch.empty(query.shape, dtype=query.dtype, device=query.device)
vsa_h3_untile(
@@ -1,9 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import os
import shutil
from dataclasses import replace
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import (
MiniMaxH3PipelineConfig,
)
@@ -140,6 +142,25 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
f"disaggregation role {role.value!r} is not supported"
)
def initialize_pipeline(self, server_args: ServerArgs) -> None:
self.pdd_heads = self._resolve_pdd_heads()
transformer = self.get_module("transformer")
if (
self.pdd_heads
and transformer is not None
and transformer.final_layer._pdd_heads is None
):
transformer.final_layer.load_pdd_fused_heads(self.pdd_heads)
def _resolve_pdd_heads(self) -> str | None:
"""The env override, else a PDD release's own fused heads."""
if envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS:
return envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
release_metadata = getattr(self, "release_metadata", None)
if release_metadata is None or release_metadata.pdd_fused_heads is None:
return None
return os.path.join(self.model_path, release_metadata.pdd_fused_heads)
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
# Per-model sigma override from model_index.json; contract tests
# construct the pipeline without model_path, hence the guard.
@@ -180,6 +201,7 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
if release_metadata is not None
else None
),
pdd_heads=getattr(self, "pdd_heads", None),
)
)
self.add_stage(
@@ -206,7 +228,8 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
class FastH3Pipeline(MiniMaxH3Pipeline):
"""FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only).
"""FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only), and the
FastH3 Omni Ref2VA PDD student.
The flat single-partition repo is materialized into the base-H3 layout by
its registered model overlay (see runtime/utils/model_overlay.py), so every stage, loader,
@@ -4,6 +4,7 @@
from __future__ import annotations
import math
import os
from dataclasses import dataclass
from typing import Any, Mapping
@@ -54,6 +55,9 @@ class MiniMaxH3ReleaseMetadata:
audio_sigma_shift: float
# trained DMD rungs of a distilled release, unshifted; None keeps the uniform grid
dmd_denoising_steps: tuple[int, ...] | None = None
# PDD release: fused per-step output heads, relative to the model dir,
# with pdd_config.json beside them
pdd_fused_heads: str | None = None
@classmethod
def from_model_index(
@@ -109,6 +113,17 @@ class MiniMaxH3ReleaseMetadata:
"strictly decreasing integers in (0, 1000]"
)
dmd_steps = tuple(dmd_steps)
pdd_fused_heads = raw.get("pdd_fused_heads")
if pdd_fused_heads is not None and (
not isinstance(pdd_fused_heads, str)
or not pdd_fused_heads.endswith(".safetensors")
or os.path.isabs(pdd_fused_heads)
or ".." in pdd_fused_heads.split("/")
):
raise ValueError(
"model_index.json._minimax_h3.pdd_fused_heads must be a relative "
".safetensors path inside the model directory"
)
metadata = cls(
schema_version=1,
partition=partition,
@@ -117,6 +132,7 @@ class MiniMaxH3ReleaseMetadata:
video_sigma_shift=video_sigma,
audio_sigma_shift=audio_sigma,
dmd_denoising_steps=dmd_steps,
pdd_fused_heads=pdd_fused_heads,
)
for task in metadata.tasks:
if canonical_minimax_h3_task(task) != task:
@@ -552,6 +552,7 @@ def comfyui_payload_to_branch_inputs(
"orig_video_shape": orig_shape,
"padded_video_shape": tuple(video.shape),
"is_ref2va": bool(refs),
"native_layout": layout is None,
"sample_sigmas": sample_sigmas,
"sigma_v": float(sigma_v),
"shift_v": shift_v,
@@ -669,6 +670,7 @@ def _step_attn_metadata(
latent_shape=latent_shape,
server_args=server_args,
device=device,
native_layout=inputs["native_layout"],
)
if build is not None:
return build
@@ -5,6 +5,7 @@ single-branch execution, and payload validation.
from __future__ import annotations
import math
from collections.abc import Callable, Mapping
from contextlib import contextmanager
from functools import partial
@@ -778,6 +779,10 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
device=device,
)
initial_video, initial_audio = _expand_initial_rows(ctx, positive)
if model.final_layer._pdd_heads is not None:
initial_video, initial_audio = _scale_pdd_initial_noise(
ctx, positive, initial_video, initial_audio
)
rollout_ctx = None
if getattr(batch, "rollout", False):
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.minimax_h3_rollout import (
@@ -1049,30 +1054,31 @@ def _maybe_prepare_vsa_h3_step_metadata(
latent_shape: tuple[int, int, int],
server_args: ServerArgs,
device: torch.device,
native_layout: bool = True,
) -> Callable[[int], Any] | None:
"""Per-step VSA-H3 metadata builder over the request-static packed layout,
or None off the VSA path."""
or None off the VSA path.
Ref2VA regions are read from the native packed layout's stream_layout;
``native_layout=False`` marks a layout serialized by ComfyUI, whose
stream_layout does not tell reference images from reference videos."""
model._resolve_attention_backend_once()
if (
model._resolved_attention_backend
is not AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3
):
return None
if is_ref2va:
if is_ref2va and not native_layout:
raise NotImplementedError(
"VSA-H3 supports the t2va/fl2va packed layout; the ref2va "
"reference-block layout is not tiled yet. Use --attention-backend "
"fa for ref2va."
"VSA-H3 tiles Ref2VA references on SGLang's native packed layout; "
"a ComfyUI-serialized Ref2VA layout is not tiled. Use "
"--attention-backend fa for it."
)
config = server_args.attention_backend_config or {}
tile_size = int(config.get("vsa_tile_size", 64))
if tile_size != 64:
raise ValueError(
"VSA-H3 in SGLang serves the trained 64-token (4, 4, 4) tile "
f"geometry; got vsa_tile_size={tile_size}."
)
default_sparsity = server_args.pipeline_config.vsa_sparsity
pipeline_config = server_args.pipeline_config
tile_size = int(config.get("vsa_tile_size", pipeline_config.vsa_tile_size))
ref_keep_rate = config.get("vsa_ref_keep_rate", pipeline_config.vsa_ref_keep_rate)
default_sparsity = pipeline_config.vsa_sparsity
sparsity = float(
config.get("VSA_sparsity", config.get("sparsity", default_sparsity))
)
@@ -1089,6 +1095,9 @@ def _maybe_prepare_vsa_h3_step_metadata(
cond_rows = int(packed["img_pos"].numel()) - video_rows
audio_rows = int(packed["audio_pos"].numel())
patch_size = server_args.pipeline_config.dit_config.arch_config.patch_size
packed_segments = (
_ref2va_vsa_segments(packed, latent_shape, patch_size) if is_ref2va else None
)
from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import (
VideoSparseAttentionH3MetadataBuilder,
@@ -1107,11 +1116,44 @@ def _maybe_prepare_vsa_h3_step_metadata(
exempt=mode == "exempt",
dense_layers=dense_layers,
dense_first_n_steps=dense_first_n_steps,
packed_segments=packed_segments,
tile_size=tile_size,
ref_keep_rate=None if ref_keep_rate is None else float(ref_keep_rate),
)
return build
def _ref2va_vsa_segments(
packed: Mapping[str, Any],
latent_shape: tuple[int, int, int],
patch_size: tuple[int, int, int],
) -> tuple[int | tuple[int, int, int], ...]:
"""Ref2VA packed order as VSA-H3 segments: text, keyframes, then each
reference (images and audio dense, videos as their own token-grid
regions), target audio, and the generated video's token grid last."""
layout = packed["stream_layout"]
shapes = layout["cond_image_shapes"]
roles = layout["cond_image_roles"]
audio_lens = layout["cond_audio_stream_lens"]
reference_visual_rows = sum(math.prod(shape) for shape in shapes)
cond_rows = int(packed["img_pos"].numel()) - int(packed["update_mask"].sum())
segments: list[int | tuple[int, int, int]] = [
int(packed["text_pos"].numel()),
cond_rows - reference_visual_rows,
]
for kind, index in layout["cond_event_orders"]:
if kind == "audio":
segments.append(int(audio_lens[index]))
elif roles[index] == "independent_cube":
segments.append(tuple(int(v) for v in shapes[index]))
else:
segments.append(math.prod(shapes[index]))
segments.append(int(packed["audio_pos"].numel()) - sum(audio_lens))
segments.append(tuple(int(n) // int(p) for n, p in zip(latent_shape, patch_size)))
return tuple(segments)
def _build_packed_layout(
ctx: _FullLoopContext,
emb: Mapping[str, Any],
@@ -1267,6 +1309,22 @@ def _expand_initial_rows(
return initial_video, initial_audio
def _scale_pdd_initial_noise(
ctx: _FullLoopContext,
positive: Any,
initial_video: torch.Tensor,
initial_audio: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Start PDD targets at the first node's noise level, as the students are
trained (``sigma_0 * eps``); condition rows stay as they are. A no-op on
grids that start at sigma 1."""
initial_video = initial_video.clone()
initial_audio = initial_audio.clone()
initial_video[positive.video_target_slice] *= float(ctx.sigmas["video"][0])
initial_audio[positive.audio_target_slice] *= float(ctx.sigmas["audio"][0])
return initial_video, initial_audio
def _publish_full_loop_outputs(
ctx: _FullLoopContext,
*,
@@ -30,6 +30,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
self,
sigma_shift_scales=None,
dmd_denoising_steps: tuple[int, ...] | None = None,
pdd_heads: str | None = None,
) -> None:
super().__init__()
# Per-model sigma shift override (model_index.json "_minimax_h3" release
@@ -38,7 +39,9 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
self.sigma_shift_scales = sigma_shift_scales
self.dmd_denoising_steps = dmd_denoising_steps
self._pdd_config = None
pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
# The pipeline resolves the release's fused heads; the env var serves
# stages built without a pipeline.
pdd_heads = pdd_heads or envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
if pdd_heads:
from safetensors import safe_open
@@ -49,6 +52,9 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
steps = f.get_slice("video_out.weight").get_shape()[0]
if self._pdd_config["num_inference_steps"] != steps:
raise ValueError("MiniMax-H3 PDD config does not match the fused heads")
sigma_max = float(self._pdd_config.get("sigma_max", 1.0))
if not 0.0 < sigma_max <= 1.0:
raise ValueError("MiniMax-H3 PDD sigma_max must be in (0, 1]")
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.resolved_plan import (
@@ -95,6 +101,10 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
)
config = self._pdd_config
# Students trained on a grid ending at sigma_max < 1 (FastGen's
# max_t) sample the [1, 0] grid scaled by it. The scale is shared by
# every interval, so the fused heads' in-block weights are unchanged.
sigma_max = float(config.get("sigma_max", 1.0))
sigmas = batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY]
for modality in ("video", "audio"):
expected = minimax_h3_time_shift_sigmas(
@@ -104,18 +114,20 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
if batch.is_warmup:
# Warmup may run fewer steps, but must use the same intervals
# as serving rather than rescaling a shorter grid to [1, 0].
sigmas[modality] = expected[: batch.num_inference_steps + 1]
continue
actual = sigmas[modality]
if len(actual) != len(expected) or any(
not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7)
for a, b in zip(actual, expected)
):
raise ValueError(
f"MiniMax-H3 PDD requires num_inference_steps="
f"{config['num_inference_steps']} and {modality} shift="
f"{config[f'{modality}_shift']} to match the fused heads"
)
expected = expected[: batch.num_inference_steps + 1]
else:
actual = sigmas[modality]
if len(actual) != len(expected) or any(
not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7)
for a, b in zip(actual, expected)
):
raise ValueError(
f"MiniMax-H3 PDD requires num_inference_steps="
f"{config['num_inference_steps']} and {modality} shift="
f"{config[f'{modality}_shift']} to match the fused heads"
)
if batch.is_warmup or sigma_max != 1.0:
sigmas[modality] = [sigma * sigma_max for sigma in expected]
@staticmethod
def _publish_native_timestep_state(batch: Req) -> None:
@@ -53,6 +53,11 @@ BUILTIN_MODEL_OVERLAY_REGISTRY: dict[str, dict[str, Any]] = {
"overlay_repo_id": "kevin-mi/VDN-H3-overlay",
"overlay_revision": "7de18275dddfe59da36a234e222bcdd274963bc3",
},
"FastVideo/FastVideo-FastH3-Omni-8-Step-V1": {
"overlay_repo_id": "Niehen6174/FastH3-Omni-overlay",
"overlay_revision": "953ede55daf20a70cc6f0b59464293e2f408608c",
"source_revision": "53e0d26a1691aa473b6262d8196a8feb2c297df0",
},
}
@@ -353,7 +353,10 @@ def test_vsa_h3_tiles_cover_comfyui_packed_rows(keyframes):
server_args = SimpleNamespace(
attention_backend_config={},
pipeline_config=SimpleNamespace(
vsa_sparsity=0.8, dit_config=MiniMaxH3DiTConfig()
vsa_sparsity=0.8,
vsa_tile_size=64,
vsa_ref_keep_rate=None,
dit_config=MiniMaxH3DiTConfig(),
),
)
build = _maybe_prepare_vsa_h3_step_metadata(
@@ -374,6 +377,32 @@ def test_vsa_h3_tiles_cover_comfyui_packed_rows(keyframes):
assert torch.equal(packed["text_pos"], torch.arange(text_len))
def test_vsa_h3_rejects_comfyui_serialized_ref2va_layout():
"""ComfyUI-serialized Ref2VA layouts do not tell reference images from
reference videos, so VSA-H3 refuses them instead of mis-tiling."""
from types import SimpleNamespace
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
_maybe_prepare_vsa_h3_step_metadata,
)
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
model = SimpleNamespace(
_resolve_attention_backend_once=lambda: None,
_resolved_attention_backend=AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3,
)
with pytest.raises(NotImplementedError, match="ComfyUI-serialized Ref2VA"):
_maybe_prepare_vsa_h3_step_metadata(
model=model,
packed={},
is_ref2va=True,
latent_shape=(4, 8, 12),
server_args=SimpleNamespace(),
device=torch.device("cpu"),
native_layout=False,
)
def test_ref2va_image_plus_video_audio_used_prefix_matches_comfyui():
kwargs = dict(text_len=5, latent_t=2, latent_h=4, latent_w=4, audio_t=5)
refs = [
@@ -0,0 +1,227 @@
# SPDX-License-Identifier: Apache-2.0
"""VSA-H3 Ref2VA regions and 128-token logical tiles.
The oracle below is written at logical-tile granularity from FastVideo's
"p2_multi_region" mask semantics, independent of the kernel-block split the
backend serves 128-token tiles with.
"""
from __future__ import annotations
import math
import pytest
import torch
from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import (
VideoSparseAttentionH3Impl,
VideoSparseAttentionH3MetadataBuilder,
_compute_topk,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.packed_sequence import (
minimax_h3_packed_sequence_ref2va_blocks,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
_ref2va_vsa_segments,
)
requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="VSA-H3 kernels need CUDA"
)
HEADS = 4
HEAD_DIM = 128
TARGET = (5, 6, 10)
# text, image ref, reference video (ragged in every tile dimension), its
# audio, a standalone audio ref, target audio, generated video
SEGMENTS = (70, 30, 24, (3, 5, 9), 40, 50, TARGET)
def _build(sparsity: float, device, *, tile_size=128, ref_keep_rate=0.25, **kw):
return VideoSparseAttentionH3MetadataBuilder().build(
current_timestep=0,
raw_latent_shape=TARGET,
patch_size=(1, 1, 1),
VSA_sparsity=sparsity,
prefix_segments=(),
device=device,
packed_segments=SEGMENTS,
tile_size=tile_size,
ref_keep_rate=ref_keep_rate,
**kw,
)
def _used() -> int:
return sum(math.prod(s) if isinstance(s, tuple) else s for s in SEGMENTS)
def test_region_geometry_accounts_every_row() -> None:
meta = _build(0.9, torch.device("cpu"))
used = _used()
assert meta.total_seq_length == used
assert meta.blocks_per_tile == 2
assert int(meta.variable_block_sizes.sum()) == used
assert int((meta.pack_index >= 0).sum()) == used
assert sorted(meta.unpack_index.tolist()) == sorted(
(meta.pack_index >= 0).nonzero().flatten().tolist()
)
# Dense rows chunk per segment into 128-token tiles: 70, 30, 24, 40, 50.
assert meta.num_prefix_tiles == 5
logical = meta.variable_block_sizes.view(-1, 2).sum(-1)
assert logical[:5].tolist() == [70, 30, 24, 40, 50]
# Reference (3, 5, 9) and target (5, 6, 10) at (4, 4, 8) tiles.
reference_tiles = 1 * 2 * 2
target_tiles = 2 * 2 * 2
assert meta.video_tile_spans == (
(5, 5 + reference_tiles),
(5 + reference_tiles, 5 + reference_tiles + target_tiles),
)
assert meta.num_video_tiles == reference_tiles + target_tiles
assert meta.span_sparsities == (0.75, 0.9)
# Each kernel block holds at most 64 rows, the lower half first.
halves = meta.variable_block_sizes.view(-1, 2)
assert torch.all(halves[:, 0] == logical.clamp(max=64))
def test_reference_regions_need_keep_rate_and_exempt() -> None:
with pytest.raises(ValueError, match="vsa_ref_keep_rate"):
_build(0.9, torch.device("cpu"), ref_keep_rate=None)
with pytest.raises(ValueError, match="compete"):
_build(0.9, torch.device("cpu"), exempt=False)
def test_ref2va_segments_follow_packed_layout() -> None:
blocks = [
{"kind": "image", "latent_h": 8, "latent_w": 12},
{
"kind": "video_audio",
"ref_audio_t": 6,
"latent_t": 3,
"latent_h": 8,
"latent_w": 12,
},
{"kind": "audio", "ref_audio_t": 5},
]
latent = (4, 10, 16)
packed = minimax_h3_packed_sequence_ref2va_blocks(
text_len=33,
latent_t=latent[0],
latent_h=latent[1],
latent_w=latent[2],
audio_t=20,
ref_blocks=blocks,
include_video_pos=True,
)
segments = _ref2va_vsa_segments(packed, latent, (1, 2, 2))
assert segments == (33, 0, 24, 12, (3, 4, 6), 10, 40, (4, 5, 8))
# Region rows are exactly the layout's sparse-eligible video rows.
cursor, region_rows = 0, []
for segment in segments:
rows = math.prod(segment) if isinstance(segment, tuple) else segment
if isinstance(segment, tuple):
region_rows.append(torch.arange(cursor, cursor + rows))
cursor += rows
assert cursor == int(packed["cu_seqlens"][1])
assert torch.equal(torch.cat(region_rows), packed["video_pos"])
def _oracle(meta, used, q, k, v, gate):
"""Masked dense attention over logical tiles, plus the gated compression."""
split = meta.blocks_per_tile
tile = meta.tile_elems
logical = meta.variable_block_sizes.view(-1, split).sum(-1).float()
n = logical.numel()
valid = meta.pack_index >= 0
src = meta.pack_index.clamp(min=0).long()
def tiled(x):
t = x[src].float().permute(1, 0, 2)
return t * valid.to(t.dtype)[None, :, None]
qt, kt, vt, gt = tiled(q), tiled(k), tiled(v), tiled(gate)
pooled = [
t.view(HEADS, n, tile, HEAD_DIM).sum(2) / logical[None, :, None]
for t in (qt, kt, vt)
]
scores = pooled[0] @ pooled[1].transpose(-1, -2) / math.sqrt(HEAD_DIM)
prefix = meta.num_prefix_tiles
mask = torch.zeros(HEADS, n, n, dtype=torch.bool, device=q.device)
for (start, end), sparsity in zip(meta.video_tile_spans, meta.span_sparsities):
keep = _compute_topk(sparsity, end - start)
picked = scores[..., start:end].topk(keep, dim=-1).indices + start
mask.scatter_(-1, picked, True)
mask[..., :prefix] = True
mask[:, :prefix, :] = True
row_mask = mask.repeat_interleave(tile, 1).repeat_interleave(tile, 2)
row_mask &= valid[None, None, :]
logits = qt @ kt.transpose(-1, -2) / math.sqrt(HEAD_DIM)
out = torch.softmax(logits.masked_fill(~row_mask, float("-inf")), -1) @ vt
compress = torch.softmax(scores, dim=-1) @ pooled[2]
out = out + compress.repeat_interleave(tile, 1) * gt
return out[:, meta.unpack_index.long()].permute(1, 0, 2)
def _impl():
return VideoSparseAttentionH3Impl(
num_heads=HEADS,
head_size=HEAD_DIM,
causal=False,
softmax_scale=HEAD_DIM**-0.5,
num_kv_heads=HEADS,
prefix="blocks.3.attn",
)
def _qkvg(device, used, seed=5):
total = (used + 63) // 64 * 64
generator = torch.Generator(device="cpu").manual_seed(seed)
q, k, v, gate = (
torch.randn((total, HEADS, HEAD_DIM), generator=generator).to(
device=device, dtype=torch.bfloat16
)
for _ in range(4)
)
return total, q, k, v, gate * 0.1
@requires_cuda
@pytest.mark.parametrize("tile_size", [64, 128])
def test_regions_match_masked_dense_oracle(tile_size: int) -> None:
device = torch.device("cuda")
meta = _build(0.5, device, tile_size=tile_size)
used = _used()
total, q, k, v, gate = _qkvg(device, used)
cu = torch.tensor([0, used, total], dtype=torch.int32, device=device)
out = _impl().forward_varlen(
q,
k,
v,
cu_seqlens=cu,
max_seqlen=used,
cu_seqlens_host=(0, used, total),
attn_metadata=meta,
gate_compress=gate,
)
reference = _oracle(meta, used, q, k, v, gate)
diff = (out[:used].float() - reference).abs().max().item()
assert diff < 2e-2, f"tile {tile_size}: regions vs oracle max diff {diff}"
assert torch.all(out[used:] == 0)
# The deferred (Ulysses) gate fold sees the same per-block compression.
from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import (
vsa_h3_fold_gate,
)
unfolded, out_compress = _impl().forward_varlen(
q,
k,
v,
cu_seqlens=cu,
max_seqlen=used,
cu_seqlens_host=(0, used, total),
attn_metadata=meta,
return_compress=True,
)
vsa_h3_fold_gate(unfolded, gate, out_compress, meta, 0)
assert torch.equal(unfolded, out)