[Feat] diffusion: support FastH3 Omni (Ref2VA 8-step PDD) (#45530)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
@@ -60,8 +60,12 @@ class MiniMaxH3PipelineConfig(PipelineConfig):
|
||||
vae_precision: str = "fp32"
|
||||
vae_decode_precision: str = "fp16"
|
||||
audio_vae_precision: str = "fp32"
|
||||
# Default VSA-H3 sparsity; --attention-backend-config VSA_sparsity overrides.
|
||||
# Default VSA-H3 sparsity, tile size and reference-video keep rate;
|
||||
# --attention-backend-config VSA_sparsity / vsa_tile_size /
|
||||
# vsa_ref_keep_rate override them.
|
||||
vsa_sparsity: float = 0.9
|
||||
vsa_tile_size: int = 64
|
||||
vsa_ref_keep_rate: float | None = None
|
||||
text_encoder_configs: tuple[MiniMaxH3Qwen3VLConfig, ...] = field(
|
||||
default_factory=lambda: (MiniMaxH3Qwen3VLConfig(),)
|
||||
)
|
||||
@@ -396,11 +400,41 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig):
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["FastH3PipelineConfig", "MiniMaxH3PipelineConfig"]
|
||||
@dataclass
|
||||
class FastH3OmniPipelineConfig(FastH3PipelineConfig):
|
||||
"""FastH3 Omni 8-Step V1: PDD + VSA-distilled MiniMax-H3, ref2va only."""
|
||||
|
||||
# The checkpoint's trained VSA (fastvideo_inference.json).
|
||||
vsa_sparsity: float = 0.9
|
||||
vsa_tile_size: int = 128
|
||||
vsa_ref_keep_rate: float | None = 0.1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.dit_config.arch_config.has_gate_compress = True
|
||||
# Stack tiling batches every VAE tile in one forward; FastH3 only
|
||||
# decodes, but Omni also encodes reference videos, which runs out of
|
||||
# memory stacked.
|
||||
|
||||
def validate_server_args(self, server_args) -> None:
|
||||
if server_args.model_variant is not None:
|
||||
raise ValueError(
|
||||
"FastH3 Omni ships one ref2va-distilled weight partition; "
|
||||
"--model-variant does not apply."
|
||||
)
|
||||
self._default_to_trained_sparse_attention(server_args)
|
||||
MiniMaxH3PipelineConfig.validate_server_args(self, server_args)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FastH3OmniPipelineConfig",
|
||||
"FastH3PipelineConfig",
|
||||
"MiniMaxH3PipelineConfig",
|
||||
]
|
||||
|
||||
|
||||
def register():
|
||||
from sglang.multimodal_gen.configs.sample.minimax_h3 import (
|
||||
FastH3OmniSamplingParams,
|
||||
FastH3SamplingParams,
|
||||
MiniMaxH3SamplingParams,
|
||||
)
|
||||
@@ -429,6 +463,19 @@ def register():
|
||||
model_detectors=[
|
||||
lambda model_id: (
|
||||
"fasth3" in model_id.lower().replace("-", "").replace("_", "")
|
||||
and "omni" not in model_id.lower()
|
||||
)
|
||||
],
|
||||
)
|
||||
register_configs(
|
||||
sampling_param_cls=FastH3OmniSamplingParams,
|
||||
pipeline_config_cls=FastH3OmniPipelineConfig,
|
||||
hf_model_paths=[
|
||||
"FastVideo/FastVideo-FastH3-Omni-8-Step-V1",
|
||||
],
|
||||
model_detectors=[
|
||||
lambda model_id: (
|
||||
"fasth3omni" in model_id.lower().replace("-", "").replace("_", "")
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
@@ -29,6 +29,9 @@ def _optional_unit_float(value: Any, field_name: str) -> float | None:
|
||||
@dataclass
|
||||
class MiniMaxH3SamplingParams(SamplingParams):
|
||||
min_num_inference_steps: ClassVar[int] = 1
|
||||
# Single-partition releases name their partition; None infers it from
|
||||
# --model-variant or the model path.
|
||||
release_partition: ClassVar[str | None] = None
|
||||
|
||||
height: int = 512
|
||||
width: int = 896
|
||||
@@ -110,7 +113,9 @@ class MiniMaxH3SamplingParams(SamplingParams):
|
||||
file delivery, while the public H3 contract correctly requires it.
|
||||
"""
|
||||
selected_variant = getattr(server_args, "model_variant", None)
|
||||
if selected_variant is not None:
|
||||
if self.release_partition is not None:
|
||||
selected_partition = self.release_partition
|
||||
elif selected_variant is not None:
|
||||
selected_partition = str(selected_variant).strip().lower()
|
||||
else:
|
||||
selected_path = server_args.model_subfolder or server_args.model_path
|
||||
@@ -378,4 +383,33 @@ class FastH3SamplingParams(MiniMaxH3SamplingParams):
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["FastH3SamplingParams", "MiniMaxH3SamplingParams"]
|
||||
@dataclass
|
||||
class FastH3OmniSamplingParams(MiniMaxH3SamplingParams):
|
||||
"""FastH3 Omni 8-Step V1: eight PDD forwards over ordered references."""
|
||||
|
||||
release_partition: ClassVar[str | None] = "ref2va"
|
||||
num_inference_steps: int = 8
|
||||
task: str | None = "ref2va"
|
||||
quality: str = "lossless"
|
||||
# Trained with target-only flow: references stay clean.
|
||||
imgvid_cond_noise_aug_for_inference: float | None = 1.0
|
||||
audio_cond_noise_aug_for_inference: float | None = 1.0
|
||||
|
||||
def _validate(self) -> None:
|
||||
super()._validate()
|
||||
if self.num_inference_steps != 8:
|
||||
raise ValueError(
|
||||
"FastH3 Omni is distilled for exactly eight PDD forwards; got "
|
||||
f"num_inference_steps={self.num_inference_steps}."
|
||||
)
|
||||
if self.task is not None and self.task.strip().lower() != "ref2va":
|
||||
raise ValueError(
|
||||
f"FastH3 Omni is distilled for ref2va only (got task={self.task!r})."
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FastH3OmniSamplingParams",
|
||||
"FastH3SamplingParams",
|
||||
"MiniMaxH3SamplingParams",
|
||||
]
|
||||
|
||||
@@ -189,6 +189,7 @@ KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
|
||||
"minimaxai/minimax-h3": "MiniMaxH3Pipeline",
|
||||
"minimax/minimax-h3": "MiniMaxH3Pipeline",
|
||||
"fastvideo/fastvideo-fasth3-8-step-v2": "FastH3Pipeline",
|
||||
"fastvideo/fastvideo-fasth3-omni-8-step-v1": "FastH3Pipeline",
|
||||
"openvdn/vdn-minimax-h3": "VDNH3Pipeline",
|
||||
"lerobot/pi05": "Pi05Pipeline",
|
||||
"pi05": "Pi05Pipeline",
|
||||
|
||||
@@ -66,6 +66,7 @@ def construct_variable_block_sizes(
|
||||
dit_seq_shape: tuple[int, int, int],
|
||||
num_tiles: tuple[int, int, int],
|
||||
device: torch.device,
|
||||
tile_size: tuple[int, int, int] = VSA_TILE_SIZE,
|
||||
) -> torch.LongTensor:
|
||||
"""
|
||||
Compute the number of valid (non‑padded) tokens inside every
|
||||
@@ -78,7 +79,7 @@ def construct_variable_block_sizes(
|
||||
"""
|
||||
# unpack
|
||||
t, h, w = dit_seq_shape
|
||||
ts_t, ts_h, ts_w = VSA_TILE_SIZE
|
||||
ts_t, ts_h, ts_w = tile_size
|
||||
n_t, n_h, n_w = num_tiles
|
||||
|
||||
def _sizes(dim_len: int, tile: int, n_tiles: int) -> torch.LongTensor:
|
||||
|
||||
+267
-74
@@ -19,6 +19,14 @@ under a FLOP-matched budget ("compete"). The compression branch is gated by
|
||||
``to_gate_compress``: base H3 has no such weights and the gate loads as
|
||||
zeros (pure sparse); VSA-distilled students (FastH3) ship trained gates
|
||||
that activate the branch.
|
||||
|
||||
Ref2VA interleaves reference videos with dense rows. Each reference video is
|
||||
its own 3-D tiled region; every video query keeps its own top-k of EACH
|
||||
region (references at ``ref_keep_rate``, the generated video at the step
|
||||
sparsity), FastVideo's "p2_multi_region" policy. Checkpoints trained on
|
||||
128-token (4, 4, 8) tiles select and compress at that logical granularity;
|
||||
each logical tile runs as two contiguous 64-token kernel tiles, so the
|
||||
selected key set is unchanged.
|
||||
"""
|
||||
|
||||
import functools
|
||||
@@ -50,83 +58,121 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels impo
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
|
||||
# The FastH3 checkpoints are trained and served at the 64-token (4, 4, 4)
|
||||
# tile geometry; the kernel block size matches it exactly.
|
||||
# The kernels run 64-token blocks. FastH3 is trained on (4, 4, 4) tiles;
|
||||
# FastH3 Omni on 128-token (4, 4, 8) tiles, each served as two kernel blocks.
|
||||
VSA_H3_TILE_SHAPE = (4, 4, 4)
|
||||
VSA_H3_TILE_ELEMS = math.prod(VSA_H3_TILE_SHAPE)
|
||||
VSA_H3_TILE_SHAPES = {64: VSA_H3_TILE_SHAPE, 128: (4, 4, 8)}
|
||||
|
||||
_DIT_BLOCK_PREFIX = re.compile(r"^blocks\.(\d+)\.")
|
||||
|
||||
PackedSegment = int | tuple[int, int, int]
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=8)
|
||||
def _h3_segment_tile_geometry(
|
||||
segments: tuple[PackedSegment, ...],
|
||||
tile_shape: tuple[int, int, int],
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, tuple[tuple[int, int], ...]]:
|
||||
"""Tile a packed sequence of interleaved dense segments and video regions.
|
||||
|
||||
``segments`` lists the packed order: an ``int`` is a dense segment's row
|
||||
count, a ``(t, h, w)`` triple one video region's token grid. Dense
|
||||
segments are chunked into the leading prefix tiles (never straddling a
|
||||
segment boundary); each region follows as a contiguous run of 3-D tiles.
|
||||
|
||||
Returns, at kernel (64-token block) granularity, (variable_block_sizes
|
||||
int32 [n_blocks], pack_index int32 [S_pad]: padded position -> packed row
|
||||
or -1, unpack_index int32 [used]: packed row -> padded position), then
|
||||
num_prefix_tiles and the per-region [start, end) spans in logical tiles.
|
||||
A 128-token logical tile is two consecutive kernel blocks; its upper half
|
||||
is empty when the tile holds at most 64 rows.
|
||||
"""
|
||||
tile_elems = math.prod(tile_shape)
|
||||
split = tile_elems // VSA_H3_TILE_ELEMS
|
||||
|
||||
prefix_sizes: list[int] = []
|
||||
prefix_rows: list[torch.Tensor] = []
|
||||
video_entries: list[tuple[int, tuple[int, int, int]]] = []
|
||||
cursor = 0
|
||||
for segment in segments:
|
||||
if isinstance(segment, tuple):
|
||||
video_entries.append((cursor, segment))
|
||||
cursor += math.prod(segment)
|
||||
continue
|
||||
full, remainder = divmod(segment, tile_elems)
|
||||
prefix_sizes.extend([tile_elems] * full)
|
||||
if remainder:
|
||||
prefix_sizes.append(remainder)
|
||||
prefix_rows.append(
|
||||
torch.arange(cursor, cursor + segment, device=device, dtype=torch.long)
|
||||
)
|
||||
cursor += segment
|
||||
num_prefix_tiles = len(prefix_sizes)
|
||||
|
||||
video_rows: list[torch.Tensor] = []
|
||||
size_parts = [torch.tensor(prefix_sizes, dtype=torch.long, device=device)]
|
||||
spans: list[tuple[int, int]] = []
|
||||
tile_cursor = num_prefix_tiles
|
||||
for start, grid in video_entries:
|
||||
num_tiles = tuple(math.ceil(n / t) for n, t in zip(grid, tile_shape))
|
||||
sizes = construct_variable_block_sizes(grid, num_tiles, device, tile_shape)
|
||||
video_rows.append(get_tile_partition_indices(grid, tile_shape, device) + start)
|
||||
size_parts.append(sizes.to(torch.long))
|
||||
spans.append((tile_cursor, tile_cursor + int(sizes.numel())))
|
||||
tile_cursor += int(sizes.numel())
|
||||
|
||||
tile_partition_indices = torch.cat(prefix_rows + video_rows)
|
||||
logical_sizes = torch.cat(size_parts)
|
||||
non_pad_index = get_non_pad_index(logical_sizes, tile_elems)
|
||||
if (
|
||||
int(logical_sizes.sum()) != cursor
|
||||
or non_pad_index.numel() != cursor
|
||||
or tile_partition_indices.numel() != cursor
|
||||
):
|
||||
raise ValueError(
|
||||
f"VSA-H3 tile geometry mismatch for segments={segments}, "
|
||||
f"tile={tile_shape}: sizes sum {int(logical_sizes.sum())}, non-pad "
|
||||
f"{non_pad_index.numel()}, expected {cursor}."
|
||||
)
|
||||
|
||||
# A logical tile's padded slots are its kernel blocks' slots in order,
|
||||
# so the pack/unpack maps are shared; only the per-block sizes split.
|
||||
seq_pad = logical_sizes.numel() * tile_elems
|
||||
pack_index = torch.full((seq_pad,), -1, dtype=torch.int32, device=device)
|
||||
pack_index[non_pad_index] = tile_partition_indices.to(torch.int32)
|
||||
unpack_index = torch.empty(cursor, dtype=torch.int32, device=device)
|
||||
unpack_index[tile_partition_indices] = non_pad_index.to(torch.int32)
|
||||
offsets = torch.arange(split, device=device) * VSA_H3_TILE_ELEMS
|
||||
block_sizes = (
|
||||
(logical_sizes[:, None] - offsets).clamp(0, VSA_H3_TILE_ELEMS).flatten()
|
||||
)
|
||||
return (
|
||||
block_sizes.to(torch.int32),
|
||||
pack_index,
|
||||
unpack_index,
|
||||
num_prefix_tiles,
|
||||
tuple(spans),
|
||||
)
|
||||
|
||||
|
||||
def _h3_tile_geometry(
|
||||
prefix_segments: tuple[int, ...],
|
||||
dit_seq_shape: tuple[int, int, int],
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
|
||||
"""Segment-pure prefix chunks, then video tiles.
|
||||
"""Segment-pure prefix chunks, then the generated video's 64-token tiles.
|
||||
|
||||
Returns (variable_block_sizes int32 [n_tiles], pack_index int32 [S_pad]:
|
||||
padded position -> packed row or -1, unpack_index int32 [used]: packed row
|
||||
-> padded position, num_prefix_tiles, num_video_tiles).
|
||||
Returns (variable_block_sizes, pack_index, unpack_index,
|
||||
num_prefix_tiles, num_video_tiles).
|
||||
"""
|
||||
prefix_len = sum(prefix_segments)
|
||||
|
||||
prefix_sizes: list[int] = []
|
||||
for segment in prefix_segments:
|
||||
full, remainder = divmod(segment, VSA_H3_TILE_ELEMS)
|
||||
prefix_sizes.extend([VSA_H3_TILE_ELEMS] * full)
|
||||
if remainder:
|
||||
prefix_sizes.append(remainder)
|
||||
num_prefix_tiles = len(prefix_sizes)
|
||||
|
||||
num_tiles = (
|
||||
math.ceil(dit_seq_shape[0] / VSA_H3_TILE_SHAPE[0]),
|
||||
math.ceil(dit_seq_shape[1] / VSA_H3_TILE_SHAPE[1]),
|
||||
math.ceil(dit_seq_shape[2] / VSA_H3_TILE_SHAPE[2]),
|
||||
)
|
||||
video_sizes = construct_variable_block_sizes(dit_seq_shape, num_tiles, device)
|
||||
num_video_tiles = int(video_sizes.numel())
|
||||
|
||||
video_indices = (
|
||||
get_tile_partition_indices(dit_seq_shape, VSA_H3_TILE_SHAPE, device)
|
||||
+ prefix_len
|
||||
)
|
||||
tile_partition_indices = torch.cat(
|
||||
[
|
||||
torch.arange(prefix_len, device=device, dtype=torch.long),
|
||||
video_indices,
|
||||
]
|
||||
)
|
||||
variable_block_sizes = torch.cat(
|
||||
[
|
||||
torch.tensor(prefix_sizes, dtype=torch.long, device=device),
|
||||
video_sizes.to(torch.long),
|
||||
]
|
||||
)
|
||||
non_pad_index = get_non_pad_index(variable_block_sizes, VSA_H3_TILE_ELEMS)
|
||||
|
||||
total = prefix_len + math.prod(dit_seq_shape)
|
||||
sizes_sum = int(variable_block_sizes.sum())
|
||||
if sizes_sum != total or non_pad_index.numel() != total:
|
||||
raise ValueError(
|
||||
f"VSA-H3 tile geometry mismatch for prefix={prefix_segments}, "
|
||||
f"video={dit_seq_shape}: sizes sum {sizes_sum}, non-pad "
|
||||
f"{non_pad_index.numel()}, expected {total}."
|
||||
sizes, pack_index, unpack_index, num_prefix_tiles, spans = (
|
||||
_h3_segment_tile_geometry(
|
||||
(*prefix_segments, tuple(dit_seq_shape)), VSA_H3_TILE_SHAPE, device
|
||||
)
|
||||
|
||||
seq_pad = variable_block_sizes.numel() * VSA_H3_TILE_ELEMS
|
||||
pack_index = torch.full((seq_pad,), -1, dtype=torch.int32, device=device)
|
||||
pack_index[non_pad_index] = tile_partition_indices.to(torch.int32)
|
||||
unpack_index = torch.empty(total, dtype=torch.int32, device=device)
|
||||
unpack_index[tile_partition_indices] = non_pad_index.to(torch.int32)
|
||||
return (
|
||||
variable_block_sizes.to(torch.int32),
|
||||
pack_index,
|
||||
unpack_index,
|
||||
num_prefix_tiles,
|
||||
num_video_tiles,
|
||||
)
|
||||
return sizes, pack_index, unpack_index, num_prefix_tiles, spans[0][1] - spans[0][0]
|
||||
|
||||
|
||||
class VideoSparseAttentionH3Backend(AttentionBackend):
|
||||
@@ -160,16 +206,32 @@ class VideoSparseAttentionH3Metadata(AttentionMetadata):
|
||||
num_prefix_tiles: int
|
||||
num_video_tiles: int
|
||||
exempt: bool
|
||||
# Kernel (64-token block) granularity; a 128-token logical tile is two
|
||||
# consecutive blocks. Tile counts above are logical.
|
||||
variable_block_sizes: torch.Tensor
|
||||
pack_index: torch.Tensor
|
||||
unpack_index: torch.Tensor
|
||||
dense_layers: tuple[int, ...] = ()
|
||||
workspace_cache: dict = field(default_factory=dict)
|
||||
tile_elems: int = VSA_H3_TILE_ELEMS
|
||||
# Logical-tile [start, end) per video region, the generated video last,
|
||||
# with each region's sparsity. None: the generated video alone, at
|
||||
# VSA_sparsity.
|
||||
video_tile_spans: tuple[tuple[int, int], ...] | None = None
|
||||
span_sparsities: tuple[float, ...] | None = None
|
||||
|
||||
@property
|
||||
def num_tiles(self) -> int:
|
||||
return self.num_prefix_tiles + self.num_video_tiles
|
||||
|
||||
@property
|
||||
def blocks_per_tile(self) -> int:
|
||||
return self.tile_elems // VSA_H3_TILE_ELEMS
|
||||
|
||||
@property
|
||||
def uses_regions(self) -> bool:
|
||||
return self.video_tile_spans is not None
|
||||
|
||||
|
||||
class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self) -> None:
|
||||
@@ -186,14 +248,50 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder):
|
||||
exempt: bool = True,
|
||||
dense_layers: tuple[int, ...] = (),
|
||||
dense_first_n_steps: int = 0,
|
||||
packed_segments: tuple[PackedSegment, ...] | None = None,
|
||||
tile_size: int = VSA_H3_TILE_ELEMS,
|
||||
ref_keep_rate: float | None = None,
|
||||
**kwargs: dict[str, Any],
|
||||
) -> VideoSparseAttentionH3Metadata:
|
||||
"""``packed_segments`` replaces ``prefix_segments`` for layouts with
|
||||
reference videos: the packed order of dense row counts and video
|
||||
token grids, ending with the generated video's grid. Every earlier
|
||||
grid is a reference region kept at ``ref_keep_rate``."""
|
||||
dit_seq_shape = (
|
||||
raw_latent_shape[0] // patch_size[0],
|
||||
raw_latent_shape[1] // patch_size[1],
|
||||
raw_latent_shape[2] // patch_size[2],
|
||||
)
|
||||
prefix_segments = tuple(int(s) for s in prefix_segments if s > 0)
|
||||
tile_shape = VSA_H3_TILE_SHAPES.get(int(tile_size))
|
||||
if tile_shape is None:
|
||||
raise ValueError(
|
||||
f"VSA-H3 tile_size must be one of {sorted(VSA_H3_TILE_SHAPES)}, "
|
||||
f"got {tile_size!r}"
|
||||
)
|
||||
if packed_segments is None:
|
||||
packed_segments = (*prefix_segments, dit_seq_shape)
|
||||
segments = tuple(
|
||||
tuple(int(v) for v in s) if isinstance(s, tuple) else int(s)
|
||||
for s in packed_segments
|
||||
if isinstance(s, tuple) or s > 0
|
||||
)
|
||||
if not segments or segments[-1] != dit_seq_shape:
|
||||
raise ValueError(
|
||||
f"VSA-H3 packed segments must end with the generated video "
|
||||
f"{dit_seq_shape}, got {segments}"
|
||||
)
|
||||
num_references = sum(isinstance(s, tuple) for s in segments) - 1
|
||||
if num_references:
|
||||
if ref_keep_rate is None or not 0.0 < float(ref_keep_rate) <= 1.0:
|
||||
raise ValueError(
|
||||
f"VSA-H3 with {num_references} reference-video region(s) "
|
||||
f"needs vsa_ref_keep_rate in (0, 1], got {ref_keep_rate!r}"
|
||||
)
|
||||
if not exempt:
|
||||
raise ValueError(
|
||||
"vsa_mode='compete' supports only the generated-video "
|
||||
"region; use vsa_mode='exempt' with reference videos"
|
||||
)
|
||||
if current_timestep < dense_first_n_steps:
|
||||
VSA_sparsity = 0.0
|
||||
|
||||
@@ -202,21 +300,31 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder):
|
||||
pack_index,
|
||||
unpack_index,
|
||||
num_prefix_tiles,
|
||||
num_video_tiles,
|
||||
) = _h3_tile_geometry(prefix_segments, dit_seq_shape, device)
|
||||
spans,
|
||||
) = _h3_segment_tile_geometry(segments, tile_shape, device)
|
||||
|
||||
regional = num_references > 0 or tile_shape != VSA_H3_TILE_SHAPE
|
||||
span_sparsities = None
|
||||
if regional:
|
||||
span_sparsities = (1.0 - float(ref_keep_rate or 1.0),) * num_references
|
||||
span_sparsities += (float(VSA_sparsity),)
|
||||
return VideoSparseAttentionH3Metadata(
|
||||
current_timestep=current_timestep,
|
||||
VSA_sparsity=float(VSA_sparsity),
|
||||
total_seq_length=sum(prefix_segments) + math.prod(dit_seq_shape),
|
||||
total_seq_length=sum(
|
||||
math.prod(s) if isinstance(s, tuple) else s for s in segments
|
||||
),
|
||||
num_prefix_tiles=num_prefix_tiles,
|
||||
num_video_tiles=num_video_tiles,
|
||||
num_video_tiles=spans[-1][1] - num_prefix_tiles,
|
||||
exempt=exempt,
|
||||
variable_block_sizes=variable_block_sizes,
|
||||
pack_index=pack_index,
|
||||
unpack_index=unpack_index,
|
||||
dense_layers=tuple(int(layer) for layer in dense_layers),
|
||||
workspace_cache=self._workspace_cache,
|
||||
tile_elems=math.prod(tile_shape),
|
||||
video_tile_spans=spans if regional else None,
|
||||
span_sparsities=span_sparsities,
|
||||
)
|
||||
|
||||
|
||||
@@ -225,6 +333,50 @@ def _compute_topk(sparsity: float, num_video_tiles: int) -> int:
|
||||
return max(1, min(keep, num_video_tiles))
|
||||
|
||||
|
||||
def _region_tile_lists(
|
||||
scores: torch.Tensor,
|
||||
num_prefix_tiles: int,
|
||||
spans: tuple[tuple[int, int], ...],
|
||||
sparsities: tuple[float, ...],
|
||||
) -> torch.Tensor:
|
||||
"""scores [H, n_tiles, n_tiles] -> ascending int32 logical kv-tile lists
|
||||
[H, num_video_tiles, width]: every prefix tile, then each region's own
|
||||
top-k (the whole region when its budget covers it)."""
|
||||
prefix = num_prefix_tiles
|
||||
video_rows = scores[:, prefix:, :]
|
||||
parts = [
|
||||
torch.arange(prefix, device=scores.device).expand(
|
||||
*video_rows.shape[:-1], prefix
|
||||
)
|
||||
]
|
||||
for (start, end), sparsity in zip(spans, sparsities):
|
||||
keep = _compute_topk(sparsity, end - start)
|
||||
if keep == end - start:
|
||||
parts.append(
|
||||
torch.arange(start, end, device=scores.device).expand(
|
||||
*video_rows.shape[:-1], keep
|
||||
)
|
||||
)
|
||||
continue
|
||||
picked = video_rows[..., start:end].topk(keep, dim=-1).indices
|
||||
parts.append(picked.sort(dim=-1).values + start)
|
||||
return torch.cat(parts, dim=-1).to(torch.int32)
|
||||
|
||||
|
||||
def _pool_logical_tiles(
|
||||
pooled: torch.Tensor, meta: VideoSparseAttentionH3Metadata
|
||||
) -> torch.Tensor:
|
||||
"""Kernel-block means [3, H, n_blocks, D] -> logical-tile means."""
|
||||
split = meta.blocks_per_tile
|
||||
if split == 1:
|
||||
return pooled
|
||||
sizes = meta.variable_block_sizes.view(-1, split).to(torch.float32)
|
||||
blocks = pooled.unflatten(2, (-1, split))
|
||||
# Empty upper halves pool to 0/0; they carry no weight.
|
||||
weighted = torch.where(sizes[..., None] > 0, blocks * sizes[..., None], 0.0)
|
||||
return weighted.sum(3) / sizes.sum(-1)[:, None]
|
||||
|
||||
|
||||
def _topk_tile_lists(
|
||||
scores: torch.Tensor,
|
||||
num_prefix_tiles: int,
|
||||
@@ -266,6 +418,8 @@ def _workspace_key(
|
||||
meta.num_tiles,
|
||||
meta.num_prefix_tiles,
|
||||
meta.exempt,
|
||||
meta.tile_elems,
|
||||
meta.uses_regions,
|
||||
heads,
|
||||
head_dim,
|
||||
has_gate,
|
||||
@@ -276,10 +430,11 @@ def _workspace_key(
|
||||
|
||||
class _Workspace:
|
||||
"""Per-geometry scratch: tiled q/k/v(/gate) [3|4, H, S_pad, D], pooled fp32
|
||||
tile means [3, H, n_tiles, D], and the kernel index lists (prefix rows and
|
||||
prefix columns are static; only the top-k video columns change per layer).
|
||||
The native kernel pairs query tiles, so the tile axis is padded to an even
|
||||
count with one empty tile.
|
||||
block means [3, H, n_blocks, D], and the kernel index lists (prefix rows
|
||||
and prefix columns are static; only the top-k video columns change per
|
||||
layer). All of it is at kernel-block granularity. The native kernel pairs
|
||||
query blocks, so the block axis is padded to an even count with one empty
|
||||
block.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -291,7 +446,7 @@ class _Workspace:
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
n_tiles = meta.num_tiles
|
||||
n_tiles = int(meta.variable_block_sizes.numel())
|
||||
self.n_tiles = n_tiles
|
||||
self.use_sm100_kernel = can_use_vsa_block_sparse_sm100(
|
||||
device.index, dtype, head_dim
|
||||
@@ -321,7 +476,7 @@ class _Workspace:
|
||||
self.dense_num[:, :n_tiles] = n_tiles
|
||||
self.q2k_index = self.dense_index.clone()
|
||||
self.q2k_num = self.dense_num.clone()
|
||||
num_video_tiles = meta.num_video_tiles
|
||||
num_video_tiles = 0 if meta.uses_regions else meta.num_video_tiles
|
||||
# Zeroed scratch for the exempt top-k compaction; kept zero between uses.
|
||||
self.topk_mask = torch.zeros(
|
||||
(heads, num_video_tiles, num_video_tiles), dtype=torch.int8, device=device
|
||||
@@ -339,7 +494,7 @@ class _Workspace:
|
||||
def _gate_tile_index(
|
||||
meta: VideoSparseAttentionH3Metadata, row_start: int, num_rows: int
|
||||
) -> torch.Tensor:
|
||||
"""Tile of each packed row in the shard, -1 for pad rows."""
|
||||
"""Kernel block of each packed row in the shard, -1 for pad rows."""
|
||||
key = ("gate_tile_index", row_start, num_rows)
|
||||
tile_index = meta.workspace_cache.get(key)
|
||||
if tile_index is None:
|
||||
@@ -390,6 +545,8 @@ def _select_kv_lists(
|
||||
sparsity: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Kernel index lists for this layer: dense, or top-k video columns."""
|
||||
if meta.uses_regions:
|
||||
return _select_region_kv_lists(ws, meta, scores, sparsity)
|
||||
keep = _compute_topk(sparsity, meta.num_video_tiles)
|
||||
if sparsity <= 0.0 or keep >= meta.num_video_tiles:
|
||||
return ws.dense_index, ws.dense_num
|
||||
@@ -414,6 +571,36 @@ def _select_kv_lists(
|
||||
)
|
||||
|
||||
|
||||
def _select_region_kv_lists(
|
||||
ws: _Workspace,
|
||||
meta: VideoSparseAttentionH3Metadata,
|
||||
scores: torch.Tensor | None,
|
||||
sparsity: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-region top-k over logical tiles, expanded to kernel blocks."""
|
||||
spans = meta.video_tile_spans
|
||||
sparsities = meta.span_sparsities
|
||||
# Dense steps and layers keep every region dense.
|
||||
if sparsity <= 0.0:
|
||||
return ws.dense_index, ws.dense_num
|
||||
if all(
|
||||
_compute_topk(s, end - start) == end - start
|
||||
for (start, end), s in zip(spans, sparsities)
|
||||
):
|
||||
return ws.dense_index, ws.dense_num
|
||||
lists = _region_tile_lists(scores, meta.num_prefix_tiles, spans, sparsities)
|
||||
split = meta.blocks_per_tile
|
||||
if split > 1:
|
||||
offsets = torch.arange(split, device=lists.device, dtype=lists.dtype)
|
||||
lists = (lists[..., None] * split + offsets).flatten(-2)
|
||||
lists = lists.repeat_interleave(split, dim=1)
|
||||
prefix_blocks = meta.num_prefix_tiles * split
|
||||
width = lists.shape[-1]
|
||||
ws.q2k_index[:, prefix_blocks : ws.n_tiles, :width] = lists
|
||||
ws.q2k_num[:, prefix_blocks : ws.n_tiles] = width
|
||||
return ws.q2k_index, ws.q2k_num
|
||||
|
||||
|
||||
class VideoSparseAttentionH3Impl(AttentionImpl):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -524,7 +711,7 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
|
||||
ws.tiled,
|
||||
ws.pooled,
|
||||
)
|
||||
q_pooled, k_pooled, v_pooled = ws.pooled
|
||||
q_pooled, k_pooled, v_pooled = _pool_logical_tiles(ws.pooled, meta)
|
||||
|
||||
scores = None
|
||||
if sparsity > 0.0 or needs_compress:
|
||||
@@ -547,6 +734,12 @@ class VideoSparseAttentionH3Impl(AttentionImpl):
|
||||
out_compress = None
|
||||
if needs_compress:
|
||||
out_compress = torch.matmul(torch.softmax(scores, dim=-1), v_pooled)
|
||||
if meta.blocks_per_tile > 1:
|
||||
# The gate folds per kernel block; both halves of a logical
|
||||
# tile share its compressed row.
|
||||
out_compress = out_compress.repeat_interleave(
|
||||
meta.blocks_per_tile, dim=1
|
||||
)
|
||||
|
||||
result = torch.empty(query.shape, dtype=query.dtype, device=query.device)
|
||||
vsa_h3_untile(
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from dataclasses import replace
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import (
|
||||
MiniMaxH3PipelineConfig,
|
||||
)
|
||||
@@ -140,6 +142,25 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
f"disaggregation role {role.value!r} is not supported"
|
||||
)
|
||||
|
||||
def initialize_pipeline(self, server_args: ServerArgs) -> None:
|
||||
self.pdd_heads = self._resolve_pdd_heads()
|
||||
transformer = self.get_module("transformer")
|
||||
if (
|
||||
self.pdd_heads
|
||||
and transformer is not None
|
||||
and transformer.final_layer._pdd_heads is None
|
||||
):
|
||||
transformer.final_layer.load_pdd_fused_heads(self.pdd_heads)
|
||||
|
||||
def _resolve_pdd_heads(self) -> str | None:
|
||||
"""The env override, else a PDD release's own fused heads."""
|
||||
if envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS:
|
||||
return envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
|
||||
release_metadata = getattr(self, "release_metadata", None)
|
||||
if release_metadata is None or release_metadata.pdd_fused_heads is None:
|
||||
return None
|
||||
return os.path.join(self.model_path, release_metadata.pdd_fused_heads)
|
||||
|
||||
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
|
||||
# Per-model sigma override from model_index.json; contract tests
|
||||
# construct the pipeline without model_path, hence the guard.
|
||||
@@ -180,6 +201,7 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
if release_metadata is not None
|
||||
else None
|
||||
),
|
||||
pdd_heads=getattr(self, "pdd_heads", None),
|
||||
)
|
||||
)
|
||||
self.add_stage(
|
||||
@@ -206,7 +228,8 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
|
||||
|
||||
class FastH3Pipeline(MiniMaxH3Pipeline):
|
||||
"""FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only).
|
||||
"""FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only), and the
|
||||
FastH3 Omni Ref2VA PDD student.
|
||||
|
||||
The flat single-partition repo is materialized into the base-H3 layout by
|
||||
its registered model overlay (see runtime/utils/model_overlay.py), so every stage, loader,
|
||||
|
||||
+16
@@ -4,6 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping
|
||||
|
||||
@@ -54,6 +55,9 @@ class MiniMaxH3ReleaseMetadata:
|
||||
audio_sigma_shift: float
|
||||
# trained DMD rungs of a distilled release, unshifted; None keeps the uniform grid
|
||||
dmd_denoising_steps: tuple[int, ...] | None = None
|
||||
# PDD release: fused per-step output heads, relative to the model dir,
|
||||
# with pdd_config.json beside them
|
||||
pdd_fused_heads: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_model_index(
|
||||
@@ -109,6 +113,17 @@ class MiniMaxH3ReleaseMetadata:
|
||||
"strictly decreasing integers in (0, 1000]"
|
||||
)
|
||||
dmd_steps = tuple(dmd_steps)
|
||||
pdd_fused_heads = raw.get("pdd_fused_heads")
|
||||
if pdd_fused_heads is not None and (
|
||||
not isinstance(pdd_fused_heads, str)
|
||||
or not pdd_fused_heads.endswith(".safetensors")
|
||||
or os.path.isabs(pdd_fused_heads)
|
||||
or ".." in pdd_fused_heads.split("/")
|
||||
):
|
||||
raise ValueError(
|
||||
"model_index.json._minimax_h3.pdd_fused_heads must be a relative "
|
||||
".safetensors path inside the model directory"
|
||||
)
|
||||
metadata = cls(
|
||||
schema_version=1,
|
||||
partition=partition,
|
||||
@@ -117,6 +132,7 @@ class MiniMaxH3ReleaseMetadata:
|
||||
video_sigma_shift=video_sigma,
|
||||
audio_sigma_shift=audio_sigma,
|
||||
dmd_denoising_steps=dmd_steps,
|
||||
pdd_fused_heads=pdd_fused_heads,
|
||||
)
|
||||
for task in metadata.tasks:
|
||||
if canonical_minimax_h3_task(task) != task:
|
||||
|
||||
+2
@@ -552,6 +552,7 @@ def comfyui_payload_to_branch_inputs(
|
||||
"orig_video_shape": orig_shape,
|
||||
"padded_video_shape": tuple(video.shape),
|
||||
"is_ref2va": bool(refs),
|
||||
"native_layout": layout is None,
|
||||
"sample_sigmas": sample_sigmas,
|
||||
"sigma_v": float(sigma_v),
|
||||
"shift_v": shift_v,
|
||||
@@ -669,6 +670,7 @@ def _step_attn_metadata(
|
||||
latent_shape=latent_shape,
|
||||
server_args=server_args,
|
||||
device=device,
|
||||
native_layout=inputs["native_layout"],
|
||||
)
|
||||
if build is not None:
|
||||
return build
|
||||
|
||||
+71
-13
@@ -5,6 +5,7 @@ single-branch execution, and payload validation.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
@@ -778,6 +779,10 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
|
||||
device=device,
|
||||
)
|
||||
initial_video, initial_audio = _expand_initial_rows(ctx, positive)
|
||||
if model.final_layer._pdd_heads is not None:
|
||||
initial_video, initial_audio = _scale_pdd_initial_noise(
|
||||
ctx, positive, initial_video, initial_audio
|
||||
)
|
||||
rollout_ctx = None
|
||||
if getattr(batch, "rollout", False):
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.minimax_h3_rollout import (
|
||||
@@ -1049,30 +1054,31 @@ def _maybe_prepare_vsa_h3_step_metadata(
|
||||
latent_shape: tuple[int, int, int],
|
||||
server_args: ServerArgs,
|
||||
device: torch.device,
|
||||
native_layout: bool = True,
|
||||
) -> Callable[[int], Any] | None:
|
||||
"""Per-step VSA-H3 metadata builder over the request-static packed layout,
|
||||
or None off the VSA path."""
|
||||
or None off the VSA path.
|
||||
|
||||
Ref2VA regions are read from the native packed layout's stream_layout;
|
||||
``native_layout=False`` marks a layout serialized by ComfyUI, whose
|
||||
stream_layout does not tell reference images from reference videos."""
|
||||
model._resolve_attention_backend_once()
|
||||
if (
|
||||
model._resolved_attention_backend
|
||||
is not AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3
|
||||
):
|
||||
return None
|
||||
if is_ref2va:
|
||||
if is_ref2va and not native_layout:
|
||||
raise NotImplementedError(
|
||||
"VSA-H3 supports the t2va/fl2va packed layout; the ref2va "
|
||||
"reference-block layout is not tiled yet. Use --attention-backend "
|
||||
"fa for ref2va."
|
||||
"VSA-H3 tiles Ref2VA references on SGLang's native packed layout; "
|
||||
"a ComfyUI-serialized Ref2VA layout is not tiled. Use "
|
||||
"--attention-backend fa for it."
|
||||
)
|
||||
|
||||
config = server_args.attention_backend_config or {}
|
||||
tile_size = int(config.get("vsa_tile_size", 64))
|
||||
if tile_size != 64:
|
||||
raise ValueError(
|
||||
"VSA-H3 in SGLang serves the trained 64-token (4, 4, 4) tile "
|
||||
f"geometry; got vsa_tile_size={tile_size}."
|
||||
)
|
||||
default_sparsity = server_args.pipeline_config.vsa_sparsity
|
||||
pipeline_config = server_args.pipeline_config
|
||||
tile_size = int(config.get("vsa_tile_size", pipeline_config.vsa_tile_size))
|
||||
ref_keep_rate = config.get("vsa_ref_keep_rate", pipeline_config.vsa_ref_keep_rate)
|
||||
default_sparsity = pipeline_config.vsa_sparsity
|
||||
sparsity = float(
|
||||
config.get("VSA_sparsity", config.get("sparsity", default_sparsity))
|
||||
)
|
||||
@@ -1089,6 +1095,9 @@ def _maybe_prepare_vsa_h3_step_metadata(
|
||||
cond_rows = int(packed["img_pos"].numel()) - video_rows
|
||||
audio_rows = int(packed["audio_pos"].numel())
|
||||
patch_size = server_args.pipeline_config.dit_config.arch_config.patch_size
|
||||
packed_segments = (
|
||||
_ref2va_vsa_segments(packed, latent_shape, patch_size) if is_ref2va else None
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import (
|
||||
VideoSparseAttentionH3MetadataBuilder,
|
||||
@@ -1107,11 +1116,44 @@ def _maybe_prepare_vsa_h3_step_metadata(
|
||||
exempt=mode == "exempt",
|
||||
dense_layers=dense_layers,
|
||||
dense_first_n_steps=dense_first_n_steps,
|
||||
packed_segments=packed_segments,
|
||||
tile_size=tile_size,
|
||||
ref_keep_rate=None if ref_keep_rate is None else float(ref_keep_rate),
|
||||
)
|
||||
|
||||
return build
|
||||
|
||||
|
||||
def _ref2va_vsa_segments(
|
||||
packed: Mapping[str, Any],
|
||||
latent_shape: tuple[int, int, int],
|
||||
patch_size: tuple[int, int, int],
|
||||
) -> tuple[int | tuple[int, int, int], ...]:
|
||||
"""Ref2VA packed order as VSA-H3 segments: text, keyframes, then each
|
||||
reference (images and audio dense, videos as their own token-grid
|
||||
regions), target audio, and the generated video's token grid last."""
|
||||
layout = packed["stream_layout"]
|
||||
shapes = layout["cond_image_shapes"]
|
||||
roles = layout["cond_image_roles"]
|
||||
audio_lens = layout["cond_audio_stream_lens"]
|
||||
reference_visual_rows = sum(math.prod(shape) for shape in shapes)
|
||||
cond_rows = int(packed["img_pos"].numel()) - int(packed["update_mask"].sum())
|
||||
segments: list[int | tuple[int, int, int]] = [
|
||||
int(packed["text_pos"].numel()),
|
||||
cond_rows - reference_visual_rows,
|
||||
]
|
||||
for kind, index in layout["cond_event_orders"]:
|
||||
if kind == "audio":
|
||||
segments.append(int(audio_lens[index]))
|
||||
elif roles[index] == "independent_cube":
|
||||
segments.append(tuple(int(v) for v in shapes[index]))
|
||||
else:
|
||||
segments.append(math.prod(shapes[index]))
|
||||
segments.append(int(packed["audio_pos"].numel()) - sum(audio_lens))
|
||||
segments.append(tuple(int(n) // int(p) for n, p in zip(latent_shape, patch_size)))
|
||||
return tuple(segments)
|
||||
|
||||
|
||||
def _build_packed_layout(
|
||||
ctx: _FullLoopContext,
|
||||
emb: Mapping[str, Any],
|
||||
@@ -1267,6 +1309,22 @@ def _expand_initial_rows(
|
||||
return initial_video, initial_audio
|
||||
|
||||
|
||||
def _scale_pdd_initial_noise(
|
||||
ctx: _FullLoopContext,
|
||||
positive: Any,
|
||||
initial_video: torch.Tensor,
|
||||
initial_audio: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Start PDD targets at the first node's noise level, as the students are
|
||||
trained (``sigma_0 * eps``); condition rows stay as they are. A no-op on
|
||||
grids that start at sigma 1."""
|
||||
initial_video = initial_video.clone()
|
||||
initial_audio = initial_audio.clone()
|
||||
initial_video[positive.video_target_slice] *= float(ctx.sigmas["video"][0])
|
||||
initial_audio[positive.audio_target_slice] *= float(ctx.sigmas["audio"][0])
|
||||
return initial_video, initial_audio
|
||||
|
||||
|
||||
def _publish_full_loop_outputs(
|
||||
ctx: _FullLoopContext,
|
||||
*,
|
||||
|
||||
+25
-13
@@ -30,6 +30,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
self,
|
||||
sigma_shift_scales=None,
|
||||
dmd_denoising_steps: tuple[int, ...] | None = None,
|
||||
pdd_heads: str | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Per-model sigma shift override (model_index.json "_minimax_h3" release
|
||||
@@ -38,7 +39,9 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
self.sigma_shift_scales = sigma_shift_scales
|
||||
self.dmd_denoising_steps = dmd_denoising_steps
|
||||
self._pdd_config = None
|
||||
pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
|
||||
# The pipeline resolves the release's fused heads; the env var serves
|
||||
# stages built without a pipeline.
|
||||
pdd_heads = pdd_heads or envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
|
||||
if pdd_heads:
|
||||
from safetensors import safe_open
|
||||
|
||||
@@ -49,6 +52,9 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
steps = f.get_slice("video_out.weight").get_shape()[0]
|
||||
if self._pdd_config["num_inference_steps"] != steps:
|
||||
raise ValueError("MiniMax-H3 PDD config does not match the fused heads")
|
||||
sigma_max = float(self._pdd_config.get("sigma_max", 1.0))
|
||||
if not 0.0 < sigma_max <= 1.0:
|
||||
raise ValueError("MiniMax-H3 PDD sigma_max must be in (0, 1]")
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.resolved_plan import (
|
||||
@@ -95,6 +101,10 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
)
|
||||
|
||||
config = self._pdd_config
|
||||
# Students trained on a grid ending at sigma_max < 1 (FastGen's
|
||||
# max_t) sample the [1, 0] grid scaled by it. The scale is shared by
|
||||
# every interval, so the fused heads' in-block weights are unchanged.
|
||||
sigma_max = float(config.get("sigma_max", 1.0))
|
||||
sigmas = batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY]
|
||||
for modality in ("video", "audio"):
|
||||
expected = minimax_h3_time_shift_sigmas(
|
||||
@@ -104,18 +114,20 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
if batch.is_warmup:
|
||||
# Warmup may run fewer steps, but must use the same intervals
|
||||
# as serving rather than rescaling a shorter grid to [1, 0].
|
||||
sigmas[modality] = expected[: batch.num_inference_steps + 1]
|
||||
continue
|
||||
actual = sigmas[modality]
|
||||
if len(actual) != len(expected) or any(
|
||||
not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7)
|
||||
for a, b in zip(actual, expected)
|
||||
):
|
||||
raise ValueError(
|
||||
f"MiniMax-H3 PDD requires num_inference_steps="
|
||||
f"{config['num_inference_steps']} and {modality} shift="
|
||||
f"{config[f'{modality}_shift']} to match the fused heads"
|
||||
)
|
||||
expected = expected[: batch.num_inference_steps + 1]
|
||||
else:
|
||||
actual = sigmas[modality]
|
||||
if len(actual) != len(expected) or any(
|
||||
not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7)
|
||||
for a, b in zip(actual, expected)
|
||||
):
|
||||
raise ValueError(
|
||||
f"MiniMax-H3 PDD requires num_inference_steps="
|
||||
f"{config['num_inference_steps']} and {modality} shift="
|
||||
f"{config[f'{modality}_shift']} to match the fused heads"
|
||||
)
|
||||
if batch.is_warmup or sigma_max != 1.0:
|
||||
sigmas[modality] = [sigma * sigma_max for sigma in expected]
|
||||
|
||||
@staticmethod
|
||||
def _publish_native_timestep_state(batch: Req) -> None:
|
||||
|
||||
@@ -53,6 +53,11 @@ BUILTIN_MODEL_OVERLAY_REGISTRY: dict[str, dict[str, Any]] = {
|
||||
"overlay_repo_id": "kevin-mi/VDN-H3-overlay",
|
||||
"overlay_revision": "7de18275dddfe59da36a234e222bcdd274963bc3",
|
||||
},
|
||||
"FastVideo/FastVideo-FastH3-Omni-8-Step-V1": {
|
||||
"overlay_repo_id": "Niehen6174/FastH3-Omni-overlay",
|
||||
"overlay_revision": "953ede55daf20a70cc6f0b59464293e2f408608c",
|
||||
"source_revision": "53e0d26a1691aa473b6262d8196a8feb2c297df0",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -353,7 +353,10 @@ def test_vsa_h3_tiles_cover_comfyui_packed_rows(keyframes):
|
||||
server_args = SimpleNamespace(
|
||||
attention_backend_config={},
|
||||
pipeline_config=SimpleNamespace(
|
||||
vsa_sparsity=0.8, dit_config=MiniMaxH3DiTConfig()
|
||||
vsa_sparsity=0.8,
|
||||
vsa_tile_size=64,
|
||||
vsa_ref_keep_rate=None,
|
||||
dit_config=MiniMaxH3DiTConfig(),
|
||||
),
|
||||
)
|
||||
build = _maybe_prepare_vsa_h3_step_metadata(
|
||||
@@ -374,6 +377,32 @@ def test_vsa_h3_tiles_cover_comfyui_packed_rows(keyframes):
|
||||
assert torch.equal(packed["text_pos"], torch.arange(text_len))
|
||||
|
||||
|
||||
def test_vsa_h3_rejects_comfyui_serialized_ref2va_layout():
|
||||
"""ComfyUI-serialized Ref2VA layouts do not tell reference images from
|
||||
reference videos, so VSA-H3 refuses them instead of mis-tiling."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
|
||||
_maybe_prepare_vsa_h3_step_metadata,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
|
||||
|
||||
model = SimpleNamespace(
|
||||
_resolve_attention_backend_once=lambda: None,
|
||||
_resolved_attention_backend=AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3,
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="ComfyUI-serialized Ref2VA"):
|
||||
_maybe_prepare_vsa_h3_step_metadata(
|
||||
model=model,
|
||||
packed={},
|
||||
is_ref2va=True,
|
||||
latent_shape=(4, 8, 12),
|
||||
server_args=SimpleNamespace(),
|
||||
device=torch.device("cpu"),
|
||||
native_layout=False,
|
||||
)
|
||||
|
||||
|
||||
def test_ref2va_image_plus_video_audio_used_prefix_matches_comfyui():
|
||||
kwargs = dict(text_len=5, latent_t=2, latent_h=4, latent_w=4, audio_t=5)
|
||||
refs = [
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""VSA-H3 Ref2VA regions and 128-token logical tiles.
|
||||
|
||||
The oracle below is written at logical-tile granularity from FastVideo's
|
||||
"p2_multi_region" mask semantics, independent of the kernel-block split the
|
||||
backend serves 128-token tiles with.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import (
|
||||
VideoSparseAttentionH3Impl,
|
||||
VideoSparseAttentionH3MetadataBuilder,
|
||||
_compute_topk,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.packed_sequence import (
|
||||
minimax_h3_packed_sequence_ref2va_blocks,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
|
||||
_ref2va_vsa_segments,
|
||||
)
|
||||
|
||||
requires_cuda = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="VSA-H3 kernels need CUDA"
|
||||
)
|
||||
|
||||
HEADS = 4
|
||||
HEAD_DIM = 128
|
||||
TARGET = (5, 6, 10)
|
||||
# text, image ref, reference video (ragged in every tile dimension), its
|
||||
# audio, a standalone audio ref, target audio, generated video
|
||||
SEGMENTS = (70, 30, 24, (3, 5, 9), 40, 50, TARGET)
|
||||
|
||||
|
||||
def _build(sparsity: float, device, *, tile_size=128, ref_keep_rate=0.25, **kw):
|
||||
return VideoSparseAttentionH3MetadataBuilder().build(
|
||||
current_timestep=0,
|
||||
raw_latent_shape=TARGET,
|
||||
patch_size=(1, 1, 1),
|
||||
VSA_sparsity=sparsity,
|
||||
prefix_segments=(),
|
||||
device=device,
|
||||
packed_segments=SEGMENTS,
|
||||
tile_size=tile_size,
|
||||
ref_keep_rate=ref_keep_rate,
|
||||
**kw,
|
||||
)
|
||||
|
||||
|
||||
def _used() -> int:
|
||||
return sum(math.prod(s) if isinstance(s, tuple) else s for s in SEGMENTS)
|
||||
|
||||
|
||||
def test_region_geometry_accounts_every_row() -> None:
|
||||
meta = _build(0.9, torch.device("cpu"))
|
||||
used = _used()
|
||||
assert meta.total_seq_length == used
|
||||
assert meta.blocks_per_tile == 2
|
||||
assert int(meta.variable_block_sizes.sum()) == used
|
||||
assert int((meta.pack_index >= 0).sum()) == used
|
||||
assert sorted(meta.unpack_index.tolist()) == sorted(
|
||||
(meta.pack_index >= 0).nonzero().flatten().tolist()
|
||||
)
|
||||
# Dense rows chunk per segment into 128-token tiles: 70, 30, 24, 40, 50.
|
||||
assert meta.num_prefix_tiles == 5
|
||||
logical = meta.variable_block_sizes.view(-1, 2).sum(-1)
|
||||
assert logical[:5].tolist() == [70, 30, 24, 40, 50]
|
||||
# Reference (3, 5, 9) and target (5, 6, 10) at (4, 4, 8) tiles.
|
||||
reference_tiles = 1 * 2 * 2
|
||||
target_tiles = 2 * 2 * 2
|
||||
assert meta.video_tile_spans == (
|
||||
(5, 5 + reference_tiles),
|
||||
(5 + reference_tiles, 5 + reference_tiles + target_tiles),
|
||||
)
|
||||
assert meta.num_video_tiles == reference_tiles + target_tiles
|
||||
assert meta.span_sparsities == (0.75, 0.9)
|
||||
# Each kernel block holds at most 64 rows, the lower half first.
|
||||
halves = meta.variable_block_sizes.view(-1, 2)
|
||||
assert torch.all(halves[:, 0] == logical.clamp(max=64))
|
||||
|
||||
|
||||
def test_reference_regions_need_keep_rate_and_exempt() -> None:
|
||||
with pytest.raises(ValueError, match="vsa_ref_keep_rate"):
|
||||
_build(0.9, torch.device("cpu"), ref_keep_rate=None)
|
||||
with pytest.raises(ValueError, match="compete"):
|
||||
_build(0.9, torch.device("cpu"), exempt=False)
|
||||
|
||||
|
||||
def test_ref2va_segments_follow_packed_layout() -> None:
|
||||
blocks = [
|
||||
{"kind": "image", "latent_h": 8, "latent_w": 12},
|
||||
{
|
||||
"kind": "video_audio",
|
||||
"ref_audio_t": 6,
|
||||
"latent_t": 3,
|
||||
"latent_h": 8,
|
||||
"latent_w": 12,
|
||||
},
|
||||
{"kind": "audio", "ref_audio_t": 5},
|
||||
]
|
||||
latent = (4, 10, 16)
|
||||
packed = minimax_h3_packed_sequence_ref2va_blocks(
|
||||
text_len=33,
|
||||
latent_t=latent[0],
|
||||
latent_h=latent[1],
|
||||
latent_w=latent[2],
|
||||
audio_t=20,
|
||||
ref_blocks=blocks,
|
||||
include_video_pos=True,
|
||||
)
|
||||
segments = _ref2va_vsa_segments(packed, latent, (1, 2, 2))
|
||||
assert segments == (33, 0, 24, 12, (3, 4, 6), 10, 40, (4, 5, 8))
|
||||
# Region rows are exactly the layout's sparse-eligible video rows.
|
||||
cursor, region_rows = 0, []
|
||||
for segment in segments:
|
||||
rows = math.prod(segment) if isinstance(segment, tuple) else segment
|
||||
if isinstance(segment, tuple):
|
||||
region_rows.append(torch.arange(cursor, cursor + rows))
|
||||
cursor += rows
|
||||
assert cursor == int(packed["cu_seqlens"][1])
|
||||
assert torch.equal(torch.cat(region_rows), packed["video_pos"])
|
||||
|
||||
|
||||
def _oracle(meta, used, q, k, v, gate):
|
||||
"""Masked dense attention over logical tiles, plus the gated compression."""
|
||||
split = meta.blocks_per_tile
|
||||
tile = meta.tile_elems
|
||||
logical = meta.variable_block_sizes.view(-1, split).sum(-1).float()
|
||||
n = logical.numel()
|
||||
valid = meta.pack_index >= 0
|
||||
src = meta.pack_index.clamp(min=0).long()
|
||||
|
||||
def tiled(x):
|
||||
t = x[src].float().permute(1, 0, 2)
|
||||
return t * valid.to(t.dtype)[None, :, None]
|
||||
|
||||
qt, kt, vt, gt = tiled(q), tiled(k), tiled(v), tiled(gate)
|
||||
pooled = [
|
||||
t.view(HEADS, n, tile, HEAD_DIM).sum(2) / logical[None, :, None]
|
||||
for t in (qt, kt, vt)
|
||||
]
|
||||
scores = pooled[0] @ pooled[1].transpose(-1, -2) / math.sqrt(HEAD_DIM)
|
||||
prefix = meta.num_prefix_tiles
|
||||
mask = torch.zeros(HEADS, n, n, dtype=torch.bool, device=q.device)
|
||||
for (start, end), sparsity in zip(meta.video_tile_spans, meta.span_sparsities):
|
||||
keep = _compute_topk(sparsity, end - start)
|
||||
picked = scores[..., start:end].topk(keep, dim=-1).indices + start
|
||||
mask.scatter_(-1, picked, True)
|
||||
mask[..., :prefix] = True
|
||||
mask[:, :prefix, :] = True
|
||||
row_mask = mask.repeat_interleave(tile, 1).repeat_interleave(tile, 2)
|
||||
row_mask &= valid[None, None, :]
|
||||
logits = qt @ kt.transpose(-1, -2) / math.sqrt(HEAD_DIM)
|
||||
out = torch.softmax(logits.masked_fill(~row_mask, float("-inf")), -1) @ vt
|
||||
compress = torch.softmax(scores, dim=-1) @ pooled[2]
|
||||
out = out + compress.repeat_interleave(tile, 1) * gt
|
||||
return out[:, meta.unpack_index.long()].permute(1, 0, 2)
|
||||
|
||||
|
||||
def _impl():
|
||||
return VideoSparseAttentionH3Impl(
|
||||
num_heads=HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
causal=False,
|
||||
softmax_scale=HEAD_DIM**-0.5,
|
||||
num_kv_heads=HEADS,
|
||||
prefix="blocks.3.attn",
|
||||
)
|
||||
|
||||
|
||||
def _qkvg(device, used, seed=5):
|
||||
total = (used + 63) // 64 * 64
|
||||
generator = torch.Generator(device="cpu").manual_seed(seed)
|
||||
q, k, v, gate = (
|
||||
torch.randn((total, HEADS, HEAD_DIM), generator=generator).to(
|
||||
device=device, dtype=torch.bfloat16
|
||||
)
|
||||
for _ in range(4)
|
||||
)
|
||||
return total, q, k, v, gate * 0.1
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("tile_size", [64, 128])
|
||||
def test_regions_match_masked_dense_oracle(tile_size: int) -> None:
|
||||
device = torch.device("cuda")
|
||||
meta = _build(0.5, device, tile_size=tile_size)
|
||||
used = _used()
|
||||
total, q, k, v, gate = _qkvg(device, used)
|
||||
cu = torch.tensor([0, used, total], dtype=torch.int32, device=device)
|
||||
out = _impl().forward_varlen(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens=cu,
|
||||
max_seqlen=used,
|
||||
cu_seqlens_host=(0, used, total),
|
||||
attn_metadata=meta,
|
||||
gate_compress=gate,
|
||||
)
|
||||
reference = _oracle(meta, used, q, k, v, gate)
|
||||
diff = (out[:used].float() - reference).abs().max().item()
|
||||
assert diff < 2e-2, f"tile {tile_size}: regions vs oracle max diff {diff}"
|
||||
assert torch.all(out[used:] == 0)
|
||||
|
||||
# The deferred (Ulysses) gate fold sees the same per-block compression.
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import (
|
||||
vsa_h3_fold_gate,
|
||||
)
|
||||
|
||||
unfolded, out_compress = _impl().forward_varlen(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens=cu,
|
||||
max_seqlen=used,
|
||||
cu_seqlens_host=(0, used, total),
|
||||
attn_metadata=meta,
|
||||
return_compress=True,
|
||||
)
|
||||
vsa_h3_fold_gate(unfolded, gate, out_compress, meta, 0)
|
||||
assert torch.equal(unfolded, out)
|
||||
Reference in New Issue
Block a user