diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index dac6391b0a..fae799a030 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -946,6 +946,63 @@ the released distillation, not of the SGLang port. Use base MiniMax-H3 when output quality matters more than latency. +### FastH3 Omni: Ref2VA, 8-step PDD + +[FastVideo/FastVideo-FastH3-Omni-8-Step-V1](https://huggingface.co/FastVideo/FastVideo-FastH3-Omni-8-Step-V1) +is a Parallel Decoding Distillation (PDD) student of the MiniMax-H3 Ref2VA +partition: eight denoiser evaluations over a 32-interval grid, with VSA at 0.9 +sparsity on 128-token (4, 4, 8) tiles and each reference video keeping 0.1 of +its tiles. Only `task: "ref2va"` is served, with up to 9 images, 3 videos and 3 +audio clips as ordered references (12 in total), at least one of them visual. + +Pass the repository directly to `--model-path`. Its registered model overlay +maps `transformer_ref/` into the base-H3 layout, fuses the 32 interval output +heads into 8 per-step heads (the math of `tools/fuse_minimax_h3_pdd_heads.py`) +and links the text encoder and VAEs from the `MiniMaxAI/MiniMax-H3` Ref2VA +partition pinned by the release. The served model is the native H3 pipeline: +the release's `pdd_fused_heads` load through the same path as the PDD LoRA +recipe above, and the PDD grid is the 8-step shifted grid scaled to the +release's `sigma_max` of 0.999 (FastGen's `max_t`). + +```bash 2×H200 +sglang serve \ + --model-path FastVideo/FastVideo-FastH3-Omni-8-Step-V1 \ + --num-gpus 2 --ulysses-degree 2 \ + --dit-cpu-offload true --text-encoder-cpu-offload true --vae-cpu-offload true \ + --port 30010 +``` + +```json Request +{ + "model": "FastVideo/FastVideo-FastH3-Omni-8-Step-V1", + "prompt": " is the appearance reference for . [reference generation] ...", + "task": "ref2va", + "conditions": [{"type": "image", "role": "reference", "uri": "/path/to/subject.png"}], + "target": {"short_edge": 480, "aspect_ratio": "16:9", "duration_seconds": 5.0}, + "seed": 0 +} +``` + +Requests default to `num_inference_steps: 8` and clean references +(`imgvid_cond_noise_aug_for_inference` and `audio_cond_noise_aug_for_inference` +of 1.0, the release's target-only flow). On CUDA the transformer selects VSA-H3 +when no backend is given; Ref2VA layouts tile every reference video as its own +region and take a per-region top-k, while text, reference images and audio +rows stay dense. `--attention-backend-config` keys `vsa_tile_size`, +`vsa_ref_keep_rate` and `VSA_sparsity` override the trained values. + +Warm request latency on 2× H200 (Ulysses 2, DiT / text encoder / VAE CPU +offload; median of three requests after one full warmup, 124 frames at 24 FPS +with audio, until the MP4 is saved): + +| References | Short edge | Latency | +| --- | --- | --- | +| Image | 480 | 26.9 s | +| Image + audio | 480 | 27.8 s | +| Video with audio | 480 | 23.2 s | +| Image | 768 | 46.6 s | +| Video with audio | 768 | 37.1 s | + ## 7. VDN-H3: hybrid attention, 8-step distill [Video DeltaNet](https://openvdn.github.io/) (VDN) is a hybrid attention diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py index 26024254a1..833b50de94 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py @@ -60,8 +60,12 @@ class MiniMaxH3PipelineConfig(PipelineConfig): vae_precision: str = "fp32" vae_decode_precision: str = "fp16" audio_vae_precision: str = "fp32" - # Default VSA-H3 sparsity; --attention-backend-config VSA_sparsity overrides. + # Default VSA-H3 sparsity, tile size and reference-video keep rate; + # --attention-backend-config VSA_sparsity / vsa_tile_size / + # vsa_ref_keep_rate override them. vsa_sparsity: float = 0.9 + vsa_tile_size: int = 64 + vsa_ref_keep_rate: float | None = None text_encoder_configs: tuple[MiniMaxH3Qwen3VLConfig, ...] = field( default_factory=lambda: (MiniMaxH3Qwen3VLConfig(),) ) @@ -396,11 +400,41 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig): ) -__all__ = ["FastH3PipelineConfig", "MiniMaxH3PipelineConfig"] +@dataclass +class FastH3OmniPipelineConfig(FastH3PipelineConfig): + """FastH3 Omni 8-Step V1: PDD + VSA-distilled MiniMax-H3, ref2va only.""" + + # The checkpoint's trained VSA (fastvideo_inference.json). + vsa_sparsity: float = 0.9 + vsa_tile_size: int = 128 + vsa_ref_keep_rate: float | None = 0.1 + + def __post_init__(self) -> None: + self.dit_config.arch_config.has_gate_compress = True + # Stack tiling batches every VAE tile in one forward; FastH3 only + # decodes, but Omni also encodes reference videos, which runs out of + # memory stacked. + + def validate_server_args(self, server_args) -> None: + if server_args.model_variant is not None: + raise ValueError( + "FastH3 Omni ships one ref2va-distilled weight partition; " + "--model-variant does not apply." + ) + self._default_to_trained_sparse_attention(server_args) + MiniMaxH3PipelineConfig.validate_server_args(self, server_args) + + +__all__ = [ + "FastH3OmniPipelineConfig", + "FastH3PipelineConfig", + "MiniMaxH3PipelineConfig", +] def register(): from sglang.multimodal_gen.configs.sample.minimax_h3 import ( + FastH3OmniSamplingParams, FastH3SamplingParams, MiniMaxH3SamplingParams, ) @@ -429,6 +463,19 @@ def register(): model_detectors=[ lambda model_id: ( "fasth3" in model_id.lower().replace("-", "").replace("_", "") + and "omni" not in model_id.lower() + ) + ], + ) + register_configs( + sampling_param_cls=FastH3OmniSamplingParams, + pipeline_config_cls=FastH3OmniPipelineConfig, + hf_model_paths=[ + "FastVideo/FastVideo-FastH3-Omni-8-Step-V1", + ], + model_detectors=[ + lambda model_id: ( + "fasth3omni" in model_id.lower().replace("-", "").replace("_", "") ) ], ) diff --git a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py index 1f6dd5dfde..ff6931c294 100644 --- a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py @@ -29,6 +29,9 @@ def _optional_unit_float(value: Any, field_name: str) -> float | None: @dataclass class MiniMaxH3SamplingParams(SamplingParams): min_num_inference_steps: ClassVar[int] = 1 + # Single-partition releases name their partition; None infers it from + # --model-variant or the model path. + release_partition: ClassVar[str | None] = None height: int = 512 width: int = 896 @@ -110,7 +113,9 @@ class MiniMaxH3SamplingParams(SamplingParams): file delivery, while the public H3 contract correctly requires it. """ selected_variant = getattr(server_args, "model_variant", None) - if selected_variant is not None: + if self.release_partition is not None: + selected_partition = self.release_partition + elif selected_variant is not None: selected_partition = str(selected_variant).strip().lower() else: selected_path = server_args.model_subfolder or server_args.model_path @@ -378,4 +383,33 @@ class FastH3SamplingParams(MiniMaxH3SamplingParams): ) -__all__ = ["FastH3SamplingParams", "MiniMaxH3SamplingParams"] +@dataclass +class FastH3OmniSamplingParams(MiniMaxH3SamplingParams): + """FastH3 Omni 8-Step V1: eight PDD forwards over ordered references.""" + + release_partition: ClassVar[str | None] = "ref2va" + num_inference_steps: int = 8 + task: str | None = "ref2va" + quality: str = "lossless" + # Trained with target-only flow: references stay clean. + imgvid_cond_noise_aug_for_inference: float | None = 1.0 + audio_cond_noise_aug_for_inference: float | None = 1.0 + + def _validate(self) -> None: + super()._validate() + if self.num_inference_steps != 8: + raise ValueError( + "FastH3 Omni is distilled for exactly eight PDD forwards; got " + f"num_inference_steps={self.num_inference_steps}." + ) + if self.task is not None and self.task.strip().lower() != "ref2va": + raise ValueError( + f"FastH3 Omni is distilled for ref2va only (got task={self.task!r})." + ) + + +__all__ = [ + "FastH3OmniSamplingParams", + "FastH3SamplingParams", + "MiniMaxH3SamplingParams", +] diff --git a/python/sglang/multimodal_gen/registry.py b/python/sglang/multimodal_gen/registry.py index 8115ff88c3..65c4d0a896 100644 --- a/python/sglang/multimodal_gen/registry.py +++ b/python/sglang/multimodal_gen/registry.py @@ -189,6 +189,7 @@ KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = { "minimaxai/minimax-h3": "MiniMaxH3Pipeline", "minimax/minimax-h3": "MiniMaxH3Pipeline", "fastvideo/fastvideo-fasth3-8-step-v2": "FastH3Pipeline", + "fastvideo/fastvideo-fasth3-omni-8-step-v1": "FastH3Pipeline", "openvdn/vdn-minimax-h3": "VDNH3Pipeline", "lerobot/pi05": "Pi05Pipeline", "pi05": "Pi05Pipeline", diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py index 6ae93233e2..b88d894cc3 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn.py @@ -66,6 +66,7 @@ def construct_variable_block_sizes( dit_seq_shape: tuple[int, int, int], num_tiles: tuple[int, int, int], device: torch.device, + tile_size: tuple[int, int, int] = VSA_TILE_SIZE, ) -> torch.LongTensor: """ Compute the number of valid (non‑padded) tokens inside every @@ -78,7 +79,7 @@ def construct_variable_block_sizes( """ # unpack t, h, w = dit_seq_shape - ts_t, ts_h, ts_w = VSA_TILE_SIZE + ts_t, ts_h, ts_w = tile_size n_t, n_h, n_w = num_tiles def _sizes(dim_len: int, tile: int, n_tiles: int) -> torch.LongTensor: diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py index 332c1d04a8..b7e57d1e82 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/video_sparse_attn_h3.py @@ -19,6 +19,14 @@ under a FLOP-matched budget ("compete"). The compression branch is gated by ``to_gate_compress``: base H3 has no such weights and the gate loads as zeros (pure sparse); VSA-distilled students (FastH3) ship trained gates that activate the branch. + +Ref2VA interleaves reference videos with dense rows. Each reference video is +its own 3-D tiled region; every video query keeps its own top-k of EACH +region (references at ``ref_keep_rate``, the generated video at the step +sparsity), FastVideo's "p2_multi_region" policy. Checkpoints trained on +128-token (4, 4, 8) tiles select and compress at that logical granularity; +each logical tile runs as two contiguous 64-token kernel tiles, so the +selected key set is unchanged. """ import functools @@ -50,83 +58,121 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels impo ) from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum -# The FastH3 checkpoints are trained and served at the 64-token (4, 4, 4) -# tile geometry; the kernel block size matches it exactly. +# The kernels run 64-token blocks. FastH3 is trained on (4, 4, 4) tiles; +# FastH3 Omni on 128-token (4, 4, 8) tiles, each served as two kernel blocks. VSA_H3_TILE_SHAPE = (4, 4, 4) VSA_H3_TILE_ELEMS = math.prod(VSA_H3_TILE_SHAPE) +VSA_H3_TILE_SHAPES = {64: VSA_H3_TILE_SHAPE, 128: (4, 4, 8)} _DIT_BLOCK_PREFIX = re.compile(r"^blocks\.(\d+)\.") +PackedSegment = int | tuple[int, int, int] + @functools.lru_cache(maxsize=8) +def _h3_segment_tile_geometry( + segments: tuple[PackedSegment, ...], + tile_shape: tuple[int, int, int], + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, tuple[tuple[int, int], ...]]: + """Tile a packed sequence of interleaved dense segments and video regions. + + ``segments`` lists the packed order: an ``int`` is a dense segment's row + count, a ``(t, h, w)`` triple one video region's token grid. Dense + segments are chunked into the leading prefix tiles (never straddling a + segment boundary); each region follows as a contiguous run of 3-D tiles. + + Returns, at kernel (64-token block) granularity, (variable_block_sizes + int32 [n_blocks], pack_index int32 [S_pad]: padded position -> packed row + or -1, unpack_index int32 [used]: packed row -> padded position), then + num_prefix_tiles and the per-region [start, end) spans in logical tiles. + A 128-token logical tile is two consecutive kernel blocks; its upper half + is empty when the tile holds at most 64 rows. + """ + tile_elems = math.prod(tile_shape) + split = tile_elems // VSA_H3_TILE_ELEMS + + prefix_sizes: list[int] = [] + prefix_rows: list[torch.Tensor] = [] + video_entries: list[tuple[int, tuple[int, int, int]]] = [] + cursor = 0 + for segment in segments: + if isinstance(segment, tuple): + video_entries.append((cursor, segment)) + cursor += math.prod(segment) + continue + full, remainder = divmod(segment, tile_elems) + prefix_sizes.extend([tile_elems] * full) + if remainder: + prefix_sizes.append(remainder) + prefix_rows.append( + torch.arange(cursor, cursor + segment, device=device, dtype=torch.long) + ) + cursor += segment + num_prefix_tiles = len(prefix_sizes) + + video_rows: list[torch.Tensor] = [] + size_parts = [torch.tensor(prefix_sizes, dtype=torch.long, device=device)] + spans: list[tuple[int, int]] = [] + tile_cursor = num_prefix_tiles + for start, grid in video_entries: + num_tiles = tuple(math.ceil(n / t) for n, t in zip(grid, tile_shape)) + sizes = construct_variable_block_sizes(grid, num_tiles, device, tile_shape) + video_rows.append(get_tile_partition_indices(grid, tile_shape, device) + start) + size_parts.append(sizes.to(torch.long)) + spans.append((tile_cursor, tile_cursor + int(sizes.numel()))) + tile_cursor += int(sizes.numel()) + + tile_partition_indices = torch.cat(prefix_rows + video_rows) + logical_sizes = torch.cat(size_parts) + non_pad_index = get_non_pad_index(logical_sizes, tile_elems) + if ( + int(logical_sizes.sum()) != cursor + or non_pad_index.numel() != cursor + or tile_partition_indices.numel() != cursor + ): + raise ValueError( + f"VSA-H3 tile geometry mismatch for segments={segments}, " + f"tile={tile_shape}: sizes sum {int(logical_sizes.sum())}, non-pad " + f"{non_pad_index.numel()}, expected {cursor}." + ) + + # A logical tile's padded slots are its kernel blocks' slots in order, + # so the pack/unpack maps are shared; only the per-block sizes split. + seq_pad = logical_sizes.numel() * tile_elems + pack_index = torch.full((seq_pad,), -1, dtype=torch.int32, device=device) + pack_index[non_pad_index] = tile_partition_indices.to(torch.int32) + unpack_index = torch.empty(cursor, dtype=torch.int32, device=device) + unpack_index[tile_partition_indices] = non_pad_index.to(torch.int32) + offsets = torch.arange(split, device=device) * VSA_H3_TILE_ELEMS + block_sizes = ( + (logical_sizes[:, None] - offsets).clamp(0, VSA_H3_TILE_ELEMS).flatten() + ) + return ( + block_sizes.to(torch.int32), + pack_index, + unpack_index, + num_prefix_tiles, + tuple(spans), + ) + + def _h3_tile_geometry( prefix_segments: tuple[int, ...], dit_seq_shape: tuple[int, int, int], device: torch.device, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]: - """Segment-pure prefix chunks, then video tiles. + """Segment-pure prefix chunks, then the generated video's 64-token tiles. - Returns (variable_block_sizes int32 [n_tiles], pack_index int32 [S_pad]: - padded position -> packed row or -1, unpack_index int32 [used]: packed row - -> padded position, num_prefix_tiles, num_video_tiles). + Returns (variable_block_sizes, pack_index, unpack_index, + num_prefix_tiles, num_video_tiles). """ - prefix_len = sum(prefix_segments) - - prefix_sizes: list[int] = [] - for segment in prefix_segments: - full, remainder = divmod(segment, VSA_H3_TILE_ELEMS) - prefix_sizes.extend([VSA_H3_TILE_ELEMS] * full) - if remainder: - prefix_sizes.append(remainder) - num_prefix_tiles = len(prefix_sizes) - - num_tiles = ( - math.ceil(dit_seq_shape[0] / VSA_H3_TILE_SHAPE[0]), - math.ceil(dit_seq_shape[1] / VSA_H3_TILE_SHAPE[1]), - math.ceil(dit_seq_shape[2] / VSA_H3_TILE_SHAPE[2]), - ) - video_sizes = construct_variable_block_sizes(dit_seq_shape, num_tiles, device) - num_video_tiles = int(video_sizes.numel()) - - video_indices = ( - get_tile_partition_indices(dit_seq_shape, VSA_H3_TILE_SHAPE, device) - + prefix_len - ) - tile_partition_indices = torch.cat( - [ - torch.arange(prefix_len, device=device, dtype=torch.long), - video_indices, - ] - ) - variable_block_sizes = torch.cat( - [ - torch.tensor(prefix_sizes, dtype=torch.long, device=device), - video_sizes.to(torch.long), - ] - ) - non_pad_index = get_non_pad_index(variable_block_sizes, VSA_H3_TILE_ELEMS) - - total = prefix_len + math.prod(dit_seq_shape) - sizes_sum = int(variable_block_sizes.sum()) - if sizes_sum != total or non_pad_index.numel() != total: - raise ValueError( - f"VSA-H3 tile geometry mismatch for prefix={prefix_segments}, " - f"video={dit_seq_shape}: sizes sum {sizes_sum}, non-pad " - f"{non_pad_index.numel()}, expected {total}." + sizes, pack_index, unpack_index, num_prefix_tiles, spans = ( + _h3_segment_tile_geometry( + (*prefix_segments, tuple(dit_seq_shape)), VSA_H3_TILE_SHAPE, device ) - - seq_pad = variable_block_sizes.numel() * VSA_H3_TILE_ELEMS - pack_index = torch.full((seq_pad,), -1, dtype=torch.int32, device=device) - pack_index[non_pad_index] = tile_partition_indices.to(torch.int32) - unpack_index = torch.empty(total, dtype=torch.int32, device=device) - unpack_index[tile_partition_indices] = non_pad_index.to(torch.int32) - return ( - variable_block_sizes.to(torch.int32), - pack_index, - unpack_index, - num_prefix_tiles, - num_video_tiles, ) + return sizes, pack_index, unpack_index, num_prefix_tiles, spans[0][1] - spans[0][0] class VideoSparseAttentionH3Backend(AttentionBackend): @@ -160,16 +206,32 @@ class VideoSparseAttentionH3Metadata(AttentionMetadata): num_prefix_tiles: int num_video_tiles: int exempt: bool + # Kernel (64-token block) granularity; a 128-token logical tile is two + # consecutive blocks. Tile counts above are logical. variable_block_sizes: torch.Tensor pack_index: torch.Tensor unpack_index: torch.Tensor dense_layers: tuple[int, ...] = () workspace_cache: dict = field(default_factory=dict) + tile_elems: int = VSA_H3_TILE_ELEMS + # Logical-tile [start, end) per video region, the generated video last, + # with each region's sparsity. None: the generated video alone, at + # VSA_sparsity. + video_tile_spans: tuple[tuple[int, int], ...] | None = None + span_sparsities: tuple[float, ...] | None = None @property def num_tiles(self) -> int: return self.num_prefix_tiles + self.num_video_tiles + @property + def blocks_per_tile(self) -> int: + return self.tile_elems // VSA_H3_TILE_ELEMS + + @property + def uses_regions(self) -> bool: + return self.video_tile_spans is not None + class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder): def __init__(self) -> None: @@ -186,14 +248,50 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder): exempt: bool = True, dense_layers: tuple[int, ...] = (), dense_first_n_steps: int = 0, + packed_segments: tuple[PackedSegment, ...] | None = None, + tile_size: int = VSA_H3_TILE_ELEMS, + ref_keep_rate: float | None = None, **kwargs: dict[str, Any], ) -> VideoSparseAttentionH3Metadata: + """``packed_segments`` replaces ``prefix_segments`` for layouts with + reference videos: the packed order of dense row counts and video + token grids, ending with the generated video's grid. Every earlier + grid is a reference region kept at ``ref_keep_rate``.""" dit_seq_shape = ( raw_latent_shape[0] // patch_size[0], raw_latent_shape[1] // patch_size[1], raw_latent_shape[2] // patch_size[2], ) - prefix_segments = tuple(int(s) for s in prefix_segments if s > 0) + tile_shape = VSA_H3_TILE_SHAPES.get(int(tile_size)) + if tile_shape is None: + raise ValueError( + f"VSA-H3 tile_size must be one of {sorted(VSA_H3_TILE_SHAPES)}, " + f"got {tile_size!r}" + ) + if packed_segments is None: + packed_segments = (*prefix_segments, dit_seq_shape) + segments = tuple( + tuple(int(v) for v in s) if isinstance(s, tuple) else int(s) + for s in packed_segments + if isinstance(s, tuple) or s > 0 + ) + if not segments or segments[-1] != dit_seq_shape: + raise ValueError( + f"VSA-H3 packed segments must end with the generated video " + f"{dit_seq_shape}, got {segments}" + ) + num_references = sum(isinstance(s, tuple) for s in segments) - 1 + if num_references: + if ref_keep_rate is None or not 0.0 < float(ref_keep_rate) <= 1.0: + raise ValueError( + f"VSA-H3 with {num_references} reference-video region(s) " + f"needs vsa_ref_keep_rate in (0, 1], got {ref_keep_rate!r}" + ) + if not exempt: + raise ValueError( + "vsa_mode='compete' supports only the generated-video " + "region; use vsa_mode='exempt' with reference videos" + ) if current_timestep < dense_first_n_steps: VSA_sparsity = 0.0 @@ -202,21 +300,31 @@ class VideoSparseAttentionH3MetadataBuilder(AttentionMetadataBuilder): pack_index, unpack_index, num_prefix_tiles, - num_video_tiles, - ) = _h3_tile_geometry(prefix_segments, dit_seq_shape, device) + spans, + ) = _h3_segment_tile_geometry(segments, tile_shape, device) + regional = num_references > 0 or tile_shape != VSA_H3_TILE_SHAPE + span_sparsities = None + if regional: + span_sparsities = (1.0 - float(ref_keep_rate or 1.0),) * num_references + span_sparsities += (float(VSA_sparsity),) return VideoSparseAttentionH3Metadata( current_timestep=current_timestep, VSA_sparsity=float(VSA_sparsity), - total_seq_length=sum(prefix_segments) + math.prod(dit_seq_shape), + total_seq_length=sum( + math.prod(s) if isinstance(s, tuple) else s for s in segments + ), num_prefix_tiles=num_prefix_tiles, - num_video_tiles=num_video_tiles, + num_video_tiles=spans[-1][1] - num_prefix_tiles, exempt=exempt, variable_block_sizes=variable_block_sizes, pack_index=pack_index, unpack_index=unpack_index, dense_layers=tuple(int(layer) for layer in dense_layers), workspace_cache=self._workspace_cache, + tile_elems=math.prod(tile_shape), + video_tile_spans=spans if regional else None, + span_sparsities=span_sparsities, ) @@ -225,6 +333,50 @@ def _compute_topk(sparsity: float, num_video_tiles: int) -> int: return max(1, min(keep, num_video_tiles)) +def _region_tile_lists( + scores: torch.Tensor, + num_prefix_tiles: int, + spans: tuple[tuple[int, int], ...], + sparsities: tuple[float, ...], +) -> torch.Tensor: + """scores [H, n_tiles, n_tiles] -> ascending int32 logical kv-tile lists + [H, num_video_tiles, width]: every prefix tile, then each region's own + top-k (the whole region when its budget covers it).""" + prefix = num_prefix_tiles + video_rows = scores[:, prefix:, :] + parts = [ + torch.arange(prefix, device=scores.device).expand( + *video_rows.shape[:-1], prefix + ) + ] + for (start, end), sparsity in zip(spans, sparsities): + keep = _compute_topk(sparsity, end - start) + if keep == end - start: + parts.append( + torch.arange(start, end, device=scores.device).expand( + *video_rows.shape[:-1], keep + ) + ) + continue + picked = video_rows[..., start:end].topk(keep, dim=-1).indices + parts.append(picked.sort(dim=-1).values + start) + return torch.cat(parts, dim=-1).to(torch.int32) + + +def _pool_logical_tiles( + pooled: torch.Tensor, meta: VideoSparseAttentionH3Metadata +) -> torch.Tensor: + """Kernel-block means [3, H, n_blocks, D] -> logical-tile means.""" + split = meta.blocks_per_tile + if split == 1: + return pooled + sizes = meta.variable_block_sizes.view(-1, split).to(torch.float32) + blocks = pooled.unflatten(2, (-1, split)) + # Empty upper halves pool to 0/0; they carry no weight. + weighted = torch.where(sizes[..., None] > 0, blocks * sizes[..., None], 0.0) + return weighted.sum(3) / sizes.sum(-1)[:, None] + + def _topk_tile_lists( scores: torch.Tensor, num_prefix_tiles: int, @@ -266,6 +418,8 @@ def _workspace_key( meta.num_tiles, meta.num_prefix_tiles, meta.exempt, + meta.tile_elems, + meta.uses_regions, heads, head_dim, has_gate, @@ -276,10 +430,11 @@ def _workspace_key( class _Workspace: """Per-geometry scratch: tiled q/k/v(/gate) [3|4, H, S_pad, D], pooled fp32 - tile means [3, H, n_tiles, D], and the kernel index lists (prefix rows and - prefix columns are static; only the top-k video columns change per layer). - The native kernel pairs query tiles, so the tile axis is padded to an even - count with one empty tile. + block means [3, H, n_blocks, D], and the kernel index lists (prefix rows + and prefix columns are static; only the top-k video columns change per + layer). All of it is at kernel-block granularity. The native kernel pairs + query blocks, so the block axis is padded to an even count with one empty + block. """ def __init__( @@ -291,7 +446,7 @@ class _Workspace: dtype: torch.dtype, device: torch.device, ) -> None: - n_tiles = meta.num_tiles + n_tiles = int(meta.variable_block_sizes.numel()) self.n_tiles = n_tiles self.use_sm100_kernel = can_use_vsa_block_sparse_sm100( device.index, dtype, head_dim @@ -321,7 +476,7 @@ class _Workspace: self.dense_num[:, :n_tiles] = n_tiles self.q2k_index = self.dense_index.clone() self.q2k_num = self.dense_num.clone() - num_video_tiles = meta.num_video_tiles + num_video_tiles = 0 if meta.uses_regions else meta.num_video_tiles # Zeroed scratch for the exempt top-k compaction; kept zero between uses. self.topk_mask = torch.zeros( (heads, num_video_tiles, num_video_tiles), dtype=torch.int8, device=device @@ -339,7 +494,7 @@ class _Workspace: def _gate_tile_index( meta: VideoSparseAttentionH3Metadata, row_start: int, num_rows: int ) -> torch.Tensor: - """Tile of each packed row in the shard, -1 for pad rows.""" + """Kernel block of each packed row in the shard, -1 for pad rows.""" key = ("gate_tile_index", row_start, num_rows) tile_index = meta.workspace_cache.get(key) if tile_index is None: @@ -390,6 +545,8 @@ def _select_kv_lists( sparsity: float, ) -> tuple[torch.Tensor, torch.Tensor]: """Kernel index lists for this layer: dense, or top-k video columns.""" + if meta.uses_regions: + return _select_region_kv_lists(ws, meta, scores, sparsity) keep = _compute_topk(sparsity, meta.num_video_tiles) if sparsity <= 0.0 or keep >= meta.num_video_tiles: return ws.dense_index, ws.dense_num @@ -414,6 +571,36 @@ def _select_kv_lists( ) +def _select_region_kv_lists( + ws: _Workspace, + meta: VideoSparseAttentionH3Metadata, + scores: torch.Tensor | None, + sparsity: float, +) -> tuple[torch.Tensor, torch.Tensor]: + """Per-region top-k over logical tiles, expanded to kernel blocks.""" + spans = meta.video_tile_spans + sparsities = meta.span_sparsities + # Dense steps and layers keep every region dense. + if sparsity <= 0.0: + return ws.dense_index, ws.dense_num + if all( + _compute_topk(s, end - start) == end - start + for (start, end), s in zip(spans, sparsities) + ): + return ws.dense_index, ws.dense_num + lists = _region_tile_lists(scores, meta.num_prefix_tiles, spans, sparsities) + split = meta.blocks_per_tile + if split > 1: + offsets = torch.arange(split, device=lists.device, dtype=lists.dtype) + lists = (lists[..., None] * split + offsets).flatten(-2) + lists = lists.repeat_interleave(split, dim=1) + prefix_blocks = meta.num_prefix_tiles * split + width = lists.shape[-1] + ws.q2k_index[:, prefix_blocks : ws.n_tiles, :width] = lists + ws.q2k_num[:, prefix_blocks : ws.n_tiles] = width + return ws.q2k_index, ws.q2k_num + + class VideoSparseAttentionH3Impl(AttentionImpl): def __init__( self, @@ -524,7 +711,7 @@ class VideoSparseAttentionH3Impl(AttentionImpl): ws.tiled, ws.pooled, ) - q_pooled, k_pooled, v_pooled = ws.pooled + q_pooled, k_pooled, v_pooled = _pool_logical_tiles(ws.pooled, meta) scores = None if sparsity > 0.0 or needs_compress: @@ -547,6 +734,12 @@ class VideoSparseAttentionH3Impl(AttentionImpl): out_compress = None if needs_compress: out_compress = torch.matmul(torch.softmax(scores, dim=-1), v_pooled) + if meta.blocks_per_tile > 1: + # The gate folds per kernel block; both halves of a logical + # tile share its compressed row. + out_compress = out_compress.repeat_interleave( + meta.blocks_per_tile, dim=1 + ) result = torch.empty(query.shape, dtype=query.dtype, device=query.device) vsa_h3_untile( diff --git a/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py index d8f80d7853..6b697ccae2 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py @@ -1,9 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +import os import shutil from dataclasses import replace +from sglang.multimodal_gen import envs from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import ( MiniMaxH3PipelineConfig, ) @@ -140,6 +142,25 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase): f"disaggregation role {role.value!r} is not supported" ) + def initialize_pipeline(self, server_args: ServerArgs) -> None: + self.pdd_heads = self._resolve_pdd_heads() + transformer = self.get_module("transformer") + if ( + self.pdd_heads + and transformer is not None + and transformer.final_layer._pdd_heads is None + ): + transformer.final_layer.load_pdd_fused_heads(self.pdd_heads) + + def _resolve_pdd_heads(self) -> str | None: + """The env override, else a PDD release's own fused heads.""" + if envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS: + return envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS + release_metadata = getattr(self, "release_metadata", None) + if release_metadata is None or release_metadata.pdd_fused_heads is None: + return None + return os.path.join(self.model_path, release_metadata.pdd_fused_heads) + def create_pipeline_stages(self, server_args: ServerArgs) -> None: # Per-model sigma override from model_index.json; contract tests # construct the pipeline without model_path, hence the guard. @@ -180,6 +201,7 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase): if release_metadata is not None else None ), + pdd_heads=getattr(self, "pdd_heads", None), ) ) self.add_stage( @@ -206,7 +228,8 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase): class FastH3Pipeline(MiniMaxH3Pipeline): - """FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only). + """FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only), and the + FastH3 Omni Ref2VA PDD student. The flat single-partition repo is materialized into the base-H3 layout by its registered model overlay (see runtime/utils/model_overlay.py), so every stage, loader, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/release_metadata.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/release_metadata.py index af41661a01..81768aab81 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/release_metadata.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/release_metadata.py @@ -4,6 +4,7 @@ from __future__ import annotations import math +import os from dataclasses import dataclass from typing import Any, Mapping @@ -54,6 +55,9 @@ class MiniMaxH3ReleaseMetadata: audio_sigma_shift: float # trained DMD rungs of a distilled release, unshifted; None keeps the uniform grid dmd_denoising_steps: tuple[int, ...] | None = None + # PDD release: fused per-step output heads, relative to the model dir, + # with pdd_config.json beside them + pdd_fused_heads: str | None = None @classmethod def from_model_index( @@ -109,6 +113,17 @@ class MiniMaxH3ReleaseMetadata: "strictly decreasing integers in (0, 1000]" ) dmd_steps = tuple(dmd_steps) + pdd_fused_heads = raw.get("pdd_fused_heads") + if pdd_fused_heads is not None and ( + not isinstance(pdd_fused_heads, str) + or not pdd_fused_heads.endswith(".safetensors") + or os.path.isabs(pdd_fused_heads) + or ".." in pdd_fused_heads.split("/") + ): + raise ValueError( + "model_index.json._minimax_h3.pdd_fused_heads must be a relative " + ".safetensors path inside the model directory" + ) metadata = cls( schema_version=1, partition=partition, @@ -117,6 +132,7 @@ class MiniMaxH3ReleaseMetadata: video_sigma_shift=video_sigma, audio_sigma_shift=audio_sigma, dmd_denoising_steps=dmd_steps, + pdd_fused_heads=pdd_fused_heads, ) for task in metadata.tasks: if canonical_minimax_h3_task(task) != task: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py index e22cec2525..a44c6f6c8d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py @@ -552,6 +552,7 @@ def comfyui_payload_to_branch_inputs( "orig_video_shape": orig_shape, "padded_video_shape": tuple(video.shape), "is_ref2va": bool(refs), + "native_layout": layout is None, "sample_sigmas": sample_sigmas, "sigma_v": float(sigma_v), "shift_v": shift_v, @@ -669,6 +670,7 @@ def _step_attn_metadata( latent_shape=latent_shape, server_args=server_args, device=device, + native_layout=inputs["native_layout"], ) if build is not None: return build diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index d4c9e3d71f..33402f56f4 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -5,6 +5,7 @@ single-branch execution, and payload validation. from __future__ import annotations +import math from collections.abc import Callable, Mapping from contextlib import contextmanager from functools import partial @@ -778,6 +779,10 @@ class MiniMaxH3DenoisingStage(DenoisingStage): device=device, ) initial_video, initial_audio = _expand_initial_rows(ctx, positive) + if model.final_layer._pdd_heads is not None: + initial_video, initial_audio = _scale_pdd_initial_noise( + ctx, positive, initial_video, initial_audio + ) rollout_ctx = None if getattr(batch, "rollout", False): from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.minimax_h3_rollout import ( @@ -1049,30 +1054,31 @@ def _maybe_prepare_vsa_h3_step_metadata( latent_shape: tuple[int, int, int], server_args: ServerArgs, device: torch.device, + native_layout: bool = True, ) -> Callable[[int], Any] | None: """Per-step VSA-H3 metadata builder over the request-static packed layout, - or None off the VSA path.""" + or None off the VSA path. + + Ref2VA regions are read from the native packed layout's stream_layout; + ``native_layout=False`` marks a layout serialized by ComfyUI, whose + stream_layout does not tell reference images from reference videos.""" model._resolve_attention_backend_once() if ( model._resolved_attention_backend is not AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3 ): return None - if is_ref2va: + if is_ref2va and not native_layout: raise NotImplementedError( - "VSA-H3 supports the t2va/fl2va packed layout; the ref2va " - "reference-block layout is not tiled yet. Use --attention-backend " - "fa for ref2va." + "VSA-H3 tiles Ref2VA references on SGLang's native packed layout; " + "a ComfyUI-serialized Ref2VA layout is not tiled. Use " + "--attention-backend fa for it." ) - config = server_args.attention_backend_config or {} - tile_size = int(config.get("vsa_tile_size", 64)) - if tile_size != 64: - raise ValueError( - "VSA-H3 in SGLang serves the trained 64-token (4, 4, 4) tile " - f"geometry; got vsa_tile_size={tile_size}." - ) - default_sparsity = server_args.pipeline_config.vsa_sparsity + pipeline_config = server_args.pipeline_config + tile_size = int(config.get("vsa_tile_size", pipeline_config.vsa_tile_size)) + ref_keep_rate = config.get("vsa_ref_keep_rate", pipeline_config.vsa_ref_keep_rate) + default_sparsity = pipeline_config.vsa_sparsity sparsity = float( config.get("VSA_sparsity", config.get("sparsity", default_sparsity)) ) @@ -1089,6 +1095,9 @@ def _maybe_prepare_vsa_h3_step_metadata( cond_rows = int(packed["img_pos"].numel()) - video_rows audio_rows = int(packed["audio_pos"].numel()) patch_size = server_args.pipeline_config.dit_config.arch_config.patch_size + packed_segments = ( + _ref2va_vsa_segments(packed, latent_shape, patch_size) if is_ref2va else None + ) from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import ( VideoSparseAttentionH3MetadataBuilder, @@ -1107,11 +1116,44 @@ def _maybe_prepare_vsa_h3_step_metadata( exempt=mode == "exempt", dense_layers=dense_layers, dense_first_n_steps=dense_first_n_steps, + packed_segments=packed_segments, + tile_size=tile_size, + ref_keep_rate=None if ref_keep_rate is None else float(ref_keep_rate), ) return build +def _ref2va_vsa_segments( + packed: Mapping[str, Any], + latent_shape: tuple[int, int, int], + patch_size: tuple[int, int, int], +) -> tuple[int | tuple[int, int, int], ...]: + """Ref2VA packed order as VSA-H3 segments: text, keyframes, then each + reference (images and audio dense, videos as their own token-grid + regions), target audio, and the generated video's token grid last.""" + layout = packed["stream_layout"] + shapes = layout["cond_image_shapes"] + roles = layout["cond_image_roles"] + audio_lens = layout["cond_audio_stream_lens"] + reference_visual_rows = sum(math.prod(shape) for shape in shapes) + cond_rows = int(packed["img_pos"].numel()) - int(packed["update_mask"].sum()) + segments: list[int | tuple[int, int, int]] = [ + int(packed["text_pos"].numel()), + cond_rows - reference_visual_rows, + ] + for kind, index in layout["cond_event_orders"]: + if kind == "audio": + segments.append(int(audio_lens[index])) + elif roles[index] == "independent_cube": + segments.append(tuple(int(v) for v in shapes[index])) + else: + segments.append(math.prod(shapes[index])) + segments.append(int(packed["audio_pos"].numel()) - sum(audio_lens)) + segments.append(tuple(int(n) // int(p) for n, p in zip(latent_shape, patch_size))) + return tuple(segments) + + def _build_packed_layout( ctx: _FullLoopContext, emb: Mapping[str, Any], @@ -1267,6 +1309,22 @@ def _expand_initial_rows( return initial_video, initial_audio +def _scale_pdd_initial_noise( + ctx: _FullLoopContext, + positive: Any, + initial_video: torch.Tensor, + initial_audio: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Start PDD targets at the first node's noise level, as the students are + trained (``sigma_0 * eps``); condition rows stay as they are. A no-op on + grids that start at sigma 1.""" + initial_video = initial_video.clone() + initial_audio = initial_audio.clone() + initial_video[positive.video_target_slice] *= float(ctx.sigmas["video"][0]) + initial_audio[positive.audio_target_slice] *= float(ctx.sigmas["audio"][0]) + return initial_video, initial_audio + + def _publish_full_loop_outputs( ctx: _FullLoopContext, *, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py index fc027eea4b..d7391acba2 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py @@ -30,6 +30,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): self, sigma_shift_scales=None, dmd_denoising_steps: tuple[int, ...] | None = None, + pdd_heads: str | None = None, ) -> None: super().__init__() # Per-model sigma shift override (model_index.json "_minimax_h3" release @@ -38,7 +39,9 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): self.sigma_shift_scales = sigma_shift_scales self.dmd_denoising_steps = dmd_denoising_steps self._pdd_config = None - pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS + # The pipeline resolves the release's fused heads; the env var serves + # stages built without a pipeline. + pdd_heads = pdd_heads or envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS if pdd_heads: from safetensors import safe_open @@ -49,6 +52,9 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): steps = f.get_slice("video_out.weight").get_shape()[0] if self._pdd_config["num_inference_steps"] != steps: raise ValueError("MiniMax-H3 PDD config does not match the fused heads") + sigma_max = float(self._pdd_config.get("sigma_max", 1.0)) + if not 0.0 < sigma_max <= 1.0: + raise ValueError("MiniMax-H3 PDD sigma_max must be in (0, 1]") def forward(self, batch: Req, server_args: ServerArgs) -> Req: from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.resolved_plan import ( @@ -95,6 +101,10 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): ) config = self._pdd_config + # Students trained on a grid ending at sigma_max < 1 (FastGen's + # max_t) sample the [1, 0] grid scaled by it. The scale is shared by + # every interval, so the fused heads' in-block weights are unchanged. + sigma_max = float(config.get("sigma_max", 1.0)) sigmas = batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] for modality in ("video", "audio"): expected = minimax_h3_time_shift_sigmas( @@ -104,18 +114,20 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): if batch.is_warmup: # Warmup may run fewer steps, but must use the same intervals # as serving rather than rescaling a shorter grid to [1, 0]. - sigmas[modality] = expected[: batch.num_inference_steps + 1] - continue - actual = sigmas[modality] - if len(actual) != len(expected) or any( - not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7) - for a, b in zip(actual, expected) - ): - raise ValueError( - f"MiniMax-H3 PDD requires num_inference_steps=" - f"{config['num_inference_steps']} and {modality} shift=" - f"{config[f'{modality}_shift']} to match the fused heads" - ) + expected = expected[: batch.num_inference_steps + 1] + else: + actual = sigmas[modality] + if len(actual) != len(expected) or any( + not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7) + for a, b in zip(actual, expected) + ): + raise ValueError( + f"MiniMax-H3 PDD requires num_inference_steps=" + f"{config['num_inference_steps']} and {modality} shift=" + f"{config[f'{modality}_shift']} to match the fused heads" + ) + if batch.is_warmup or sigma_max != 1.0: + sigmas[modality] = [sigma * sigma_max for sigma in expected] @staticmethod def _publish_native_timestep_state(batch: Req) -> None: diff --git a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py index f72775b025..9835b020e0 100644 --- a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py +++ b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py @@ -53,6 +53,11 @@ BUILTIN_MODEL_OVERLAY_REGISTRY: dict[str, dict[str, Any]] = { "overlay_repo_id": "kevin-mi/VDN-H3-overlay", "overlay_revision": "7de18275dddfe59da36a234e222bcdd274963bc3", }, + "FastVideo/FastVideo-FastH3-Omni-8-Step-V1": { + "overlay_repo_id": "Niehen6174/FastH3-Omni-overlay", + "overlay_revision": "953ede55daf20a70cc6f0b59464293e2f408608c", + "source_revision": "53e0d26a1691aa473b6262d8196a8feb2c297df0", + }, } diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py index 7ad3c920c9..59f294de0b 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py @@ -353,7 +353,10 @@ def test_vsa_h3_tiles_cover_comfyui_packed_rows(keyframes): server_args = SimpleNamespace( attention_backend_config={}, pipeline_config=SimpleNamespace( - vsa_sparsity=0.8, dit_config=MiniMaxH3DiTConfig() + vsa_sparsity=0.8, + vsa_tile_size=64, + vsa_ref_keep_rate=None, + dit_config=MiniMaxH3DiTConfig(), ), ) build = _maybe_prepare_vsa_h3_step_metadata( @@ -374,6 +377,32 @@ def test_vsa_h3_tiles_cover_comfyui_packed_rows(keyframes): assert torch.equal(packed["text_pos"], torch.arange(text_len)) +def test_vsa_h3_rejects_comfyui_serialized_ref2va_layout(): + """ComfyUI-serialized Ref2VA layouts do not tell reference images from + reference videos, so VSA-H3 refuses them instead of mis-tiling.""" + from types import SimpleNamespace + + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import ( + _maybe_prepare_vsa_h3_step_metadata, + ) + from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum + + model = SimpleNamespace( + _resolve_attention_backend_once=lambda: None, + _resolved_attention_backend=AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3, + ) + with pytest.raises(NotImplementedError, match="ComfyUI-serialized Ref2VA"): + _maybe_prepare_vsa_h3_step_metadata( + model=model, + packed={}, + is_ref2va=True, + latent_shape=(4, 8, 12), + server_args=SimpleNamespace(), + device=torch.device("cpu"), + native_layout=False, + ) + + def test_ref2va_image_plus_video_audio_used_prefix_matches_comfyui(): kwargs = dict(text_len=5, latent_t=2, latent_h=4, latent_w=4, audio_t=5) refs = [ diff --git a/python/sglang/multimodal_gen/test/unit/test_vsa_h3_ref2va_regions.py b/python/sglang/multimodal_gen/test/unit/test_vsa_h3_ref2va_regions.py new file mode 100644 index 0000000000..ec32385375 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_vsa_h3_ref2va_regions.py @@ -0,0 +1,227 @@ +# SPDX-License-Identifier: Apache-2.0 +"""VSA-H3 Ref2VA regions and 128-token logical tiles. + +The oracle below is written at logical-tile granularity from FastVideo's +"p2_multi_region" mask semantics, independent of the kernel-block split the +backend serves 128-token tiles with. +""" + +from __future__ import annotations + +import math + +import pytest +import torch + +from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import ( + VideoSparseAttentionH3Impl, + VideoSparseAttentionH3MetadataBuilder, + _compute_topk, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.packed_sequence import ( + minimax_h3_packed_sequence_ref2va_blocks, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import ( + _ref2va_vsa_segments, +) + +requires_cuda = pytest.mark.skipif( + not torch.cuda.is_available(), reason="VSA-H3 kernels need CUDA" +) + +HEADS = 4 +HEAD_DIM = 128 +TARGET = (5, 6, 10) +# text, image ref, reference video (ragged in every tile dimension), its +# audio, a standalone audio ref, target audio, generated video +SEGMENTS = (70, 30, 24, (3, 5, 9), 40, 50, TARGET) + + +def _build(sparsity: float, device, *, tile_size=128, ref_keep_rate=0.25, **kw): + return VideoSparseAttentionH3MetadataBuilder().build( + current_timestep=0, + raw_latent_shape=TARGET, + patch_size=(1, 1, 1), + VSA_sparsity=sparsity, + prefix_segments=(), + device=device, + packed_segments=SEGMENTS, + tile_size=tile_size, + ref_keep_rate=ref_keep_rate, + **kw, + ) + + +def _used() -> int: + return sum(math.prod(s) if isinstance(s, tuple) else s for s in SEGMENTS) + + +def test_region_geometry_accounts_every_row() -> None: + meta = _build(0.9, torch.device("cpu")) + used = _used() + assert meta.total_seq_length == used + assert meta.blocks_per_tile == 2 + assert int(meta.variable_block_sizes.sum()) == used + assert int((meta.pack_index >= 0).sum()) == used + assert sorted(meta.unpack_index.tolist()) == sorted( + (meta.pack_index >= 0).nonzero().flatten().tolist() + ) + # Dense rows chunk per segment into 128-token tiles: 70, 30, 24, 40, 50. + assert meta.num_prefix_tiles == 5 + logical = meta.variable_block_sizes.view(-1, 2).sum(-1) + assert logical[:5].tolist() == [70, 30, 24, 40, 50] + # Reference (3, 5, 9) and target (5, 6, 10) at (4, 4, 8) tiles. + reference_tiles = 1 * 2 * 2 + target_tiles = 2 * 2 * 2 + assert meta.video_tile_spans == ( + (5, 5 + reference_tiles), + (5 + reference_tiles, 5 + reference_tiles + target_tiles), + ) + assert meta.num_video_tiles == reference_tiles + target_tiles + assert meta.span_sparsities == (0.75, 0.9) + # Each kernel block holds at most 64 rows, the lower half first. + halves = meta.variable_block_sizes.view(-1, 2) + assert torch.all(halves[:, 0] == logical.clamp(max=64)) + + +def test_reference_regions_need_keep_rate_and_exempt() -> None: + with pytest.raises(ValueError, match="vsa_ref_keep_rate"): + _build(0.9, torch.device("cpu"), ref_keep_rate=None) + with pytest.raises(ValueError, match="compete"): + _build(0.9, torch.device("cpu"), exempt=False) + + +def test_ref2va_segments_follow_packed_layout() -> None: + blocks = [ + {"kind": "image", "latent_h": 8, "latent_w": 12}, + { + "kind": "video_audio", + "ref_audio_t": 6, + "latent_t": 3, + "latent_h": 8, + "latent_w": 12, + }, + {"kind": "audio", "ref_audio_t": 5}, + ] + latent = (4, 10, 16) + packed = minimax_h3_packed_sequence_ref2va_blocks( + text_len=33, + latent_t=latent[0], + latent_h=latent[1], + latent_w=latent[2], + audio_t=20, + ref_blocks=blocks, + include_video_pos=True, + ) + segments = _ref2va_vsa_segments(packed, latent, (1, 2, 2)) + assert segments == (33, 0, 24, 12, (3, 4, 6), 10, 40, (4, 5, 8)) + # Region rows are exactly the layout's sparse-eligible video rows. + cursor, region_rows = 0, [] + for segment in segments: + rows = math.prod(segment) if isinstance(segment, tuple) else segment + if isinstance(segment, tuple): + region_rows.append(torch.arange(cursor, cursor + rows)) + cursor += rows + assert cursor == int(packed["cu_seqlens"][1]) + assert torch.equal(torch.cat(region_rows), packed["video_pos"]) + + +def _oracle(meta, used, q, k, v, gate): + """Masked dense attention over logical tiles, plus the gated compression.""" + split = meta.blocks_per_tile + tile = meta.tile_elems + logical = meta.variable_block_sizes.view(-1, split).sum(-1).float() + n = logical.numel() + valid = meta.pack_index >= 0 + src = meta.pack_index.clamp(min=0).long() + + def tiled(x): + t = x[src].float().permute(1, 0, 2) + return t * valid.to(t.dtype)[None, :, None] + + qt, kt, vt, gt = tiled(q), tiled(k), tiled(v), tiled(gate) + pooled = [ + t.view(HEADS, n, tile, HEAD_DIM).sum(2) / logical[None, :, None] + for t in (qt, kt, vt) + ] + scores = pooled[0] @ pooled[1].transpose(-1, -2) / math.sqrt(HEAD_DIM) + prefix = meta.num_prefix_tiles + mask = torch.zeros(HEADS, n, n, dtype=torch.bool, device=q.device) + for (start, end), sparsity in zip(meta.video_tile_spans, meta.span_sparsities): + keep = _compute_topk(sparsity, end - start) + picked = scores[..., start:end].topk(keep, dim=-1).indices + start + mask.scatter_(-1, picked, True) + mask[..., :prefix] = True + mask[:, :prefix, :] = True + row_mask = mask.repeat_interleave(tile, 1).repeat_interleave(tile, 2) + row_mask &= valid[None, None, :] + logits = qt @ kt.transpose(-1, -2) / math.sqrt(HEAD_DIM) + out = torch.softmax(logits.masked_fill(~row_mask, float("-inf")), -1) @ vt + compress = torch.softmax(scores, dim=-1) @ pooled[2] + out = out + compress.repeat_interleave(tile, 1) * gt + return out[:, meta.unpack_index.long()].permute(1, 0, 2) + + +def _impl(): + return VideoSparseAttentionH3Impl( + num_heads=HEADS, + head_size=HEAD_DIM, + causal=False, + softmax_scale=HEAD_DIM**-0.5, + num_kv_heads=HEADS, + prefix="blocks.3.attn", + ) + + +def _qkvg(device, used, seed=5): + total = (used + 63) // 64 * 64 + generator = torch.Generator(device="cpu").manual_seed(seed) + q, k, v, gate = ( + torch.randn((total, HEADS, HEAD_DIM), generator=generator).to( + device=device, dtype=torch.bfloat16 + ) + for _ in range(4) + ) + return total, q, k, v, gate * 0.1 + + +@requires_cuda +@pytest.mark.parametrize("tile_size", [64, 128]) +def test_regions_match_masked_dense_oracle(tile_size: int) -> None: + device = torch.device("cuda") + meta = _build(0.5, device, tile_size=tile_size) + used = _used() + total, q, k, v, gate = _qkvg(device, used) + cu = torch.tensor([0, used, total], dtype=torch.int32, device=device) + out = _impl().forward_varlen( + q, + k, + v, + cu_seqlens=cu, + max_seqlen=used, + cu_seqlens_host=(0, used, total), + attn_metadata=meta, + gate_compress=gate, + ) + reference = _oracle(meta, used, q, k, v, gate) + diff = (out[:used].float() - reference).abs().max().item() + assert diff < 2e-2, f"tile {tile_size}: regions vs oracle max diff {diff}" + assert torch.all(out[used:] == 0) + + # The deferred (Ulysses) gate fold sees the same per-block compression. + from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import ( + vsa_h3_fold_gate, + ) + + unfolded, out_compress = _impl().forward_varlen( + q, + k, + v, + cu_seqlens=cu, + max_seqlen=used, + cu_seqlens_host=(0, used, total), + attn_metadata=meta, + return_compress=True, + ) + vsa_h3_fold_gate(unfolded, gate, out_compress, meta, 0) + assert torch.equal(unfolded, out)