[diffusion] feat: minimax-h3 spectrum skip-step + fused RMSNorm/AdaLN (#35684)

This commit is contained in:
WenhaoZhang
2026-09-27 17:56:13 +08:00
committed by GitHub
parent 2a14f65fd7
commit ae47bcd4da
10 changed files with 461 additions and 70 deletions
@@ -168,7 +168,7 @@ These options **trade output quality** for speed or VRAM savings. Results will d
| **Cache-DiT** | Native: per-request `--enable-cache-dit true\|false` + `--cache-dit-params <json>` (sampling params; also via API `extra_body`). `SGLANG_CACHE_DIT_ENABLED` / `SGLANG_CACHE_DIT_*` env vars are the server-wide defaults for requests that leave them unset. Diffusers backend: `--backend diffusers --cache-dit-config <yaml-or-json>` | Caches intermediate residuals across denoising steps and skips redundant computations via DBCache, TaylorSeer, and optional SCM. | ~1.5-2x on supported models | Quality depends on cache policy. Compatible with `--dit-layerwise-offload`: skipped blocks are not streamed, and the first layer after a skip may sync-load. Models that touch every layer before the block loop (for example a full-stack AdaLN prepass) must keep that prepass off while caching. Do not pass `--cache-dit-config` for native SGLang tuning unless you are intentionally using the diffusers backend flow. |
| **CFG Gating** | Per-request `--cfg-gate-step 0.5` (sampling param; also via API `extra_body`). `SGLANG_DIFFUSION_CFG_GATE_STEP` is the server-wide default (1.0 = off). | After the given fraction of denoising steps, reuses the cached cond-uncond residual instead of running the unconditional branch each step. | Up to ~2x on the gated tail of CFG models (skips one of two branches) | Lossy; no-op without classifier-free guidance or with `--enable-cfg-parallel`. Lower fractions gate earlier and drift more. |
| **TeaCache** | `--enable-teacache` (uses model sampling presets) | Reuses residuals when adjacent denoising steps are sufficiently similar. | Model- and threshold-dependent | Approximate and model-specific. Mutually exclusive with Spectrum. Fix prompt/seed/shape/steps and validate temporal consistency, not only single frames. |
| **Spectrum** | `--enable-spectrum` plus optional `--spectrum-*` controls | Forecasts DiT features and skips selected denoising steps. | Defaults target an accuracy/speed tradeoff; aggressive windows can be much faster | Native `sglang generate` only for FLUX.1, Wan, HunyuanVideo, and SD3; not FLUX.2 or server requests. Mutually exclusive with TeaCache. `--debug` adds shadow validation and is not representative latency. |
| **Spectrum** | `--enable-spectrum` plus optional `--spectrum-*` controls | Forecasts DiT features and skips selected denoising steps. | Defaults target an accuracy/speed tradeoff; aggressive windows can be much faster | Native `sglang generate` for FLUX.1, Wan, HunyuanVideo, SD3, and MiniMax-H3; not FLUX.2 or server requests. Mutually exclusive with TeaCache. `--debug` is not representative latency. |
| **Progressive Resolution** | `--progressive-mode dct_rewind --progressive-levels N --progressive-delta D` | Runs early denoising at lower latent resolution, then spectrally upsamples and switches to the target resolution. | Model- and schedule-dependent | Approximate and pipeline-specific. Keep the switch schedule fixed and compare detail, composition, and temporal stability. |
| **Causal KV-Cache Quantization** | `--kv-cache-quant int4\|int2` plus optional `--kv-cache-quant-*` controls | Compresses completed causal KV-cache chunks with Quant-VideoGen PRQ while keeping the mutable/current chunk and recent chunks in BF16. | Primarily a long-session memory saving | Currently limited to LingBot World realtime causal serving; requires `quant-videogen`. INT4 is the starting point; INT2 saves more memory with more error. It quantizes cache state, not checkpoint weights. |
| **Quantized Models (Nunchaku / SVDQuant)** | `--enable-svdquant --transformer-weights-path <path>` + optional `--quantization-precision int4\|nvfp4`, `--quantization-rank 32` | W4A4-style quantization via [Nunchaku](https://nunchaku.tech). Reduces DiT weight memory by ~4x. Precision/rank can be auto-inferred from weight filename or set explicitly. | ~1.5–2x compute speedup | Lossy quantization; quality depends on rank and precision. Requires pre-quantized weights. Ampere (SM8x) or SM12x only (no Hopper SM90). Higher rank = better quality but more memory. |
@@ -240,6 +240,27 @@ class MiniMaxH3SamplingParams(SamplingParams):
"MiniMax H3 does not support enable_teacache: its packed "
"video/audio denoise loop has no lossless TeaCache contract"
)
if self.enable_spectrum:
if self.quality == "high":
raise ValueError(
"MiniMax H3 enable_spectrum cannot be combined with "
'quality="high" (Cache-DiT). Use one skip-step accelerator.'
)
if self.enable_cache_dit:
raise ValueError(
"MiniMax H3 enable_spectrum cannot be combined with "
"enable_cache_dit. Use one skip-step accelerator."
)
from sglang.multimodal_gen.configs.sample.spectrum import SpectrumParams
from sglang.multimodal_gen.runtime.cache.spectrum import (
apply_h3_spectrum_param_defaults,
)
if isinstance(self.spectrum_params, dict):
self.spectrum_params = SpectrumParams(**self.spectrum_params)
elif self.spectrum_params is None:
self.spectrum_params = SpectrumParams()
apply_h3_spectrum_param_defaults(self.spectrum_params)
if self.rollout:
raise ValueError(
"MiniMax H3 does not support rollout: its coupled video/audio "
+100
View File
@@ -644,3 +644,103 @@ class SpectrumMixin:
predicted = forecaster.predict(step_idx)
self._spectrum_ctx = None
return predicted.to(dtype=template.dtype, device=template.device)
# MiniMax-H3: forecast packed target audio|video only (Comfy / Wan2GP).
H3_SPECTRUM_HISTORY_SIZE = 8
H3_AUDIO_BLEND_WEIGHT = 0.0
H3_VIDEO_BLEND_WEIGHT = 0.5
def apply_h3_spectrum_param_defaults(params: SpectrumParams) -> SpectrumParams:
"""Shrink the generic 100-slot GPU buffer default to the H3 CPU window."""
if params.history_size == 100:
params.history_size = H3_SPECTRUM_HISTORY_SIZE
return params
def target_audio_video_span(
audio_pos: torch.Tensor,
infer_out_pos: torch.Tensor,
) -> tuple[int, int, int]:
"""Return ``(audio_start, audio_stop, video_stop)`` in packed row indices.
Target video is ``infer_out_pos``. Target audio is the contiguous
``audio_pos`` run that ends immediately before that video span — the
packed t2va / fl2va / ref2va suffix. Earlier reference-audio rows are
dropped, matching Wan2GP ``hidden[audio_start:]``.
"""
audio_ids = audio_pos.detach().reshape(-1).to(dtype=torch.long, device="cpu")
video_ids = infer_out_pos.detach().reshape(-1).to(dtype=torch.long, device="cpu")
if audio_ids.numel() == 0 or video_ids.numel() == 0:
raise ValueError(
"MiniMax-H3 Spectrum needs non-empty target audio and video rows"
)
video_start = int(video_ids.min())
video_stop = int(video_ids.max()) + 1
if int(video_ids.numel()) != video_stop - video_start:
raise ValueError(
"MiniMax-H3 Spectrum target video rows must be a contiguous packed span"
)
preceding = audio_ids[audio_ids < video_start]
if preceding.numel() == 0 or int(preceding.max()) + 1 != video_start:
raise ValueError(
"MiniMax-H3 Spectrum expects packed target audio rows immediately "
f"before target video rows starting at {video_start}"
)
ordered = preceding.sort().values
expected = video_start - 1
cursor = int(ordered.numel()) - 1
while cursor >= 0 and int(ordered[cursor]) == expected:
expected -= 1
cursor -= 1
audio_start = expected + 1
return audio_start, video_start, video_stop
def local_target_hidden(
hidden: torch.Tensor,
audio_start: int,
video_stop: int,
row_start: int,
row_stop: int,
) -> tuple[torch.Tensor, int, int] | None:
"""Slice rank-local ``hidden`` to the overlap with packed target media."""
start = max(audio_start, row_start)
stop = min(video_stop, row_stop)
if stop <= start:
return None
local_start = start - row_start
local_stop = stop - row_start
return hidden[local_start:local_stop], local_start, local_stop
def blend_h3_spectrum_prediction(
predicted: torch.Tensor,
*,
n_audio: int,
last_audio: torch.Tensor | None,
last_video: torch.Tensor | None,
audio_blend: float = H3_AUDIO_BLEND_WEIGHT,
video_blend: float = H3_VIDEO_BLEND_WEIGHT,
) -> torch.Tensor:
"""Apply Comfy segment blends on a ``[audio | video]`` forecast."""
if n_audio < 0 or n_audio > int(predicted.shape[0]):
raise ValueError(
f"n_audio={n_audio} is outside predicted rows {predicted.shape[0]}"
)
out = predicted
audio = out[:n_audio]
video = out[n_audio:]
if last_audio is not None and audio.numel():
held = last_audio.to(device=audio.device, dtype=audio.dtype)
if audio_blend <= 0.0:
audio.copy_(held)
elif audio_blend < 1.0:
audio.copy_(torch.lerp(held, audio, audio_blend))
if last_video is not None and video.numel() and video_blend < 1.0:
held = last_video.to(device=video.device, dtype=video.dtype)
video.copy_(torch.lerp(held, video, video_blend))
return out
@@ -41,6 +41,12 @@ from sglang.multimodal_gen.configs.models.dits.minimax_h3 import (
MiniMaxH3DiTConfig,
)
from sglang.multimodal_gen.configs.models.fsdp import is_block
from sglang.multimodal_gen.runtime.cache.spectrum import (
SpectrumMixin,
blend_h3_spectrum_prediction,
local_target_hidden,
target_audio_video_span,
)
from sglang.multimodal_gen.runtime.distributed import (
get_tp_world_size,
tensor_model_parallel_all_gather,
@@ -372,6 +378,23 @@ def _rotate_half(x: torch.Tensor) -> torch.Tensor:
return torch.cat((-x2, x1), dim=-1)
def _modulate_rmsnorm_scale_shift(
x: torch.Tensor,
norm: nn.RMSNorm,
shift: torch.Tensor,
scale: torch.Tensor,
indices: torch.Tensor,
*,
dtype: torch.dtype,
) -> torch.Tensor:
"""RMSNorm + indexed AdaLN.
Keep ``nn.RMSNorm``; a fused RMSNorm+AdaLN kernel drifted 2-GPU
consistency GT for ~0.3% e2e, so it was removed.
"""
return _modulate_scale_shift(norm(x), shift, scale, indices, dtype=dtype)
def _accepts_mxfp8_input(linear: nn.Module) -> bool:
return linear.quant_method is not None and linear.quant_method.accepts_mxfp8_input(
linear
@@ -1430,17 +1453,27 @@ class MiniMaxH3DiTBlock(nn.Module):
# first gated residual writes to that tensor; the second one operates on
# a block-local buffer.
residual = x
h = self.norm1(x)
h_prequant = None
if _accepts_mxfp8_input(self.attn.qkv_proj) and can_use_mxfp8_swizzled(h):
# the bf16 modulated rows stay in h for the VDN branch projections
h, h_fp8, h_scales = indexed_scale_shift_mxfp8_(
h, shift_msa, scale_msa, combined_indices, keep_bf16=True
)
h_prequant = (h_fp8, h_scales)
if _accepts_mxfp8_input(self.attn.qkv_proj):
h = self.norm1(x)
if can_use_mxfp8_swizzled(h):
# the bf16 modulated rows stay in h for the VDN branch projections
h, h_fp8, h_scales = indexed_scale_shift_mxfp8_(
h, shift_msa, scale_msa, combined_indices, keep_bf16=True
)
h_prequant = (h_fp8, h_scales)
else:
h = _modulate_scale_shift(
h, shift_msa, scale_msa, combined_indices, dtype=_BF16_DTYPE
)
else:
h = _modulate_scale_shift(
h, shift_msa, scale_msa, combined_indices, dtype=_BF16_DTYPE
h = _modulate_rmsnorm_scale_shift(
x,
self.norm1,
shift_msa,
scale_msa,
combined_indices,
dtype=_BF16_DTYPE,
)
h = self.attn(
h,
@@ -1463,15 +1496,26 @@ class MiniMaxH3DiTBlock(nn.Module):
)
residual = x
h = self.norm2(x)
if _accepts_mxfp8_input(self.mlp.fc1) and can_use_mxfp8_swizzled(h):
_, h_fp8, h_scales = indexed_scale_shift_mxfp8_(
h, shift_mlp, scale_mlp, combined_indices, keep_bf16=False
)
h = self.mlp((h_fp8, h_scales))
if _accepts_mxfp8_input(self.mlp.fc1):
h = self.norm2(x)
if can_use_mxfp8_swizzled(h):
_, h_fp8, h_scales = indexed_scale_shift_mxfp8_(
h, shift_mlp, scale_mlp, combined_indices, keep_bf16=False
)
h = self.mlp((h_fp8, h_scales))
else:
h = _modulate_scale_shift(
h, shift_mlp, scale_mlp, combined_indices, dtype=_BF16_DTYPE
)
h = self.mlp(h)
else:
h = _modulate_scale_shift(
h, shift_mlp, scale_mlp, combined_indices, dtype=_BF16_DTYPE
h = _modulate_rmsnorm_scale_shift(
x,
self.norm2,
shift_mlp,
scale_mlp,
combined_indices,
dtype=_BF16_DTYPE,
)
h = self.mlp(h)
# `residual` is block-local here (see above), so this stays in-place
@@ -1609,9 +1653,9 @@ class MiniMaxH3FinalLayer(nn.Module):
video = audio = None
for start in range(0, x.shape[0], _MPS_MLP_TOKEN_CHUNK_SIZE):
stop = min(start + _MPS_MLP_TOKEN_CHUNK_SIZE, x.shape[0])
h = self.norm(x[start:stop])
h = _modulate_scale_shift(
h,
h = _modulate_rmsnorm_scale_shift(
x[start:stop],
self.norm,
shift,
scale,
inverse_indices[start:stop],
@@ -1636,8 +1680,9 @@ class MiniMaxH3FinalLayer(nn.Module):
torch.mps.empty_cache()
assert video is not None and audio is not None
return video, audio
h = self.norm(x)
h = _modulate_scale_shift(h, shift, scale, inverse_indices, dtype=_BF16_DTYPE)
h = _modulate_rmsnorm_scale_shift(
x, self.norm, shift, scale, inverse_indices, dtype=_BF16_DTYPE
)
# Preserve full precision through both final output projections.
h = h.to(_FP32_DTYPE)
return self._project(h)
@@ -1661,7 +1706,7 @@ def _reject_adaln_lora(names: list[str]) -> None:
)
class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
class MiniMaxH3DiTModel(SpectrumMixin, BaseDiT, LayerwiseOffloadableModuleMixin):
_aliases = [
"MiniMaxH3Transformer3DModel",
"MiniMaxH3PrunedTransformer3DModel",
@@ -2033,7 +2078,11 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
for index in range(arch.num_layers)
]
)
self.layer_names = ["token_refiner.blocks", "blocks"]
# Offload the 50-layer DiT stack before token_refiner. Each
# LayerwiseOffloadManager ends with model.to(device) for leftovers;
# putting the 2-layer refiner first would copy the full INT8 DiT onto
# a 24GB card and OOM.
self.layer_names = ["blocks", "token_refiner.blocks"]
self.final_layer = MiniMaxH3FinalLayer(
arch,
quant_config,
@@ -2064,6 +2113,9 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
)
self._resolved_attention_backend: AttentionBackendEnum | None = None
self._mark_missing_params_required()
self._init_spectrum_state()
self._h3_spectrum_last_audio: torch.Tensor | None = None
self._h3_spectrum_last_video: torch.Tensor | None = None
def set_cache_dit_input_preservation(self, enabled: bool) -> None:
"""Stop the blocks from overwriting the input Cache-DiT holds by reference.
@@ -2081,6 +2133,76 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
for block in self.blocks:
block.preserve_input_for_cache_dit = enabled
def _h3_spectrum_requested(self, *, sp_ws: int) -> bool:
"""True when this forward should use Spectrum. Off path stays bit-exact."""
try:
forward_batch = get_forward_context().forward_batch
except AssertionError:
return False
if forward_batch is None or not forward_batch.enable_spectrum:
return False
if sp_ws > 1:
raise ValueError(
"MiniMax-H3 Spectrum requires sequence-parallel size 1 so "
"target audio/video rows stay on one rank. Disable "
"--enable-spectrum or run without SP / Ulysses / ring."
)
return True
def _h3_spectrum_record_targets(
self,
hidden: torch.Tensor,
audio_pos: torch.Tensor,
infer_out_pos: torch.Tensor,
row_start: int,
row_stop: int,
) -> None:
audio_start, audio_stop, video_stop = target_audio_video_span(
audio_pos, infer_out_pos
)
sliced = local_target_hidden(
hidden, audio_start, video_stop, row_start, row_stop
)
if sliced is None:
return
local, _, _ = sliced
n_audio = audio_stop - audio_start
cpu_local = local.detach().to(device="cpu", dtype=local.dtype)
self._h3_spectrum_last_audio = cpu_local[:n_audio].contiguous()
self._h3_spectrum_last_video = cpu_local[n_audio:].contiguous()
self.spectrum_record_features(cpu_local)
def _h3_spectrum_predict_targets(
self,
hidden: torch.Tensor,
audio_pos: torch.Tensor,
infer_out_pos: torch.Tensor,
row_start: int,
row_stop: int,
) -> torch.Tensor:
audio_start, audio_stop, video_stop = target_audio_video_span(
audio_pos, infer_out_pos
)
sliced = local_target_hidden(
hidden, audio_start, video_stop, row_start, row_stop
)
if sliced is None:
return hidden
local, local_start, local_stop = sliced
predicted = self.spectrum_predict_features(
local.detach().to(device="cpu", dtype=local.dtype)
)
predicted = blend_h3_spectrum_prediction(
predicted,
n_audio=audio_stop - audio_start,
last_audio=self._h3_spectrum_last_audio,
last_video=self._h3_spectrum_last_video,
)
hidden[local_start:local_stop].copy_(
predicted.to(device=hidden.device, dtype=hidden.dtype)
)
return hidden
def _resolve_attention_backend_once(self) -> None:
if self._resolved_attention_backend is not None:
return
@@ -2612,50 +2734,83 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
hidden = decoder_input
cu_seqlens = cu_seqlens.to(device)
spectrum_on = self._h3_spectrum_requested(sp_ws=sp_ws)
run_transformer_blocks = self.begin_spectrum_step() if spectrum_on else True
if spectrum_on:
try:
timestep = get_forward_context().current_timestep
except AssertionError:
timestep = None
if timestep == 0:
self._h3_spectrum_last_audio = None
self._h3_spectrum_last_video = None
block_adaln_params = None
adaln_cache_plan_index = None
if self.adaln_cache is not None:
# prepare_adaln_plans resolved the slot on the host; the device
# lookup remains for callers that drive forward() directly.
# Skip-step forecasts still run embed + final_layer. Do not touch
# per-block AdaLN or the 50-layer stack — that is what lets Spectrum
# coexist with DiT layerwise offload.
if run_transformer_blocks:
if self.adaln_cache is not None:
# prepare_adaln_plans resolved the slot on the host; the device
# lookup remains for callers that drive forward() directly.
adaln_cache_plan_index = kwargs.get("adaln_cache_slot")
if adaln_cache_plan_index is None:
adaln_cache_plan_index = self.adaln_cache.lookup(
unique_timesteps.view(-1).to(device)
)
block_adaln_params = self.adaln_cache.block_all(
cache_plan_index=adaln_cache_plan_index,
num_timesteps=adaln_input.shape[0],
)
elif self._can_batch_block_adaln():
local_adaln = torch.stack(
[
block.adaln_proj.project_local(adaln_input)
for block in self.blocks
]
)
gathered_adaln = tensor_model_parallel_all_gather(local_adaln)
block_adaln_params = tuple(
block.adaln_proj.split_output(output)
for block, output in zip(self.blocks, gathered_adaln)
)
# With sequence parallelism, shard rows across the group for the
# block stack. Attention trades sequence for heads internally
# (Ulysses) and/or ring-rotates KV across ring ranks; everything
# else, including the final layer, is row-local. Only the narrow
# video/audio logits are gathered after the final layer.
for index, block in enumerate(self.blocks):
hidden = block(
hidden,
adaln_input=adaln_input,
combined_indices=block_combined,
rope_cache=rope_cache,
cu_seqlens=cu_seqlens,
cu_seqlens_host=cu_seqlens_host,
max_seqlen=max_seqlen,
subblock_sparse_query_block_mask=subblock_sparse_query_block_mask,
ulysses_active=ulysses_ws > 1,
ring_active=ring_ws > 1,
adaln_params=(
None
if block_adaln_params is None
else block_adaln_params[index]
),
)
if spectrum_on:
self._h3_spectrum_record_targets(
hidden, audio_pos, infer_out_pos, row_start, row_stop
)
elif spectrum_on:
hidden = self._h3_spectrum_predict_targets(
hidden, audio_pos, infer_out_pos, row_start, row_stop
)
if self.adaln_cache is not None and adaln_cache_plan_index is None:
adaln_cache_plan_index = kwargs.get("adaln_cache_slot")
if adaln_cache_plan_index is None:
adaln_cache_plan_index = self.adaln_cache.lookup(
unique_timesteps.view(-1).to(device)
)
block_adaln_params = self.adaln_cache.block_all(
cache_plan_index=adaln_cache_plan_index,
num_timesteps=adaln_input.shape[0],
)
elif self._can_batch_block_adaln():
local_adaln = torch.stack(
[block.adaln_proj.project_local(adaln_input) for block in self.blocks]
)
gathered_adaln = tensor_model_parallel_all_gather(local_adaln)
block_adaln_params = tuple(
block.adaln_proj.split_output(output)
for block, output in zip(self.blocks, gathered_adaln)
)
# With sequence parallelism, shard rows across the group for the
# block stack. Attention trades sequence for heads internally
# (Ulysses) and/or ring-rotates KV across ring ranks; everything
# else, including the final layer, is row-local. Only the narrow
# video/audio logits are gathered after the final layer.
for index, block in enumerate(self.blocks):
hidden = block(
hidden,
adaln_input=adaln_input,
combined_indices=block_combined,
rope_cache=rope_cache,
cu_seqlens=cu_seqlens,
cu_seqlens_host=cu_seqlens_host,
max_seqlen=max_seqlen,
subblock_sparse_query_block_mask=subblock_sparse_query_block_mask,
ulysses_active=ulysses_ws > 1,
ring_active=ring_ws > 1,
adaln_params=(
None if block_adaln_params is None else block_adaln_params[index]
),
)
self.materialize_mps_non_layer_weights("final_layer")
video_logits, audio_logits = self.final_layer(
hidden,
@@ -482,6 +482,9 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
if enable_override is False:
# The per-request kill switch wins over quality="high".
desired_mode = None
elif batch.sampling_params.enable_spectrum:
# Spectrum skips the block stack; Cache-DiT wraps those blocks.
desired_mode = None
elif quality == "high":
desired_mode = "high"
else:
@@ -408,6 +408,7 @@ def test_high_quality_request_warns_when_bcg_suppresses_cache_dit():
_explicit_fields={"quality"},
enable_cache_dit=None,
cache_dit_params=None,
enable_spectrum=False,
)
)
@@ -465,6 +466,7 @@ def test_extra_high_quality_does_not_enable_h3_cache_dit():
_explicit_fields={"quality"},
enable_cache_dit=None,
cache_dit_params=None,
enable_spectrum=False,
)
)
@@ -263,7 +263,7 @@ def test_cache_dit_preservation_only_makes_first_gate_out_of_place():
with (
patch(
"sglang.multimodal_gen.runtime.models.dits.minimax_h3._modulate_scale_shift",
"sglang.multimodal_gen.runtime.models.dits.minimax_h3._modulate_rmsnorm_scale_shift",
side_effect=lambda value, *_args, **_kwargs: value,
),
patch(
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
"""H3 Spectrum packing helpers and sampling-param gates."""
from __future__ import annotations
import pytest
import torch
from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams
from sglang.multimodal_gen.runtime.cache.spectrum import (
H3_SPECTRUM_HISTORY_SIZE,
blend_h3_spectrum_prediction,
local_target_hidden,
target_audio_video_span,
)
def test_target_span_uses_contiguous_audio_before_video():
audio_pos = torch.tensor([4, 5, 6, 7, 8, 9])
infer_out_pos = torch.tensor([10, 11, 12, 13])
start, stop, video_stop = target_audio_video_span(audio_pos, infer_out_pos)
assert (start, stop, video_stop) == (4, 10, 14)
def test_target_span_drops_earlier_ref_audio_gap():
audio_pos = torch.tensor([1, 2, 8, 9])
infer_out_pos = torch.tensor([10, 11])
start, stop, video_stop = target_audio_video_span(audio_pos, infer_out_pos)
assert (start, stop, video_stop) == (8, 10, 12)
def test_local_target_hidden_is_rank_local_view():
hidden = torch.arange(20, dtype=torch.float32).unsqueeze(-1).expand(20, 3).clone()
sliced = local_target_hidden(
hidden, audio_start=4, video_stop=10, row_start=0, row_stop=20
)
assert sliced is not None
local, start, stop = sliced
assert start == 4 and stop == 10
assert local.shape == (6, 3)
def test_blend_holds_audio_and_mixes_video():
predicted = torch.ones(6, 2)
last_audio = torch.full((2, 2), 3.0)
last_video = torch.full((4, 2), 5.0)
out = blend_h3_spectrum_prediction(
predicted.clone(),
n_audio=2,
last_audio=last_audio,
last_video=last_video,
audio_blend=0.0,
video_blend=0.5,
)
assert torch.equal(out[:2], last_audio)
assert torch.allclose(out[2:], torch.full((4, 2), 3.0))
def test_h3_sampling_params_reject_quality_high_with_spectrum():
with pytest.raises(ValueError, match="enable_spectrum cannot be combined"):
MiniMaxH3SamplingParams(
prompt="cat",
enable_spectrum=True,
quality="high",
)
def test_h3_sampling_params_default_history_is_comfy_window():
params = MiniMaxH3SamplingParams(prompt="cat", enable_spectrum=True)
assert params.spectrum_params is not None
assert params.spectrum_params.history_size == H3_SPECTRUM_HISTORY_SIZE
def test_h3_sampling_params_keep_explicit_history_size():
params = MiniMaxH3SamplingParams(
prompt="cat",
enable_spectrum=True,
spectrum_params={"history_size": 6},
)
assert params.spectrum_params.history_size == 6