[diffusion] feat: minimax-h3 spectrum skip-step + fused RMSNorm/AdaLN (#35684)
This commit is contained in:
@@ -168,7 +168,7 @@ These options **trade output quality** for speed or VRAM savings. Results will d
|
||||
| **Cache-DiT** | Native: per-request `--enable-cache-dit true\|false` + `--cache-dit-params <json>` (sampling params; also via API `extra_body`). `SGLANG_CACHE_DIT_ENABLED` / `SGLANG_CACHE_DIT_*` env vars are the server-wide defaults for requests that leave them unset. Diffusers backend: `--backend diffusers --cache-dit-config <yaml-or-json>` | Caches intermediate residuals across denoising steps and skips redundant computations via DBCache, TaylorSeer, and optional SCM. | ~1.5-2x on supported models | Quality depends on cache policy. Compatible with `--dit-layerwise-offload`: skipped blocks are not streamed, and the first layer after a skip may sync-load. Models that touch every layer before the block loop (for example a full-stack AdaLN prepass) must keep that prepass off while caching. Do not pass `--cache-dit-config` for native SGLang tuning unless you are intentionally using the diffusers backend flow. |
|
||||
| **CFG Gating** | Per-request `--cfg-gate-step 0.5` (sampling param; also via API `extra_body`). `SGLANG_DIFFUSION_CFG_GATE_STEP` is the server-wide default (1.0 = off). | After the given fraction of denoising steps, reuses the cached cond-uncond residual instead of running the unconditional branch each step. | Up to ~2x on the gated tail of CFG models (skips one of two branches) | Lossy; no-op without classifier-free guidance or with `--enable-cfg-parallel`. Lower fractions gate earlier and drift more. |
|
||||
| **TeaCache** | `--enable-teacache` (uses model sampling presets) | Reuses residuals when adjacent denoising steps are sufficiently similar. | Model- and threshold-dependent | Approximate and model-specific. Mutually exclusive with Spectrum. Fix prompt/seed/shape/steps and validate temporal consistency, not only single frames. |
|
||||
| **Spectrum** | `--enable-spectrum` plus optional `--spectrum-*` controls | Forecasts DiT features and skips selected denoising steps. | Defaults target an accuracy/speed tradeoff; aggressive windows can be much faster | Native `sglang generate` only for FLUX.1, Wan, HunyuanVideo, and SD3; not FLUX.2 or server requests. Mutually exclusive with TeaCache. `--debug` adds shadow validation and is not representative latency. |
|
||||
| **Spectrum** | `--enable-spectrum` plus optional `--spectrum-*` controls | Forecasts DiT features and skips selected denoising steps. | Defaults target an accuracy/speed tradeoff; aggressive windows can be much faster | Native `sglang generate` for FLUX.1, Wan, HunyuanVideo, SD3, and MiniMax-H3; not FLUX.2 or server requests. Mutually exclusive with TeaCache. `--debug` is not representative latency. |
|
||||
| **Progressive Resolution** | `--progressive-mode dct_rewind --progressive-levels N --progressive-delta D` | Runs early denoising at lower latent resolution, then spectrally upsamples and switches to the target resolution. | Model- and schedule-dependent | Approximate and pipeline-specific. Keep the switch schedule fixed and compare detail, composition, and temporal stability. |
|
||||
| **Causal KV-Cache Quantization** | `--kv-cache-quant int4\|int2` plus optional `--kv-cache-quant-*` controls | Compresses completed causal KV-cache chunks with Quant-VideoGen PRQ while keeping the mutable/current chunk and recent chunks in BF16. | Primarily a long-session memory saving | Currently limited to LingBot World realtime causal serving; requires `quant-videogen`. INT4 is the starting point; INT2 saves more memory with more error. It quantizes cache state, not checkpoint weights. |
|
||||
| **Quantized Models (Nunchaku / SVDQuant)** | `--enable-svdquant --transformer-weights-path <path>` + optional `--quantization-precision int4\|nvfp4`, `--quantization-rank 32` | W4A4-style quantization via [Nunchaku](https://nunchaku.tech). Reduces DiT weight memory by ~4x. Precision/rank can be auto-inferred from weight filename or set explicitly. | ~1.5–2x compute speedup | Lossy quantization; quality depends on rank and precision. Requires pre-quantized weights. Ampere (SM8x) or SM12x only (no Hopper SM90). Higher rank = better quality but more memory. |
|
||||
|
||||
@@ -240,6 +240,27 @@ class MiniMaxH3SamplingParams(SamplingParams):
|
||||
"MiniMax H3 does not support enable_teacache: its packed "
|
||||
"video/audio denoise loop has no lossless TeaCache contract"
|
||||
)
|
||||
if self.enable_spectrum:
|
||||
if self.quality == "high":
|
||||
raise ValueError(
|
||||
"MiniMax H3 enable_spectrum cannot be combined with "
|
||||
'quality="high" (Cache-DiT). Use one skip-step accelerator.'
|
||||
)
|
||||
if self.enable_cache_dit:
|
||||
raise ValueError(
|
||||
"MiniMax H3 enable_spectrum cannot be combined with "
|
||||
"enable_cache_dit. Use one skip-step accelerator."
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.spectrum import SpectrumParams
|
||||
from sglang.multimodal_gen.runtime.cache.spectrum import (
|
||||
apply_h3_spectrum_param_defaults,
|
||||
)
|
||||
|
||||
if isinstance(self.spectrum_params, dict):
|
||||
self.spectrum_params = SpectrumParams(**self.spectrum_params)
|
||||
elif self.spectrum_params is None:
|
||||
self.spectrum_params = SpectrumParams()
|
||||
apply_h3_spectrum_param_defaults(self.spectrum_params)
|
||||
if self.rollout:
|
||||
raise ValueError(
|
||||
"MiniMax H3 does not support rollout: its coupled video/audio "
|
||||
|
||||
@@ -644,3 +644,103 @@ class SpectrumMixin:
|
||||
predicted = forecaster.predict(step_idx)
|
||||
self._spectrum_ctx = None
|
||||
return predicted.to(dtype=template.dtype, device=template.device)
|
||||
|
||||
|
||||
# MiniMax-H3: forecast packed target audio|video only (Comfy / Wan2GP).
|
||||
|
||||
H3_SPECTRUM_HISTORY_SIZE = 8
|
||||
H3_AUDIO_BLEND_WEIGHT = 0.0
|
||||
H3_VIDEO_BLEND_WEIGHT = 0.5
|
||||
|
||||
|
||||
def apply_h3_spectrum_param_defaults(params: SpectrumParams) -> SpectrumParams:
|
||||
"""Shrink the generic 100-slot GPU buffer default to the H3 CPU window."""
|
||||
if params.history_size == 100:
|
||||
params.history_size = H3_SPECTRUM_HISTORY_SIZE
|
||||
return params
|
||||
|
||||
|
||||
def target_audio_video_span(
|
||||
audio_pos: torch.Tensor,
|
||||
infer_out_pos: torch.Tensor,
|
||||
) -> tuple[int, int, int]:
|
||||
"""Return ``(audio_start, audio_stop, video_stop)`` in packed row indices.
|
||||
|
||||
Target video is ``infer_out_pos``. Target audio is the contiguous
|
||||
``audio_pos`` run that ends immediately before that video span — the
|
||||
packed t2va / fl2va / ref2va suffix. Earlier reference-audio rows are
|
||||
dropped, matching Wan2GP ``hidden[audio_start:]``.
|
||||
"""
|
||||
audio_ids = audio_pos.detach().reshape(-1).to(dtype=torch.long, device="cpu")
|
||||
video_ids = infer_out_pos.detach().reshape(-1).to(dtype=torch.long, device="cpu")
|
||||
if audio_ids.numel() == 0 or video_ids.numel() == 0:
|
||||
raise ValueError(
|
||||
"MiniMax-H3 Spectrum needs non-empty target audio and video rows"
|
||||
)
|
||||
video_start = int(video_ids.min())
|
||||
video_stop = int(video_ids.max()) + 1
|
||||
if int(video_ids.numel()) != video_stop - video_start:
|
||||
raise ValueError(
|
||||
"MiniMax-H3 Spectrum target video rows must be a contiguous packed span"
|
||||
)
|
||||
|
||||
preceding = audio_ids[audio_ids < video_start]
|
||||
if preceding.numel() == 0 or int(preceding.max()) + 1 != video_start:
|
||||
raise ValueError(
|
||||
"MiniMax-H3 Spectrum expects packed target audio rows immediately "
|
||||
f"before target video rows starting at {video_start}"
|
||||
)
|
||||
ordered = preceding.sort().values
|
||||
expected = video_start - 1
|
||||
cursor = int(ordered.numel()) - 1
|
||||
while cursor >= 0 and int(ordered[cursor]) == expected:
|
||||
expected -= 1
|
||||
cursor -= 1
|
||||
audio_start = expected + 1
|
||||
return audio_start, video_start, video_stop
|
||||
|
||||
|
||||
def local_target_hidden(
|
||||
hidden: torch.Tensor,
|
||||
audio_start: int,
|
||||
video_stop: int,
|
||||
row_start: int,
|
||||
row_stop: int,
|
||||
) -> tuple[torch.Tensor, int, int] | None:
|
||||
"""Slice rank-local ``hidden`` to the overlap with packed target media."""
|
||||
start = max(audio_start, row_start)
|
||||
stop = min(video_stop, row_stop)
|
||||
if stop <= start:
|
||||
return None
|
||||
local_start = start - row_start
|
||||
local_stop = stop - row_start
|
||||
return hidden[local_start:local_stop], local_start, local_stop
|
||||
|
||||
|
||||
def blend_h3_spectrum_prediction(
|
||||
predicted: torch.Tensor,
|
||||
*,
|
||||
n_audio: int,
|
||||
last_audio: torch.Tensor | None,
|
||||
last_video: torch.Tensor | None,
|
||||
audio_blend: float = H3_AUDIO_BLEND_WEIGHT,
|
||||
video_blend: float = H3_VIDEO_BLEND_WEIGHT,
|
||||
) -> torch.Tensor:
|
||||
"""Apply Comfy segment blends on a ``[audio | video]`` forecast."""
|
||||
if n_audio < 0 or n_audio > int(predicted.shape[0]):
|
||||
raise ValueError(
|
||||
f"n_audio={n_audio} is outside predicted rows {predicted.shape[0]}"
|
||||
)
|
||||
out = predicted
|
||||
audio = out[:n_audio]
|
||||
video = out[n_audio:]
|
||||
if last_audio is not None and audio.numel():
|
||||
held = last_audio.to(device=audio.device, dtype=audio.dtype)
|
||||
if audio_blend <= 0.0:
|
||||
audio.copy_(held)
|
||||
elif audio_blend < 1.0:
|
||||
audio.copy_(torch.lerp(held, audio, audio_blend))
|
||||
if last_video is not None and video.numel() and video_blend < 1.0:
|
||||
held = last_video.to(device=video.device, dtype=video.dtype)
|
||||
video.copy_(torch.lerp(held, video, video_blend))
|
||||
return out
|
||||
|
||||
@@ -41,6 +41,12 @@ from sglang.multimodal_gen.configs.models.dits.minimax_h3 import (
|
||||
MiniMaxH3DiTConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.fsdp import is_block
|
||||
from sglang.multimodal_gen.runtime.cache.spectrum import (
|
||||
SpectrumMixin,
|
||||
blend_h3_spectrum_prediction,
|
||||
local_target_hidden,
|
||||
target_audio_video_span,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_tp_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
@@ -372,6 +378,23 @@ def _rotate_half(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.cat((-x2, x1), dim=-1)
|
||||
|
||||
|
||||
def _modulate_rmsnorm_scale_shift(
|
||||
x: torch.Tensor,
|
||||
norm: nn.RMSNorm,
|
||||
shift: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""RMSNorm + indexed AdaLN.
|
||||
|
||||
Keep ``nn.RMSNorm``; a fused RMSNorm+AdaLN kernel drifted 2-GPU
|
||||
consistency GT for ~0.3% e2e, so it was removed.
|
||||
"""
|
||||
return _modulate_scale_shift(norm(x), shift, scale, indices, dtype=dtype)
|
||||
|
||||
|
||||
def _accepts_mxfp8_input(linear: nn.Module) -> bool:
|
||||
return linear.quant_method is not None and linear.quant_method.accepts_mxfp8_input(
|
||||
linear
|
||||
@@ -1430,17 +1453,27 @@ class MiniMaxH3DiTBlock(nn.Module):
|
||||
# first gated residual writes to that tensor; the second one operates on
|
||||
# a block-local buffer.
|
||||
residual = x
|
||||
h = self.norm1(x)
|
||||
h_prequant = None
|
||||
if _accepts_mxfp8_input(self.attn.qkv_proj) and can_use_mxfp8_swizzled(h):
|
||||
# the bf16 modulated rows stay in h for the VDN branch projections
|
||||
h, h_fp8, h_scales = indexed_scale_shift_mxfp8_(
|
||||
h, shift_msa, scale_msa, combined_indices, keep_bf16=True
|
||||
)
|
||||
h_prequant = (h_fp8, h_scales)
|
||||
if _accepts_mxfp8_input(self.attn.qkv_proj):
|
||||
h = self.norm1(x)
|
||||
if can_use_mxfp8_swizzled(h):
|
||||
# the bf16 modulated rows stay in h for the VDN branch projections
|
||||
h, h_fp8, h_scales = indexed_scale_shift_mxfp8_(
|
||||
h, shift_msa, scale_msa, combined_indices, keep_bf16=True
|
||||
)
|
||||
h_prequant = (h_fp8, h_scales)
|
||||
else:
|
||||
h = _modulate_scale_shift(
|
||||
h, shift_msa, scale_msa, combined_indices, dtype=_BF16_DTYPE
|
||||
)
|
||||
else:
|
||||
h = _modulate_scale_shift(
|
||||
h, shift_msa, scale_msa, combined_indices, dtype=_BF16_DTYPE
|
||||
h = _modulate_rmsnorm_scale_shift(
|
||||
x,
|
||||
self.norm1,
|
||||
shift_msa,
|
||||
scale_msa,
|
||||
combined_indices,
|
||||
dtype=_BF16_DTYPE,
|
||||
)
|
||||
h = self.attn(
|
||||
h,
|
||||
@@ -1463,15 +1496,26 @@ class MiniMaxH3DiTBlock(nn.Module):
|
||||
)
|
||||
|
||||
residual = x
|
||||
h = self.norm2(x)
|
||||
if _accepts_mxfp8_input(self.mlp.fc1) and can_use_mxfp8_swizzled(h):
|
||||
_, h_fp8, h_scales = indexed_scale_shift_mxfp8_(
|
||||
h, shift_mlp, scale_mlp, combined_indices, keep_bf16=False
|
||||
)
|
||||
h = self.mlp((h_fp8, h_scales))
|
||||
if _accepts_mxfp8_input(self.mlp.fc1):
|
||||
h = self.norm2(x)
|
||||
if can_use_mxfp8_swizzled(h):
|
||||
_, h_fp8, h_scales = indexed_scale_shift_mxfp8_(
|
||||
h, shift_mlp, scale_mlp, combined_indices, keep_bf16=False
|
||||
)
|
||||
h = self.mlp((h_fp8, h_scales))
|
||||
else:
|
||||
h = _modulate_scale_shift(
|
||||
h, shift_mlp, scale_mlp, combined_indices, dtype=_BF16_DTYPE
|
||||
)
|
||||
h = self.mlp(h)
|
||||
else:
|
||||
h = _modulate_scale_shift(
|
||||
h, shift_mlp, scale_mlp, combined_indices, dtype=_BF16_DTYPE
|
||||
h = _modulate_rmsnorm_scale_shift(
|
||||
x,
|
||||
self.norm2,
|
||||
shift_mlp,
|
||||
scale_mlp,
|
||||
combined_indices,
|
||||
dtype=_BF16_DTYPE,
|
||||
)
|
||||
h = self.mlp(h)
|
||||
# `residual` is block-local here (see above), so this stays in-place
|
||||
@@ -1609,9 +1653,9 @@ class MiniMaxH3FinalLayer(nn.Module):
|
||||
video = audio = None
|
||||
for start in range(0, x.shape[0], _MPS_MLP_TOKEN_CHUNK_SIZE):
|
||||
stop = min(start + _MPS_MLP_TOKEN_CHUNK_SIZE, x.shape[0])
|
||||
h = self.norm(x[start:stop])
|
||||
h = _modulate_scale_shift(
|
||||
h,
|
||||
h = _modulate_rmsnorm_scale_shift(
|
||||
x[start:stop],
|
||||
self.norm,
|
||||
shift,
|
||||
scale,
|
||||
inverse_indices[start:stop],
|
||||
@@ -1636,8 +1680,9 @@ class MiniMaxH3FinalLayer(nn.Module):
|
||||
torch.mps.empty_cache()
|
||||
assert video is not None and audio is not None
|
||||
return video, audio
|
||||
h = self.norm(x)
|
||||
h = _modulate_scale_shift(h, shift, scale, inverse_indices, dtype=_BF16_DTYPE)
|
||||
h = _modulate_rmsnorm_scale_shift(
|
||||
x, self.norm, shift, scale, inverse_indices, dtype=_BF16_DTYPE
|
||||
)
|
||||
# Preserve full precision through both final output projections.
|
||||
h = h.to(_FP32_DTYPE)
|
||||
return self._project(h)
|
||||
@@ -1661,7 +1706,7 @@ def _reject_adaln_lora(names: list[str]) -> None:
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
class MiniMaxH3DiTModel(SpectrumMixin, BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
_aliases = [
|
||||
"MiniMaxH3Transformer3DModel",
|
||||
"MiniMaxH3PrunedTransformer3DModel",
|
||||
@@ -2033,7 +2078,11 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
for index in range(arch.num_layers)
|
||||
]
|
||||
)
|
||||
self.layer_names = ["token_refiner.blocks", "blocks"]
|
||||
# Offload the 50-layer DiT stack before token_refiner. Each
|
||||
# LayerwiseOffloadManager ends with model.to(device) for leftovers;
|
||||
# putting the 2-layer refiner first would copy the full INT8 DiT onto
|
||||
# a 24GB card and OOM.
|
||||
self.layer_names = ["blocks", "token_refiner.blocks"]
|
||||
self.final_layer = MiniMaxH3FinalLayer(
|
||||
arch,
|
||||
quant_config,
|
||||
@@ -2064,6 +2113,9 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
)
|
||||
self._resolved_attention_backend: AttentionBackendEnum | None = None
|
||||
self._mark_missing_params_required()
|
||||
self._init_spectrum_state()
|
||||
self._h3_spectrum_last_audio: torch.Tensor | None = None
|
||||
self._h3_spectrum_last_video: torch.Tensor | None = None
|
||||
|
||||
def set_cache_dit_input_preservation(self, enabled: bool) -> None:
|
||||
"""Stop the blocks from overwriting the input Cache-DiT holds by reference.
|
||||
@@ -2081,6 +2133,76 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
for block in self.blocks:
|
||||
block.preserve_input_for_cache_dit = enabled
|
||||
|
||||
def _h3_spectrum_requested(self, *, sp_ws: int) -> bool:
|
||||
"""True when this forward should use Spectrum. Off path stays bit-exact."""
|
||||
try:
|
||||
forward_batch = get_forward_context().forward_batch
|
||||
except AssertionError:
|
||||
return False
|
||||
if forward_batch is None or not forward_batch.enable_spectrum:
|
||||
return False
|
||||
if sp_ws > 1:
|
||||
raise ValueError(
|
||||
"MiniMax-H3 Spectrum requires sequence-parallel size 1 so "
|
||||
"target audio/video rows stay on one rank. Disable "
|
||||
"--enable-spectrum or run without SP / Ulysses / ring."
|
||||
)
|
||||
return True
|
||||
|
||||
def _h3_spectrum_record_targets(
|
||||
self,
|
||||
hidden: torch.Tensor,
|
||||
audio_pos: torch.Tensor,
|
||||
infer_out_pos: torch.Tensor,
|
||||
row_start: int,
|
||||
row_stop: int,
|
||||
) -> None:
|
||||
audio_start, audio_stop, video_stop = target_audio_video_span(
|
||||
audio_pos, infer_out_pos
|
||||
)
|
||||
sliced = local_target_hidden(
|
||||
hidden, audio_start, video_stop, row_start, row_stop
|
||||
)
|
||||
if sliced is None:
|
||||
return
|
||||
local, _, _ = sliced
|
||||
n_audio = audio_stop - audio_start
|
||||
cpu_local = local.detach().to(device="cpu", dtype=local.dtype)
|
||||
self._h3_spectrum_last_audio = cpu_local[:n_audio].contiguous()
|
||||
self._h3_spectrum_last_video = cpu_local[n_audio:].contiguous()
|
||||
self.spectrum_record_features(cpu_local)
|
||||
|
||||
def _h3_spectrum_predict_targets(
|
||||
self,
|
||||
hidden: torch.Tensor,
|
||||
audio_pos: torch.Tensor,
|
||||
infer_out_pos: torch.Tensor,
|
||||
row_start: int,
|
||||
row_stop: int,
|
||||
) -> torch.Tensor:
|
||||
audio_start, audio_stop, video_stop = target_audio_video_span(
|
||||
audio_pos, infer_out_pos
|
||||
)
|
||||
sliced = local_target_hidden(
|
||||
hidden, audio_start, video_stop, row_start, row_stop
|
||||
)
|
||||
if sliced is None:
|
||||
return hidden
|
||||
local, local_start, local_stop = sliced
|
||||
predicted = self.spectrum_predict_features(
|
||||
local.detach().to(device="cpu", dtype=local.dtype)
|
||||
)
|
||||
predicted = blend_h3_spectrum_prediction(
|
||||
predicted,
|
||||
n_audio=audio_stop - audio_start,
|
||||
last_audio=self._h3_spectrum_last_audio,
|
||||
last_video=self._h3_spectrum_last_video,
|
||||
)
|
||||
hidden[local_start:local_stop].copy_(
|
||||
predicted.to(device=hidden.device, dtype=hidden.dtype)
|
||||
)
|
||||
return hidden
|
||||
|
||||
def _resolve_attention_backend_once(self) -> None:
|
||||
if self._resolved_attention_backend is not None:
|
||||
return
|
||||
@@ -2612,50 +2734,83 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
|
||||
hidden = decoder_input
|
||||
cu_seqlens = cu_seqlens.to(device)
|
||||
spectrum_on = self._h3_spectrum_requested(sp_ws=sp_ws)
|
||||
run_transformer_blocks = self.begin_spectrum_step() if spectrum_on else True
|
||||
if spectrum_on:
|
||||
try:
|
||||
timestep = get_forward_context().current_timestep
|
||||
except AssertionError:
|
||||
timestep = None
|
||||
if timestep == 0:
|
||||
self._h3_spectrum_last_audio = None
|
||||
self._h3_spectrum_last_video = None
|
||||
block_adaln_params = None
|
||||
adaln_cache_plan_index = None
|
||||
if self.adaln_cache is not None:
|
||||
# prepare_adaln_plans resolved the slot on the host; the device
|
||||
# lookup remains for callers that drive forward() directly.
|
||||
# Skip-step forecasts still run embed + final_layer. Do not touch
|
||||
# per-block AdaLN or the 50-layer stack — that is what lets Spectrum
|
||||
# coexist with DiT layerwise offload.
|
||||
if run_transformer_blocks:
|
||||
if self.adaln_cache is not None:
|
||||
# prepare_adaln_plans resolved the slot on the host; the device
|
||||
# lookup remains for callers that drive forward() directly.
|
||||
adaln_cache_plan_index = kwargs.get("adaln_cache_slot")
|
||||
if adaln_cache_plan_index is None:
|
||||
adaln_cache_plan_index = self.adaln_cache.lookup(
|
||||
unique_timesteps.view(-1).to(device)
|
||||
)
|
||||
block_adaln_params = self.adaln_cache.block_all(
|
||||
cache_plan_index=adaln_cache_plan_index,
|
||||
num_timesteps=adaln_input.shape[0],
|
||||
)
|
||||
elif self._can_batch_block_adaln():
|
||||
local_adaln = torch.stack(
|
||||
[
|
||||
block.adaln_proj.project_local(adaln_input)
|
||||
for block in self.blocks
|
||||
]
|
||||
)
|
||||
gathered_adaln = tensor_model_parallel_all_gather(local_adaln)
|
||||
block_adaln_params = tuple(
|
||||
block.adaln_proj.split_output(output)
|
||||
for block, output in zip(self.blocks, gathered_adaln)
|
||||
)
|
||||
# With sequence parallelism, shard rows across the group for the
|
||||
# block stack. Attention trades sequence for heads internally
|
||||
# (Ulysses) and/or ring-rotates KV across ring ranks; everything
|
||||
# else, including the final layer, is row-local. Only the narrow
|
||||
# video/audio logits are gathered after the final layer.
|
||||
for index, block in enumerate(self.blocks):
|
||||
hidden = block(
|
||||
hidden,
|
||||
adaln_input=adaln_input,
|
||||
combined_indices=block_combined,
|
||||
rope_cache=rope_cache,
|
||||
cu_seqlens=cu_seqlens,
|
||||
cu_seqlens_host=cu_seqlens_host,
|
||||
max_seqlen=max_seqlen,
|
||||
subblock_sparse_query_block_mask=subblock_sparse_query_block_mask,
|
||||
ulysses_active=ulysses_ws > 1,
|
||||
ring_active=ring_ws > 1,
|
||||
adaln_params=(
|
||||
None
|
||||
if block_adaln_params is None
|
||||
else block_adaln_params[index]
|
||||
),
|
||||
)
|
||||
if spectrum_on:
|
||||
self._h3_spectrum_record_targets(
|
||||
hidden, audio_pos, infer_out_pos, row_start, row_stop
|
||||
)
|
||||
elif spectrum_on:
|
||||
hidden = self._h3_spectrum_predict_targets(
|
||||
hidden, audio_pos, infer_out_pos, row_start, row_stop
|
||||
)
|
||||
if self.adaln_cache is not None and adaln_cache_plan_index is None:
|
||||
adaln_cache_plan_index = kwargs.get("adaln_cache_slot")
|
||||
if adaln_cache_plan_index is None:
|
||||
adaln_cache_plan_index = self.adaln_cache.lookup(
|
||||
unique_timesteps.view(-1).to(device)
|
||||
)
|
||||
block_adaln_params = self.adaln_cache.block_all(
|
||||
cache_plan_index=adaln_cache_plan_index,
|
||||
num_timesteps=adaln_input.shape[0],
|
||||
)
|
||||
elif self._can_batch_block_adaln():
|
||||
local_adaln = torch.stack(
|
||||
[block.adaln_proj.project_local(adaln_input) for block in self.blocks]
|
||||
)
|
||||
gathered_adaln = tensor_model_parallel_all_gather(local_adaln)
|
||||
block_adaln_params = tuple(
|
||||
block.adaln_proj.split_output(output)
|
||||
for block, output in zip(self.blocks, gathered_adaln)
|
||||
)
|
||||
# With sequence parallelism, shard rows across the group for the
|
||||
# block stack. Attention trades sequence for heads internally
|
||||
# (Ulysses) and/or ring-rotates KV across ring ranks; everything
|
||||
# else, including the final layer, is row-local. Only the narrow
|
||||
# video/audio logits are gathered after the final layer.
|
||||
for index, block in enumerate(self.blocks):
|
||||
hidden = block(
|
||||
hidden,
|
||||
adaln_input=adaln_input,
|
||||
combined_indices=block_combined,
|
||||
rope_cache=rope_cache,
|
||||
cu_seqlens=cu_seqlens,
|
||||
cu_seqlens_host=cu_seqlens_host,
|
||||
max_seqlen=max_seqlen,
|
||||
subblock_sparse_query_block_mask=subblock_sparse_query_block_mask,
|
||||
ulysses_active=ulysses_ws > 1,
|
||||
ring_active=ring_ws > 1,
|
||||
adaln_params=(
|
||||
None if block_adaln_params is None else block_adaln_params[index]
|
||||
),
|
||||
)
|
||||
self.materialize_mps_non_layer_weights("final_layer")
|
||||
video_logits, audio_logits = self.final_layer(
|
||||
hidden,
|
||||
|
||||
+3
@@ -482,6 +482,9 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
|
||||
if enable_override is False:
|
||||
# The per-request kill switch wins over quality="high".
|
||||
desired_mode = None
|
||||
elif batch.sampling_params.enable_spectrum:
|
||||
# Spectrum skips the block stack; Cache-DiT wraps those blocks.
|
||||
desired_mode = None
|
||||
elif quality == "high":
|
||||
desired_mode = "high"
|
||||
else:
|
||||
|
||||
@@ -408,6 +408,7 @@ def test_high_quality_request_warns_when_bcg_suppresses_cache_dit():
|
||||
_explicit_fields={"quality"},
|
||||
enable_cache_dit=None,
|
||||
cache_dit_params=None,
|
||||
enable_spectrum=False,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -465,6 +466,7 @@ def test_extra_high_quality_does_not_enable_h3_cache_dit():
|
||||
_explicit_fields={"quality"},
|
||||
enable_cache_dit=None,
|
||||
cache_dit_params=None,
|
||||
enable_spectrum=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -263,7 +263,7 @@ def test_cache_dit_preservation_only_makes_first_gate_out_of_place():
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.models.dits.minimax_h3._modulate_scale_shift",
|
||||
"sglang.multimodal_gen.runtime.models.dits.minimax_h3._modulate_rmsnorm_scale_shift",
|
||||
side_effect=lambda value, *_args, **_kwargs: value,
|
||||
),
|
||||
patch(
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""H3 Spectrum packing helpers and sampling-param gates."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams
|
||||
from sglang.multimodal_gen.runtime.cache.spectrum import (
|
||||
H3_SPECTRUM_HISTORY_SIZE,
|
||||
blend_h3_spectrum_prediction,
|
||||
local_target_hidden,
|
||||
target_audio_video_span,
|
||||
)
|
||||
|
||||
|
||||
def test_target_span_uses_contiguous_audio_before_video():
|
||||
audio_pos = torch.tensor([4, 5, 6, 7, 8, 9])
|
||||
infer_out_pos = torch.tensor([10, 11, 12, 13])
|
||||
start, stop, video_stop = target_audio_video_span(audio_pos, infer_out_pos)
|
||||
assert (start, stop, video_stop) == (4, 10, 14)
|
||||
|
||||
|
||||
def test_target_span_drops_earlier_ref_audio_gap():
|
||||
audio_pos = torch.tensor([1, 2, 8, 9])
|
||||
infer_out_pos = torch.tensor([10, 11])
|
||||
start, stop, video_stop = target_audio_video_span(audio_pos, infer_out_pos)
|
||||
assert (start, stop, video_stop) == (8, 10, 12)
|
||||
|
||||
|
||||
def test_local_target_hidden_is_rank_local_view():
|
||||
hidden = torch.arange(20, dtype=torch.float32).unsqueeze(-1).expand(20, 3).clone()
|
||||
sliced = local_target_hidden(
|
||||
hidden, audio_start=4, video_stop=10, row_start=0, row_stop=20
|
||||
)
|
||||
assert sliced is not None
|
||||
local, start, stop = sliced
|
||||
assert start == 4 and stop == 10
|
||||
assert local.shape == (6, 3)
|
||||
|
||||
|
||||
def test_blend_holds_audio_and_mixes_video():
|
||||
predicted = torch.ones(6, 2)
|
||||
last_audio = torch.full((2, 2), 3.0)
|
||||
last_video = torch.full((4, 2), 5.0)
|
||||
out = blend_h3_spectrum_prediction(
|
||||
predicted.clone(),
|
||||
n_audio=2,
|
||||
last_audio=last_audio,
|
||||
last_video=last_video,
|
||||
audio_blend=0.0,
|
||||
video_blend=0.5,
|
||||
)
|
||||
assert torch.equal(out[:2], last_audio)
|
||||
assert torch.allclose(out[2:], torch.full((4, 2), 3.0))
|
||||
|
||||
|
||||
def test_h3_sampling_params_reject_quality_high_with_spectrum():
|
||||
with pytest.raises(ValueError, match="enable_spectrum cannot be combined"):
|
||||
MiniMaxH3SamplingParams(
|
||||
prompt="cat",
|
||||
enable_spectrum=True,
|
||||
quality="high",
|
||||
)
|
||||
|
||||
|
||||
def test_h3_sampling_params_default_history_is_comfy_window():
|
||||
params = MiniMaxH3SamplingParams(prompt="cat", enable_spectrum=True)
|
||||
assert params.spectrum_params is not None
|
||||
assert params.spectrum_params.history_size == H3_SPECTRUM_HISTORY_SIZE
|
||||
|
||||
|
||||
def test_h3_sampling_params_keep_explicit_history_size():
|
||||
params = MiniMaxH3SamplingParams(
|
||||
prompt="cat",
|
||||
enable_spectrum=True,
|
||||
spectrum_params={"history_size": 6},
|
||||
)
|
||||
assert params.spectrum_params.history_size == 6
|
||||
Reference in New Issue
Block a user