From ae47bcd4da527933ae599a8e95d7058c26999d32 Mon Sep 17 00:00:00 2001 From: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> Date: Sun, 27 Sep 2026 17:56:13 +0800 Subject: [PATCH] [diffusion] feat: minimax-h3 spectrum skip-step + fused RMSNorm/AdaLN (#35684) --- .../cookbook/diffusion/MiniMax/MiniMax-H3.mdx | 28 +- docs/docs/sglang-diffusion/spectrum.mdx | 16 +- .../sglang-diffusion-performance/SKILL.md | 2 +- .../configs/sample/minimax_h3.py | 21 ++ .../multimodal_gen/runtime/cache/spectrum.py | 100 +++++++ .../runtime/models/dits/minimax_h3.py | 277 ++++++++++++++---- .../minimax_h3/stages/denoising.py | 3 + .../test/unit/test_minimax_h3_admission.py | 2 + .../test/unit/test_minimax_h3_dit_contract.py | 2 +- .../test/unit/test_minimax_h3_spectrum.py | 80 +++++ 10 files changed, 461 insertions(+), 70 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_minimax_h3_spectrum.py diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 151f7c0362..467850be34 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -273,6 +273,19 @@ baseline; everything else stays identical. GPU peak stays about 18 GB either way because streaming offload is set by the offload buffers and VAE decode, not the weight dtype. +To skip DiT steps on 4090 while keeping `fa`, add Spectrum to the generate +command above. Do not combine it with `quality="high"` or Cache-DiT. +On this card, 50-step BF16+FA (**1008.6 s**) drops to **285.3 s (3.53×)** +with exact attention, up to **219.0 s (4.61×)** with Sage→Sol hybrid + +Spectrum. See the 4090 table in Benchmarks. + +```bash + --enable-spectrum \ + --spectrum-warmup-steps 11 \ + --spectrum-window-size 5 \ + --spectrum-flex-window 1.0 +``` + The first launch resolves every selected source through the normal Hub path. If a repository requires authentication, export a Hugging Face token in the server environment; no manual pre-download is required. @@ -1417,6 +1430,7 @@ fold decision is not node-boundary aware: - `--use-fsdp-inference true` shards only the DiT. MiniMax-H3 preserves the original FP32 dtype of its patch, time, and output projections during FSDP all-gather, so this path does not trade numerical correctness for memory. On 4×H100, prefer TP2 + Ulysses2 for speed; use FSDP as an explicit capacity policy rather than assuming it is faster. - `speed` keeps model components resident, while `auto` applies the model-aware 120 GiB residency threshold. `memory` prioritizes avoiding OOM and includes the executable VAE decoder in its default layerwise set. A measured recipe with sufficient headroom can opt into `--component-residency vae=resident`; the 2×H100 CI recipe does this because the VAE's 4.8 GiB/GPU cost avoids repeated decoder transfers during tiled decode. DiT residency and prefetch knobs remain scoped to the DiT. Use `speed` only after confirming that the complete target workload fits. - Breakable CUDA graph execution is an explicit opt-in, not part of the recommended `speed` preset. It requires `--enable-breakable-cuda-graph`, every served size in `--warmup-resolutions`, and `--bcg-text-buckets` that cover the live H3 condition sequence. The validated 1344×768 Ref2VA recipe uses 5504; other task profiles and reference sets may need a different value. It preserves eager output for matching captured signatures, but graph capture consumes additional GPU memory and may provide little latency benefit when Ulysses attention and collectives dominate, so benchmark it on the target topology before enabling it. +- `--enable-spectrum` skips selected DiT steps. On H3 it cannot combine with Cache-DiT / `quality="high"` or SP > 1. 50-step 4090 recipe is in §3. ## 11. Benchmarks @@ -1436,7 +1450,7 @@ the configurations with collected measurements: | Ascend NPU | 8 NPUs, TP2 + SP4, Laser Attention | 4 NPUs, TP2 + SP2, Laser Attention | | MI300X / MI355X | 8× Ulysses8 resident | 1×, 2×, and 4× scaling runs | | RTX 5090 | 1× layerwise offload, measured on a physical desktop | 2× TP2 + layerwise offload | -| RTX 4090 24 GB | 1× layerwise offload + `kitchen_int8` | Approximate attention backends are opt-in | +| RTX 4090 24 GB | 1× layerwise offload + `kitchen_int8` | Spectrum and approximate attention backends are opt-in | ### GB300 single-host @@ -2086,6 +2100,18 @@ and pixel fidelity rank in opposite orders there. Default remains `kitchen_int8` + `fa`. Cache-DiT can share this layerwise recipe; omit `quality` and see the quality-level section. +Same card, 50-step run. Speedup is vs BF16+FA. PSNR is vs same-tree INT8+FA (Spectrum off). + +| Config | e2e | vs BF16+FA | PSNR mean / min | +| --- | ---: | ---: | ---: | +| BF16 + FlashAttention | 1008.6 s | 1.00× | — | +| `kitchen_int8` + FA | 718.5 s | 1.40× | — | +| `kitchen_int8` + FA + Spectrum `11 / 5 / 1.0` | **285.3 s** | **3.53×** | **27.4 / 25.3** | +| `kitchen_int8` + `sage_attn` + Spectrum `15 / 5 / 0` | 245.3 s | 4.11× | 23.6 / 21.0 | +| `kitchen_int8` + Sage→Sol hybrid + Spectrum `15 / 5 / 0.5` | **219.0 s** | **4.61×** | 20.6 / 17.8 | + +Prefer FA + `11 / 5 / 1.0` when the bar is FA-like. Sage / hybrid + Spectrum is faster; quality then follows the attention backend, not the skip schedule. + ### AMD Instinct task and scaling runs The AMD recipes keep the released BF16/FP32 precision policy and use AITER diff --git a/docs/docs/sglang-diffusion/spectrum.mdx b/docs/docs/sglang-diffusion/spectrum.mdx index 55ca7fc7cb..cdc657517b 100644 --- a/docs/docs/sglang-diffusion/spectrum.mdx +++ b/docs/docs/sglang-diffusion/spectrum.mdx @@ -20,15 +20,19 @@ sglang generate \ ## Scope and constraints -- Available only on native FLUX.1, Wan, HunyuanVideo, and SD3 implementation - paths. It is not a `--backend diffusers` feature and does not currently - cover FLUX.2. +- Available on native FLUX.1, Wan, HunyuanVideo, SD3, and MiniMax-H3 + implementation paths. It is not a `--backend diffusers` feature and does + not currently cover FLUX.2. - The request control is available through `sglang generate` and Python sampling parameters. It is not a `sglang serve` or OpenAI-server request option yet. -- Spectrum and `--enable-teacache` are mutually exclusive. -- Start with the defaults. `--debug` adds shadow-prediction validation work, so - its latency is not representative of normal Spectrum execution. +- Spectrum and `--enable-teacache` are mutually exclusive. On MiniMax-H3 + it also cannot run with Cache-DiT / `quality="high"`, or with SP > 1. +- MiniMax-H3 50-step recipe: `--spectrum-warmup-steps 11 + --spectrum-window-size 5 --spectrum-flex-window 1.0`. Defaults are the + 20-step schedule. See the [MiniMax-H3 cookbook](/cookbook/diffusion/MiniMax/MiniMax-H3). +- Start with the defaults. `--debug` adds shadow-prediction validation work, + so its latency is not representative of normal Spectrum execution. ## Advanced controls diff --git a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md index 459ed2ea3c..7abdf1ee94 100644 --- a/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md +++ b/python/sglang/multimodal_gen/.claude/skills/sglang-diffusion-performance/SKILL.md @@ -168,7 +168,7 @@ These options **trade output quality** for speed or VRAM savings. Results will d | **Cache-DiT** | Native: per-request `--enable-cache-dit true\|false` + `--cache-dit-params ` (sampling params; also via API `extra_body`). `SGLANG_CACHE_DIT_ENABLED` / `SGLANG_CACHE_DIT_*` env vars are the server-wide defaults for requests that leave them unset. Diffusers backend: `--backend diffusers --cache-dit-config ` | Caches intermediate residuals across denoising steps and skips redundant computations via DBCache, TaylorSeer, and optional SCM. | ~1.5-2x on supported models | Quality depends on cache policy. Compatible with `--dit-layerwise-offload`: skipped blocks are not streamed, and the first layer after a skip may sync-load. Models that touch every layer before the block loop (for example a full-stack AdaLN prepass) must keep that prepass off while caching. Do not pass `--cache-dit-config` for native SGLang tuning unless you are intentionally using the diffusers backend flow. | | **CFG Gating** | Per-request `--cfg-gate-step 0.5` (sampling param; also via API `extra_body`). `SGLANG_DIFFUSION_CFG_GATE_STEP` is the server-wide default (1.0 = off). | After the given fraction of denoising steps, reuses the cached cond-uncond residual instead of running the unconditional branch each step. | Up to ~2x on the gated tail of CFG models (skips one of two branches) | Lossy; no-op without classifier-free guidance or with `--enable-cfg-parallel`. Lower fractions gate earlier and drift more. | | **TeaCache** | `--enable-teacache` (uses model sampling presets) | Reuses residuals when adjacent denoising steps are sufficiently similar. | Model- and threshold-dependent | Approximate and model-specific. Mutually exclusive with Spectrum. Fix prompt/seed/shape/steps and validate temporal consistency, not only single frames. | -| **Spectrum** | `--enable-spectrum` plus optional `--spectrum-*` controls | Forecasts DiT features and skips selected denoising steps. | Defaults target an accuracy/speed tradeoff; aggressive windows can be much faster | Native `sglang generate` only for FLUX.1, Wan, HunyuanVideo, and SD3; not FLUX.2 or server requests. Mutually exclusive with TeaCache. `--debug` adds shadow validation and is not representative latency. | +| **Spectrum** | `--enable-spectrum` plus optional `--spectrum-*` controls | Forecasts DiT features and skips selected denoising steps. | Defaults target an accuracy/speed tradeoff; aggressive windows can be much faster | Native `sglang generate` for FLUX.1, Wan, HunyuanVideo, SD3, and MiniMax-H3; not FLUX.2 or server requests. Mutually exclusive with TeaCache. `--debug` is not representative latency. | | **Progressive Resolution** | `--progressive-mode dct_rewind --progressive-levels N --progressive-delta D` | Runs early denoising at lower latent resolution, then spectrally upsamples and switches to the target resolution. | Model- and schedule-dependent | Approximate and pipeline-specific. Keep the switch schedule fixed and compare detail, composition, and temporal stability. | | **Causal KV-Cache Quantization** | `--kv-cache-quant int4\|int2` plus optional `--kv-cache-quant-*` controls | Compresses completed causal KV-cache chunks with Quant-VideoGen PRQ while keeping the mutable/current chunk and recent chunks in BF16. | Primarily a long-session memory saving | Currently limited to LingBot World realtime causal serving; requires `quant-videogen`. INT4 is the starting point; INT2 saves more memory with more error. It quantizes cache state, not checkpoint weights. | | **Quantized Models (Nunchaku / SVDQuant)** | `--enable-svdquant --transformer-weights-path ` + optional `--quantization-precision int4\|nvfp4`, `--quantization-rank 32` | W4A4-style quantization via [Nunchaku](https://nunchaku.tech). Reduces DiT weight memory by ~4x. Precision/rank can be auto-inferred from weight filename or set explicitly. | ~1.5–2x compute speedup | Lossy quantization; quality depends on rank and precision. Requires pre-quantized weights. Ampere (SM8x) or SM12x only (no Hopper SM90). Higher rank = better quality but more memory. | diff --git a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py index c78216d5b1..a6c158c738 100644 --- a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py @@ -240,6 +240,27 @@ class MiniMaxH3SamplingParams(SamplingParams): "MiniMax H3 does not support enable_teacache: its packed " "video/audio denoise loop has no lossless TeaCache contract" ) + if self.enable_spectrum: + if self.quality == "high": + raise ValueError( + "MiniMax H3 enable_spectrum cannot be combined with " + 'quality="high" (Cache-DiT). Use one skip-step accelerator.' + ) + if self.enable_cache_dit: + raise ValueError( + "MiniMax H3 enable_spectrum cannot be combined with " + "enable_cache_dit. Use one skip-step accelerator." + ) + from sglang.multimodal_gen.configs.sample.spectrum import SpectrumParams + from sglang.multimodal_gen.runtime.cache.spectrum import ( + apply_h3_spectrum_param_defaults, + ) + + if isinstance(self.spectrum_params, dict): + self.spectrum_params = SpectrumParams(**self.spectrum_params) + elif self.spectrum_params is None: + self.spectrum_params = SpectrumParams() + apply_h3_spectrum_param_defaults(self.spectrum_params) if self.rollout: raise ValueError( "MiniMax H3 does not support rollout: its coupled video/audio " diff --git a/python/sglang/multimodal_gen/runtime/cache/spectrum.py b/python/sglang/multimodal_gen/runtime/cache/spectrum.py index 1bcb9b9de6..4e39e45bfc 100644 --- a/python/sglang/multimodal_gen/runtime/cache/spectrum.py +++ b/python/sglang/multimodal_gen/runtime/cache/spectrum.py @@ -644,3 +644,103 @@ class SpectrumMixin: predicted = forecaster.predict(step_idx) self._spectrum_ctx = None return predicted.to(dtype=template.dtype, device=template.device) + + +# MiniMax-H3: forecast packed target audio|video only (Comfy / Wan2GP). + +H3_SPECTRUM_HISTORY_SIZE = 8 +H3_AUDIO_BLEND_WEIGHT = 0.0 +H3_VIDEO_BLEND_WEIGHT = 0.5 + + +def apply_h3_spectrum_param_defaults(params: SpectrumParams) -> SpectrumParams: + """Shrink the generic 100-slot GPU buffer default to the H3 CPU window.""" + if params.history_size == 100: + params.history_size = H3_SPECTRUM_HISTORY_SIZE + return params + + +def target_audio_video_span( + audio_pos: torch.Tensor, + infer_out_pos: torch.Tensor, +) -> tuple[int, int, int]: + """Return ``(audio_start, audio_stop, video_stop)`` in packed row indices. + + Target video is ``infer_out_pos``. Target audio is the contiguous + ``audio_pos`` run that ends immediately before that video span — the + packed t2va / fl2va / ref2va suffix. Earlier reference-audio rows are + dropped, matching Wan2GP ``hidden[audio_start:]``. + """ + audio_ids = audio_pos.detach().reshape(-1).to(dtype=torch.long, device="cpu") + video_ids = infer_out_pos.detach().reshape(-1).to(dtype=torch.long, device="cpu") + if audio_ids.numel() == 0 or video_ids.numel() == 0: + raise ValueError( + "MiniMax-H3 Spectrum needs non-empty target audio and video rows" + ) + video_start = int(video_ids.min()) + video_stop = int(video_ids.max()) + 1 + if int(video_ids.numel()) != video_stop - video_start: + raise ValueError( + "MiniMax-H3 Spectrum target video rows must be a contiguous packed span" + ) + + preceding = audio_ids[audio_ids < video_start] + if preceding.numel() == 0 or int(preceding.max()) + 1 != video_start: + raise ValueError( + "MiniMax-H3 Spectrum expects packed target audio rows immediately " + f"before target video rows starting at {video_start}" + ) + ordered = preceding.sort().values + expected = video_start - 1 + cursor = int(ordered.numel()) - 1 + while cursor >= 0 and int(ordered[cursor]) == expected: + expected -= 1 + cursor -= 1 + audio_start = expected + 1 + return audio_start, video_start, video_stop + + +def local_target_hidden( + hidden: torch.Tensor, + audio_start: int, + video_stop: int, + row_start: int, + row_stop: int, +) -> tuple[torch.Tensor, int, int] | None: + """Slice rank-local ``hidden`` to the overlap with packed target media.""" + start = max(audio_start, row_start) + stop = min(video_stop, row_stop) + if stop <= start: + return None + local_start = start - row_start + local_stop = stop - row_start + return hidden[local_start:local_stop], local_start, local_stop + + +def blend_h3_spectrum_prediction( + predicted: torch.Tensor, + *, + n_audio: int, + last_audio: torch.Tensor | None, + last_video: torch.Tensor | None, + audio_blend: float = H3_AUDIO_BLEND_WEIGHT, + video_blend: float = H3_VIDEO_BLEND_WEIGHT, +) -> torch.Tensor: + """Apply Comfy segment blends on a ``[audio | video]`` forecast.""" + if n_audio < 0 or n_audio > int(predicted.shape[0]): + raise ValueError( + f"n_audio={n_audio} is outside predicted rows {predicted.shape[0]}" + ) + out = predicted + audio = out[:n_audio] + video = out[n_audio:] + if last_audio is not None and audio.numel(): + held = last_audio.to(device=audio.device, dtype=audio.dtype) + if audio_blend <= 0.0: + audio.copy_(held) + elif audio_blend < 1.0: + audio.copy_(torch.lerp(held, audio, audio_blend)) + if last_video is not None and video.numel() and video_blend < 1.0: + held = last_video.to(device=video.device, dtype=video.dtype) + video.copy_(torch.lerp(held, video, video_blend)) + return out diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py index 276baf5281..29e8eaa01b 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py @@ -41,6 +41,12 @@ from sglang.multimodal_gen.configs.models.dits.minimax_h3 import ( MiniMaxH3DiTConfig, ) from sglang.multimodal_gen.configs.models.fsdp import is_block +from sglang.multimodal_gen.runtime.cache.spectrum import ( + SpectrumMixin, + blend_h3_spectrum_prediction, + local_target_hidden, + target_audio_video_span, +) from sglang.multimodal_gen.runtime.distributed import ( get_tp_world_size, tensor_model_parallel_all_gather, @@ -372,6 +378,23 @@ def _rotate_half(x: torch.Tensor) -> torch.Tensor: return torch.cat((-x2, x1), dim=-1) +def _modulate_rmsnorm_scale_shift( + x: torch.Tensor, + norm: nn.RMSNorm, + shift: torch.Tensor, + scale: torch.Tensor, + indices: torch.Tensor, + *, + dtype: torch.dtype, +) -> torch.Tensor: + """RMSNorm + indexed AdaLN. + + Keep ``nn.RMSNorm``; a fused RMSNorm+AdaLN kernel drifted 2-GPU + consistency GT for ~0.3% e2e, so it was removed. + """ + return _modulate_scale_shift(norm(x), shift, scale, indices, dtype=dtype) + + def _accepts_mxfp8_input(linear: nn.Module) -> bool: return linear.quant_method is not None and linear.quant_method.accepts_mxfp8_input( linear @@ -1430,17 +1453,27 @@ class MiniMaxH3DiTBlock(nn.Module): # first gated residual writes to that tensor; the second one operates on # a block-local buffer. residual = x - h = self.norm1(x) h_prequant = None - if _accepts_mxfp8_input(self.attn.qkv_proj) and can_use_mxfp8_swizzled(h): - # the bf16 modulated rows stay in h for the VDN branch projections - h, h_fp8, h_scales = indexed_scale_shift_mxfp8_( - h, shift_msa, scale_msa, combined_indices, keep_bf16=True - ) - h_prequant = (h_fp8, h_scales) + if _accepts_mxfp8_input(self.attn.qkv_proj): + h = self.norm1(x) + if can_use_mxfp8_swizzled(h): + # the bf16 modulated rows stay in h for the VDN branch projections + h, h_fp8, h_scales = indexed_scale_shift_mxfp8_( + h, shift_msa, scale_msa, combined_indices, keep_bf16=True + ) + h_prequant = (h_fp8, h_scales) + else: + h = _modulate_scale_shift( + h, shift_msa, scale_msa, combined_indices, dtype=_BF16_DTYPE + ) else: - h = _modulate_scale_shift( - h, shift_msa, scale_msa, combined_indices, dtype=_BF16_DTYPE + h = _modulate_rmsnorm_scale_shift( + x, + self.norm1, + shift_msa, + scale_msa, + combined_indices, + dtype=_BF16_DTYPE, ) h = self.attn( h, @@ -1463,15 +1496,26 @@ class MiniMaxH3DiTBlock(nn.Module): ) residual = x - h = self.norm2(x) - if _accepts_mxfp8_input(self.mlp.fc1) and can_use_mxfp8_swizzled(h): - _, h_fp8, h_scales = indexed_scale_shift_mxfp8_( - h, shift_mlp, scale_mlp, combined_indices, keep_bf16=False - ) - h = self.mlp((h_fp8, h_scales)) + if _accepts_mxfp8_input(self.mlp.fc1): + h = self.norm2(x) + if can_use_mxfp8_swizzled(h): + _, h_fp8, h_scales = indexed_scale_shift_mxfp8_( + h, shift_mlp, scale_mlp, combined_indices, keep_bf16=False + ) + h = self.mlp((h_fp8, h_scales)) + else: + h = _modulate_scale_shift( + h, shift_mlp, scale_mlp, combined_indices, dtype=_BF16_DTYPE + ) + h = self.mlp(h) else: - h = _modulate_scale_shift( - h, shift_mlp, scale_mlp, combined_indices, dtype=_BF16_DTYPE + h = _modulate_rmsnorm_scale_shift( + x, + self.norm2, + shift_mlp, + scale_mlp, + combined_indices, + dtype=_BF16_DTYPE, ) h = self.mlp(h) # `residual` is block-local here (see above), so this stays in-place @@ -1609,9 +1653,9 @@ class MiniMaxH3FinalLayer(nn.Module): video = audio = None for start in range(0, x.shape[0], _MPS_MLP_TOKEN_CHUNK_SIZE): stop = min(start + _MPS_MLP_TOKEN_CHUNK_SIZE, x.shape[0]) - h = self.norm(x[start:stop]) - h = _modulate_scale_shift( - h, + h = _modulate_rmsnorm_scale_shift( + x[start:stop], + self.norm, shift, scale, inverse_indices[start:stop], @@ -1636,8 +1680,9 @@ class MiniMaxH3FinalLayer(nn.Module): torch.mps.empty_cache() assert video is not None and audio is not None return video, audio - h = self.norm(x) - h = _modulate_scale_shift(h, shift, scale, inverse_indices, dtype=_BF16_DTYPE) + h = _modulate_rmsnorm_scale_shift( + x, self.norm, shift, scale, inverse_indices, dtype=_BF16_DTYPE + ) # Preserve full precision through both final output projections. h = h.to(_FP32_DTYPE) return self._project(h) @@ -1661,7 +1706,7 @@ def _reject_adaln_lora(names: list[str]) -> None: ) -class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin): +class MiniMaxH3DiTModel(SpectrumMixin, BaseDiT, LayerwiseOffloadableModuleMixin): _aliases = [ "MiniMaxH3Transformer3DModel", "MiniMaxH3PrunedTransformer3DModel", @@ -2033,7 +2078,11 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin): for index in range(arch.num_layers) ] ) - self.layer_names = ["token_refiner.blocks", "blocks"] + # Offload the 50-layer DiT stack before token_refiner. Each + # LayerwiseOffloadManager ends with model.to(device) for leftovers; + # putting the 2-layer refiner first would copy the full INT8 DiT onto + # a 24GB card and OOM. + self.layer_names = ["blocks", "token_refiner.blocks"] self.final_layer = MiniMaxH3FinalLayer( arch, quant_config, @@ -2064,6 +2113,9 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin): ) self._resolved_attention_backend: AttentionBackendEnum | None = None self._mark_missing_params_required() + self._init_spectrum_state() + self._h3_spectrum_last_audio: torch.Tensor | None = None + self._h3_spectrum_last_video: torch.Tensor | None = None def set_cache_dit_input_preservation(self, enabled: bool) -> None: """Stop the blocks from overwriting the input Cache-DiT holds by reference. @@ -2081,6 +2133,76 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin): for block in self.blocks: block.preserve_input_for_cache_dit = enabled + def _h3_spectrum_requested(self, *, sp_ws: int) -> bool: + """True when this forward should use Spectrum. Off path stays bit-exact.""" + try: + forward_batch = get_forward_context().forward_batch + except AssertionError: + return False + if forward_batch is None or not forward_batch.enable_spectrum: + return False + if sp_ws > 1: + raise ValueError( + "MiniMax-H3 Spectrum requires sequence-parallel size 1 so " + "target audio/video rows stay on one rank. Disable " + "--enable-spectrum or run without SP / Ulysses / ring." + ) + return True + + def _h3_spectrum_record_targets( + self, + hidden: torch.Tensor, + audio_pos: torch.Tensor, + infer_out_pos: torch.Tensor, + row_start: int, + row_stop: int, + ) -> None: + audio_start, audio_stop, video_stop = target_audio_video_span( + audio_pos, infer_out_pos + ) + sliced = local_target_hidden( + hidden, audio_start, video_stop, row_start, row_stop + ) + if sliced is None: + return + local, _, _ = sliced + n_audio = audio_stop - audio_start + cpu_local = local.detach().to(device="cpu", dtype=local.dtype) + self._h3_spectrum_last_audio = cpu_local[:n_audio].contiguous() + self._h3_spectrum_last_video = cpu_local[n_audio:].contiguous() + self.spectrum_record_features(cpu_local) + + def _h3_spectrum_predict_targets( + self, + hidden: torch.Tensor, + audio_pos: torch.Tensor, + infer_out_pos: torch.Tensor, + row_start: int, + row_stop: int, + ) -> torch.Tensor: + audio_start, audio_stop, video_stop = target_audio_video_span( + audio_pos, infer_out_pos + ) + sliced = local_target_hidden( + hidden, audio_start, video_stop, row_start, row_stop + ) + if sliced is None: + return hidden + local, local_start, local_stop = sliced + predicted = self.spectrum_predict_features( + local.detach().to(device="cpu", dtype=local.dtype) + ) + predicted = blend_h3_spectrum_prediction( + predicted, + n_audio=audio_stop - audio_start, + last_audio=self._h3_spectrum_last_audio, + last_video=self._h3_spectrum_last_video, + ) + hidden[local_start:local_stop].copy_( + predicted.to(device=hidden.device, dtype=hidden.dtype) + ) + return hidden + def _resolve_attention_backend_once(self) -> None: if self._resolved_attention_backend is not None: return @@ -2612,50 +2734,83 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin): hidden = decoder_input cu_seqlens = cu_seqlens.to(device) + spectrum_on = self._h3_spectrum_requested(sp_ws=sp_ws) + run_transformer_blocks = self.begin_spectrum_step() if spectrum_on else True + if spectrum_on: + try: + timestep = get_forward_context().current_timestep + except AssertionError: + timestep = None + if timestep == 0: + self._h3_spectrum_last_audio = None + self._h3_spectrum_last_video = None block_adaln_params = None adaln_cache_plan_index = None - if self.adaln_cache is not None: - # prepare_adaln_plans resolved the slot on the host; the device - # lookup remains for callers that drive forward() directly. + # Skip-step forecasts still run embed + final_layer. Do not touch + # per-block AdaLN or the 50-layer stack — that is what lets Spectrum + # coexist with DiT layerwise offload. + if run_transformer_blocks: + if self.adaln_cache is not None: + # prepare_adaln_plans resolved the slot on the host; the device + # lookup remains for callers that drive forward() directly. + adaln_cache_plan_index = kwargs.get("adaln_cache_slot") + if adaln_cache_plan_index is None: + adaln_cache_plan_index = self.adaln_cache.lookup( + unique_timesteps.view(-1).to(device) + ) + block_adaln_params = self.adaln_cache.block_all( + cache_plan_index=adaln_cache_plan_index, + num_timesteps=adaln_input.shape[0], + ) + elif self._can_batch_block_adaln(): + local_adaln = torch.stack( + [ + block.adaln_proj.project_local(adaln_input) + for block in self.blocks + ] + ) + gathered_adaln = tensor_model_parallel_all_gather(local_adaln) + block_adaln_params = tuple( + block.adaln_proj.split_output(output) + for block, output in zip(self.blocks, gathered_adaln) + ) + # With sequence parallelism, shard rows across the group for the + # block stack. Attention trades sequence for heads internally + # (Ulysses) and/or ring-rotates KV across ring ranks; everything + # else, including the final layer, is row-local. Only the narrow + # video/audio logits are gathered after the final layer. + for index, block in enumerate(self.blocks): + hidden = block( + hidden, + adaln_input=adaln_input, + combined_indices=block_combined, + rope_cache=rope_cache, + cu_seqlens=cu_seqlens, + cu_seqlens_host=cu_seqlens_host, + max_seqlen=max_seqlen, + subblock_sparse_query_block_mask=subblock_sparse_query_block_mask, + ulysses_active=ulysses_ws > 1, + ring_active=ring_ws > 1, + adaln_params=( + None + if block_adaln_params is None + else block_adaln_params[index] + ), + ) + if spectrum_on: + self._h3_spectrum_record_targets( + hidden, audio_pos, infer_out_pos, row_start, row_stop + ) + elif spectrum_on: + hidden = self._h3_spectrum_predict_targets( + hidden, audio_pos, infer_out_pos, row_start, row_stop + ) + if self.adaln_cache is not None and adaln_cache_plan_index is None: adaln_cache_plan_index = kwargs.get("adaln_cache_slot") if adaln_cache_plan_index is None: adaln_cache_plan_index = self.adaln_cache.lookup( unique_timesteps.view(-1).to(device) ) - block_adaln_params = self.adaln_cache.block_all( - cache_plan_index=adaln_cache_plan_index, - num_timesteps=adaln_input.shape[0], - ) - elif self._can_batch_block_adaln(): - local_adaln = torch.stack( - [block.adaln_proj.project_local(adaln_input) for block in self.blocks] - ) - gathered_adaln = tensor_model_parallel_all_gather(local_adaln) - block_adaln_params = tuple( - block.adaln_proj.split_output(output) - for block, output in zip(self.blocks, gathered_adaln) - ) - # With sequence parallelism, shard rows across the group for the - # block stack. Attention trades sequence for heads internally - # (Ulysses) and/or ring-rotates KV across ring ranks; everything - # else, including the final layer, is row-local. Only the narrow - # video/audio logits are gathered after the final layer. - for index, block in enumerate(self.blocks): - hidden = block( - hidden, - adaln_input=adaln_input, - combined_indices=block_combined, - rope_cache=rope_cache, - cu_seqlens=cu_seqlens, - cu_seqlens_host=cu_seqlens_host, - max_seqlen=max_seqlen, - subblock_sparse_query_block_mask=subblock_sparse_query_block_mask, - ulysses_active=ulysses_ws > 1, - ring_active=ring_ws > 1, - adaln_params=( - None if block_adaln_params is None else block_adaln_params[index] - ), - ) self.materialize_mps_non_layer_weights("final_layer") video_logits, audio_logits = self.final_layer( hidden, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index ab65f65c54..1b7b65c846 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -482,6 +482,9 @@ class MiniMaxH3DenoisingStage(DenoisingStage): if enable_override is False: # The per-request kill switch wins over quality="high". desired_mode = None + elif batch.sampling_params.enable_spectrum: + # Spectrum skips the block stack; Cache-DiT wraps those blocks. + desired_mode = None elif quality == "high": desired_mode = "high" else: diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_admission.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_admission.py index 1c53dce4fc..a3cfe95c26 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_admission.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_admission.py @@ -408,6 +408,7 @@ def test_high_quality_request_warns_when_bcg_suppresses_cache_dit(): _explicit_fields={"quality"}, enable_cache_dit=None, cache_dit_params=None, + enable_spectrum=False, ) ) @@ -465,6 +466,7 @@ def test_extra_high_quality_does_not_enable_h3_cache_dit(): _explicit_fields={"quality"}, enable_cache_dit=None, cache_dit_params=None, + enable_spectrum=False, ) ) diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_dit_contract.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_dit_contract.py index ab7393cf5b..a6179eda03 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_dit_contract.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_dit_contract.py @@ -263,7 +263,7 @@ def test_cache_dit_preservation_only_makes_first_gate_out_of_place(): with ( patch( - "sglang.multimodal_gen.runtime.models.dits.minimax_h3._modulate_scale_shift", + "sglang.multimodal_gen.runtime.models.dits.minimax_h3._modulate_rmsnorm_scale_shift", side_effect=lambda value, *_args, **_kwargs: value, ), patch( diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_spectrum.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_spectrum.py new file mode 100644 index 0000000000..e8fb6acc80 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_spectrum.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +"""H3 Spectrum packing helpers and sampling-param gates.""" + +from __future__ import annotations + +import pytest +import torch + +from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams +from sglang.multimodal_gen.runtime.cache.spectrum import ( + H3_SPECTRUM_HISTORY_SIZE, + blend_h3_spectrum_prediction, + local_target_hidden, + target_audio_video_span, +) + + +def test_target_span_uses_contiguous_audio_before_video(): + audio_pos = torch.tensor([4, 5, 6, 7, 8, 9]) + infer_out_pos = torch.tensor([10, 11, 12, 13]) + start, stop, video_stop = target_audio_video_span(audio_pos, infer_out_pos) + assert (start, stop, video_stop) == (4, 10, 14) + + +def test_target_span_drops_earlier_ref_audio_gap(): + audio_pos = torch.tensor([1, 2, 8, 9]) + infer_out_pos = torch.tensor([10, 11]) + start, stop, video_stop = target_audio_video_span(audio_pos, infer_out_pos) + assert (start, stop, video_stop) == (8, 10, 12) + + +def test_local_target_hidden_is_rank_local_view(): + hidden = torch.arange(20, dtype=torch.float32).unsqueeze(-1).expand(20, 3).clone() + sliced = local_target_hidden( + hidden, audio_start=4, video_stop=10, row_start=0, row_stop=20 + ) + assert sliced is not None + local, start, stop = sliced + assert start == 4 and stop == 10 + assert local.shape == (6, 3) + + +def test_blend_holds_audio_and_mixes_video(): + predicted = torch.ones(6, 2) + last_audio = torch.full((2, 2), 3.0) + last_video = torch.full((4, 2), 5.0) + out = blend_h3_spectrum_prediction( + predicted.clone(), + n_audio=2, + last_audio=last_audio, + last_video=last_video, + audio_blend=0.0, + video_blend=0.5, + ) + assert torch.equal(out[:2], last_audio) + assert torch.allclose(out[2:], torch.full((4, 2), 3.0)) + + +def test_h3_sampling_params_reject_quality_high_with_spectrum(): + with pytest.raises(ValueError, match="enable_spectrum cannot be combined"): + MiniMaxH3SamplingParams( + prompt="cat", + enable_spectrum=True, + quality="high", + ) + + +def test_h3_sampling_params_default_history_is_comfy_window(): + params = MiniMaxH3SamplingParams(prompt="cat", enable_spectrum=True) + assert params.spectrum_params is not None + assert params.spectrum_params.history_size == H3_SPECTRUM_HISTORY_SIZE + + +def test_h3_sampling_params_keep_explicit_history_size(): + params = MiniMaxH3SamplingParams( + prompt="cat", + enable_spectrum=True, + spectrum_params={"history_size": 6}, + ) + assert params.spectrum_params.history_size == 6