diff --git a/docker/npu.Dockerfile b/docker/npu.Dockerfile index a2ba523e5..39ebaf042 100644 --- a/docker/npu.Dockerfile +++ b/docker/npu.Dockerfile @@ -55,6 +55,7 @@ RUN apt-get update -y && apt upgrade -y && apt-get install -y \ clang \ locales \ ccache \ + ffmpeg \ openssl \ libssl-dev \ pkg-config \ diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index eae27a0d6..81d4e06de 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -369,6 +369,70 @@ values. SGLang rejects a request outside that coverage instead of silently changing conditioning. Cache mode supports the matching unquantized checkpoint only. +### Serve MiniMax-H3 on Ascend NPUs + +For Ascend NPU, follow the +[NPU installation guide](/docs/hardware-platforms/ascend-npus/getting-started/installation) +before starting the server. + +The Ascend commands below explicitly enable the Cache-DiT configuration used +for the reported performance measurements. Remove these `SGLANG_CACHE_DIT_*` +variables to use lossless denoising. See the **Ascend NPU topology comparison** +in the Benchmarks section for the measured eight- and four-NPU latency. + +The measured latency configuration also passes `--dit-cpu-offload false` to +keep the transformer resident on the NPUs. Omit this flag when lower device +memory usage is more important than avoiding CPU-to-NPU transfer latency. + +For an eight-NPU host, the validated topology is TP2 + SP4 with Laser +Attention. Use Ascend Flash Attention by replacing `laser_attn` with `fa`. + +```bash 8-NPU +SGLANG_CACHE_DIT_ENABLED=true \ +SGLANG_CACHE_DIT_FN=2 \ +SGLANG_CACHE_DIT_BN=1 \ +SGLANG_CACHE_DIT_WARMUP=4 \ +SGLANG_CACHE_DIT_RDT=0.4 \ +SGLANG_CACHE_DIT_MC=4 \ +SGLANG_CACHE_DIT_TAYLORSEER=true \ +SGLANG_CACHE_DIT_TS_ORDER=2 \ +HCCL_BUFFSIZE=256 sglang serve \ + --model-path MiniMaxAI/MiniMax-H3 \ + --model-type diffusion \ + --model-variant fl2va \ + --dit-cpu-offload false \ + --num-gpus 8 \ + --tp-size 2 \ + --sp-degree 4 \ + --attention-backend laser_attn \ + --port 30088 \ + --component-residency text_encoder=layerwise-offload +``` + +For a four-NPU host, use TP2 + SP2: + +```bash 4-NPU +SGLANG_CACHE_DIT_ENABLED=true \ +SGLANG_CACHE_DIT_FN=2 \ +SGLANG_CACHE_DIT_BN=1 \ +SGLANG_CACHE_DIT_WARMUP=4 \ +SGLANG_CACHE_DIT_RDT=0.4 \ +SGLANG_CACHE_DIT_MC=4 \ +SGLANG_CACHE_DIT_TAYLORSEER=true \ +SGLANG_CACHE_DIT_TS_ORDER=2 \ +HCCL_BUFFSIZE=256 sglang serve \ + --model-path MiniMaxAI/MiniMax-H3 \ + --model-type diffusion \ + --model-variant fl2va \ + --dit-cpu-offload false \ + --num-gpus 4 \ + --tp-size 2 \ + --sp-degree 2 \ + --attention-backend laser_attn \ + --port 30088 \ + --component-residency text_encoder=layerwise-offload +``` + ## 4. Generate video and audio MiniMax-H3 uses the asynchronous OpenAI-compatible video endpoint. Choose a @@ -1087,10 +1151,27 @@ the configurations with collected measurements: | B200 | 8× Ulysses8 resident | 4× FSDP + Ulysses4 | | H200 | 4× Ulysses4 resident | 4× FSDP + Ulysses4; 4× TP2 + Ulysses2; 2 nodes × 8× Ulysses8×Ring2 cross-node | | H100 | 4× TP2 + Ulysses2 resident | 4× TP4 + Ulysses1; 4× FSDP + Ulysses4 | +| Ascend NPU | 8 NPUs, TP2 + SP4, Laser Attention | 4 NPUs, TP2 + SP2, Laser Attention | | MI300X / MI355X | 8× Ulysses8 resident | 1×, 2×, and 4× scaling runs | | RTX 5090 | 2× TP2 + layerwise offload | — | | RTX 4090 24 GB | 1× layerwise offload + `kitchen_int8` | Approximate attention backends are opt-in | +### Ascend NPU topology comparison + +Both topologies used Laser Attention and the explicit Cache-DiT configuration +from the Ascend launch commands, with `--dit-cpu-offload false` keeping the DiT +resident. The measured workload was one 5-second T2VA request at 1344×768, +124 frames, 24 fps, and 50 inference steps. + +| NPU count | Topology | End-to-end latency | +| ---: | --- | ---: | +| 8 | TP2 + SP4 | **55.07 s** | +| 4 | TP2 + SP2 | **103.57 s** | + +These are individual end-to-end measurements for each topology, not averages. +The eight-NPU topology had 46.8% lower end-to-end latency than the four-NPU +topology. + ### B300 precision and encoder placement A 12-configuration sweep on a single 8× B300 host, covering both checkpoint diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/laser_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/laser_attn.py index 21fe86310..487a4f936 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/laser_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/laser_attn.py @@ -9,18 +9,26 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import SDPABac from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +logger = init_logger(__name__) + # Import to use torch.ops.attentions, install package with sgl_kernel_npu try: import attentions # noqa: F401 except ImportError as e: + logger.warning_once( + "The 'attentions' library is not installed. Laser Attention is unavailable. " + "Installing this library may improve performance on NPU. " + "See: sgl-project/sgl-kernel-npu" + ) raise ImportError( ( - "The required 'attentions' package is not installed." - "The package can be installed with sgl_kernel_npu" + "The required 'attentions' package is not installed. " + "Install it from sgl-project/sgl-kernel-npu." ) ) from e -logger = init_logger(__name__) +# The current NPU kernel stores QK scores and V in FP16 even for BF16 inputs. +_BF16_LASER_SCALE = 256.0 class LaserAttentionBackend(AttentionBackend): @@ -102,14 +110,32 @@ class LaserAttentionImpl(AttentionImpl): return torch.nn.functional.pad(input_tensor, pad_list) def _la_preprocess_input( - self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + *, + preserve_bf16_range: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, float, float]: # Currently BSND input layout is not supported q = query.transpose(1, 2) k = key.transpose(1, 2) v = value.transpose(1, 2) - if q.dtype != torch.float16: + q_scale = 1.0 + k_scale = 1.0 + value_scale = 1.0 + if preserve_bf16_range: + q_scale = _BF16_LASER_SCALE if q.dtype == torch.bfloat16 else 1.0 + k_scale = _BF16_LASER_SCALE if k.dtype == torch.bfloat16 else 1.0 + value_scale = _BF16_LASER_SCALE if v.dtype == torch.bfloat16 else 1.0 + if q.dtype != torch.float16: + q = q.mul(1.0 / q_scale).to(torch.float16) + if k.dtype != torch.float16: + k = k.mul(1.0 / k_scale).to(torch.float16) + if v.dtype != torch.float16: + v = v.mul(1.0 / value_scale).to(torch.float16) + elif q.dtype != torch.float16: q = q.to(torch.float16) k = k.to(torch.float16) v = v.to(torch.float16) @@ -118,7 +144,7 @@ class LaserAttentionImpl(AttentionImpl): k = self._pad(k) v = self._pad(v) - return q, k, v + return q, k, v, q_scale * k_scale, value_scale def _la_postprocess_output( self, @@ -141,6 +167,7 @@ class LaserAttentionImpl(AttentionImpl): value: torch.Tensor, head_num: int, pre_tokens: int, + scale_value: float, ) -> tuple[torch.Tensor, torch.Tensor]: return torch.ops.attentions.la( query=query, @@ -149,7 +176,7 @@ class LaserAttentionImpl(AttentionImpl): atten_mask=None, alibi_mask=None, drop_mask=None, - scale_value=self.softmax_scale, + scale_value=scale_value, head_num=head_num, input_layout="BNSD", keep_prob=1.0, @@ -164,6 +191,17 @@ class LaserAttentionImpl(AttentionImpl): key: torch.Tensor, value: torch.Tensor, attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + return self._forward_dense(query, key, value, attn_metadata) + + def _forward_dense( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata, + *, + preserve_bf16_range: bool = False, ) -> torch.Tensor: q_seqlen, head_dim = query.shape[1], query.shape[3] kv_seqlen = key.shape[1] @@ -182,10 +220,91 @@ class LaserAttentionImpl(AttentionImpl): kv_seqlen // self.seq_len_pad_base + 1 ) * self.seq_len_pad_base - kv_seqlen - q, k, v = self._la_preprocess_input(query, key, value) - _, la_output = self._laser_attention(q, k, v, q.shape[1], pre_tokens) + q, k, v, qk_scale, value_scale = self._la_preprocess_input( + query, + key, + value, + preserve_bf16_range=preserve_bf16_range, + ) + _, la_output = self._laser_attention( + q, + k, + v, + q.shape[1], + pre_tokens, + self.softmax_scale * qk_scale, + ) + if value_scale != 1.0: + la_output.mul_(value_scale) output = self._la_postprocess_output( la_output, query.dtype, q_seqlen, head_dim ) return output + + def forward_varlen( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + *, + cu_seqlens: torch.Tensor, + max_seqlen: int, + cu_seqlens_host: tuple[int, ...] | None = None, + ) -> torch.Tensor: + del max_seqlen + bounds = ( + cu_seqlens_host + if cu_seqlens_host is not None + else tuple(int(item) for item in cu_seqlens.tolist()) + ) + # MiniMax-H3 is the current Laser varlen caller and encodes one real + # segment followed by alignment padding as [0, used, padded]. + padding_start = ( + bounds[1] + if len(bounds) == 3 and bounds[0] == 0 and bounds[-1] == query.shape[0] + else None + ) + # Packed segments are independent; a single dense call would let real + # tokens attend alignment padding in MiniMax-H3. + # MiniMax-H3 packs one real segment followed by alignment padding. + # Delay allocation until Laser releases its temporary tensors and + # avoid allocating/copying the result when no padding was added. + if ( + padding_start is not None + and padding_start > 0 + and bounds[0] == 0 + and bounds[1] == padding_start + ): + segment = self._forward_dense( + query[:padding_start].unsqueeze(0), + key[:padding_start].unsqueeze(0), + value[:padding_start].unsqueeze(0), + None, + preserve_bf16_range=True, + )[0] + if padding_start == query.shape[0]: + return segment + + output = torch.empty_like(query) + output[:padding_start].copy_(segment) + output[padding_start:].zero_() + return output + + output = torch.empty_like(query) + for start, stop in zip(bounds[:-1], bounds[1:]): + if padding_start is not None and start >= padding_start: + break + if start == stop: + continue + segment = self._forward_dense( + query[start:stop].unsqueeze(0), + key[start:stop].unsqueeze(0), + value[start:stop].unsqueeze(0), + None, + preserve_bf16_range=padding_start is not None, + ) + output[start:stop].copy_(segment[0]) + if padding_start is not None: + output[padding_start:].zero_() + return output diff --git a/python/sglang/multimodal_gen/runtime/layers/layernorm.py b/python/sglang/multimodal_gen/runtime/layers/layernorm.py index fe47a9fab..356d25ffa 100755 --- a/python/sglang/multimodal_gen/runtime/layers/layernorm.py +++ b/python/sglang/multimodal_gen/runtime/layers/layernorm.py @@ -37,6 +37,7 @@ _is_musa = current_platform.is_musa() _is_cpu = current_platform.is_cpu() _is_xpu = current_platform.is_xpu() _use_rocm_flydsl = get_bool_env_var("SGLANG_USE_ROCM_FLYDSL") +_has_attentions = False if _is_cuda or _is_xpu: from sgl_kernel import fused_add_rmsnorm, rmsnorm @@ -47,6 +48,20 @@ if _is_npu: fused_rmsnorm_without_weight, ) + try: + import attentions # noqa: F401 + + _has_attentions = True + except ImportError: + from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + + logger = init_logger(__name__) # pylint: disable=invalid-name + logger.warning_once( + "The 'attentions' library is not installed. Falling back to native layernorm. " + "Installing this library may improve performance on NPU. " + "See: sgl-project/sgl-kernel-npu" + ) + if _is_musa: from sgl_kernel import fused_add_rmsnorm @@ -452,21 +467,8 @@ class FP32LayerNorm(CustomOp, nn.LayerNorm): ) self._forward_method = self.dispatch_forward() - if _is_npu: - try: - import attentions # noqa: F401 - except ImportError: - from sglang.multimodal_gen.runtime.utils.logging_utils import ( - init_logger, - ) - - logger = init_logger(__name__) # pylint: disable=invalid-name - logger.warning( - "The 'attentions' library is not installed. Falling back to native layernorm. " - "Installing this library may improve performance on NPU." - "See: sgl-project/sgl-kernel-npu" - ) - self._forward_method = self.forward_native + if _is_npu and not _has_attentions: + self._forward_method = self.forward_native def _cached_fp32_param( self, attr: str, param: torch.Tensor | None, device: torch.device diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py index 865da3c63..d2e5e4489 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py @@ -478,9 +478,11 @@ class ComponentResidencyManager: if should_keep: return strategy = self.strategy_for(use.component_name, module) - was_on_cuda = self._module_on_cuda(module) + was_on_supported_device = self._module_on_supported_device(module) strategy.finish_use(module, use, self.state) - self._empty_cache_after_large_release(use, strategy, module, was_on_cuda) + self._empty_cache_after_large_release( + use, strategy, module, was_on_supported_device + ) def finish_request(self) -> None: self.finish_active_use(prefetch_next=False) @@ -503,9 +505,11 @@ class ComponentResidencyManager: not self._is_single_dit_component(component_name) or keep_single_dit ) strategy = self.strategy_for(component_name, module) - was_on_cuda = self._module_on_cuda(module) + was_on_supported_device = self._module_on_supported_device(module) strategy.finish_request(module, use, self.state, preferred=preferred) - self._empty_cache_after_large_release(use, strategy, module, was_on_cuda) + self._empty_cache_after_large_release( + use, strategy, module, was_on_supported_device + ) def stage_name(self, stage: ComponentResidencyStage) -> str: return self._stage_names_by_id.get(id(stage), stage.__class__.__name__) @@ -634,21 +638,33 @@ class ComponentResidencyManager: buffer = next(module.buffers(), None) return buffer.device.type if buffer is not None else None - def _module_on_cuda(self, module: nn.Module | None) -> bool: - return self._module_device(module) == "cuda" + def _module_on_supported_device(self, module: nn.Module | None) -> bool: + is_supported_platform = ( + current_platform.is_cuda() + or current_platform.is_rocm() + or current_platform.is_npu() + ) + return is_supported_platform and current_platform.is_device_type( + self._module_device(module) + ) def _empty_cache_after_large_release( self, use: ComponentUse, strategy: ComponentResidencyStrategy, module: nn.Module, - was_on_cuda: bool, + was_on_supported_device: bool, ) -> None: if not use.memory_intensive: return - released_cuda_storage = was_on_cuda and not self._module_on_cuda(module) + released_device_storage = ( + was_on_supported_device and not self._module_on_supported_device(module) + ) released_layerwise_storage = isinstance(strategy, LayerwiseOffloadStrategy) - if not (released_cuda_storage or released_layerwise_storage): + should_empty_component_cache = ( + released_device_storage and not current_platform.is_npu() + ) + if not (should_empty_component_cache or released_layerwise_storage): return if not torch.get_device_module().is_available(): return diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/base_module.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/base_module.py index a270a5cd9..ec6727388 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/base_module.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/base_module.py @@ -13,6 +13,8 @@ from sglang.kernels.ops.activation.activation import ( silu_and_mul_with_activation_rounding, ) from sglang.kernels.ops.diffusion import try_fused_scaled_residual_add_exact +from sglang.multimodal_gen.runtime.layers.activation import SiluAndMul +from sglang.multimodal_gen.runtime.platforms import current_platform from .attention import Attention from .vit_utils import _env_flag, _vit_torch_compile_kwargs @@ -62,6 +64,12 @@ class FeedForward(nn.Module): else: raise ValueError(f"Unsupported activation function: {activation_fn}") + self.silu_and_mul = ( + SiluAndMul() + if use_gated and activation_fn == "silu" and current_platform.is_npu() + else None + ) + self.w2 = nn.Linear(inner_dim, dim_out, bias=bias) self._compile_forward_enabled = _env_flag( "MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE", "0" @@ -83,6 +91,8 @@ class FeedForward(nn.Module): and hidden_states.shape[-1] % 32 == 0 ): hidden_states = silu_and_mul_with_activation_rounding(hidden_states) + elif self.silu_and_mul is not None: + hidden_states = self.silu_and_mul(hidden_states) else: gate, hidden_states = hidden_states.chunk(2, dim=-1) hidden_states = self.act_fn(gate).mul_(hidden_states) diff --git a/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py index 1b1cf38f6..8aee42fcf 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +import shutil + from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import ( MiniMaxH3PipelineConfig, ) @@ -24,6 +26,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.m MiniMaxH3PartitionAdmissionStage, MiniMaxH3ReleaseMetadata, ) +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.server_args import ServerArgs @@ -46,6 +49,24 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase): "transformer", ] + def __init__(self, *args, **kwargs): + # TODO: Enable this check on ROCm after adding ffmpeg to the AMD Docker + # image and CI dependency installer. + if not current_platform.is_rocm(): + missing_media_tools = [ + executable + for executable in ("ffmpeg", "ffprobe") + if shutil.which(executable) is None + ] + if missing_media_tools: + raise RuntimeError( + "MiniMax H3 requires ffmpeg and ffprobe for media processing " + "and validated output delivery; missing executables: " + f"{', '.join(missing_media_tools)}. Install the ffmpeg system " + "package before starting SGLang." + ) + super().__init__(*args, **kwargs) + @staticmethod def model_subfolder_for_variant(variant: str) -> str: if not isinstance(variant, str) or not variant.strip(): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/keyframe_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/keyframe_encoding.py index e59564715..49d74ae63 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/keyframe_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/keyframe_encoding.py @@ -26,6 +26,7 @@ from sglang.multimodal_gen.configs.models.vaes.minimax_h3_video import ( from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.packed_tokens import ( minimax_h3_patchify_video_latent, ) +from sglang.multimodal_gen.runtime.platforms import current_platform MINIMAX_H3_KEYFRAME_ENCODE_SEED = 42 MINIMAX_H3_KEYFRAME_PATCH_SIZE = (1, 2, 2) @@ -37,16 +38,37 @@ def minimax_h3_scoped_encode_rng(seed: int, device: torch.device | None = None): The encode recipes seed the default torch generators right before a posterior-sampled VAE encode. Forking restores the process-global CPU and - CUDA generators after the encode while preserving the exact sampled result. + device generators after the encode while preserving the exact sampled + result. """ devices: list[torch.device] = [] - if device is not None and device.type == "cuda" and torch.cuda.is_available(): - devices = [device] - with torch.random.fork_rng(devices=devices): + device_module = None + device_type = None + is_supported_backend = ( + current_platform.is_cuda() + or current_platform.is_rocm() + or current_platform.is_npu() + ) + if ( + device is not None + and is_supported_backend + and device.type == current_platform.device_type + ): + device_module = torch.get_device_module(device) + if device_module.is_available(): + devices = [device] + device_type = current_platform.device_type + fork_rng_context = ( + torch.random.fork_rng(devices=devices) + if device_type is None + else torch.random.fork_rng(devices=devices, device_type=device_type) + ) + with fork_rng_context: torch.default_generator.manual_seed(int(seed)) for forked_device in devices: - with torch.cuda.device(forked_device): - torch.cuda.manual_seed(int(seed)) + assert device_module is not None + with device_module.device(forked_device): + device_module.manual_seed(int(seed)) yield diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py index 33d80ad94..29ab53965 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py @@ -29,7 +29,8 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( ) from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.precision import ( - autocast_enabled, + autocast_context, + autocast_enabled_for_device, resolve_decode_precision, resolve_precision, ) @@ -293,9 +294,8 @@ class MiniMaxH3DecodingStage(DecodingStage): audio_vae_dtype = resolve_precision( server_args, "audio_vae", precision_attr="audio_vae_precision" ) - audio_autocast_enabled = ( - audio_latent.device.type == "cuda" - and autocast_enabled(audio_vae_dtype, server_args.disable_autocast) + audio_autocast_enabled = autocast_enabled_for_device( + audio_latent, audio_vae_dtype, server_args.disable_autocast ) autocast_context = ( torch.autocast( @@ -352,22 +352,16 @@ class MiniMaxH3DecodingStage(DecodingStage): name="video_vae", ) video_vae_dtype = resolve_decode_precision(server_args, "video_vae") - visual_autocast_enabled = ( - visual_latent.device.type == "cuda" - and autocast_enabled(video_vae_dtype, server_args.disable_autocast) + visual_autocast_enabled = autocast_enabled_for_device( + visual_latent, video_vae_dtype, server_args.disable_autocast ) if visual_autocast_enabled: selected_video_vae.prepare_decoder_autocast_weights(video_vae_dtype) - autocast_context = ( - torch.autocast( - device_type="cuda", - dtype=video_vae_dtype, - enabled=visual_autocast_enabled, - ) - if visual_latent.is_cuda - else nullcontext() - ) - with autocast_context: + with autocast_context( + video_vae_dtype, + server_args.disable_autocast, + enabled=visual_autocast_enabled, + ): video_decode = self._get_vae_decode_fn( selected_video_vae, server_args, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index 1337459f9..da7f26bcf 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -620,8 +620,15 @@ class MiniMaxH3DenoisingStage(DenoisingStage): ctx = _resolve_full_loop_context(batch) - if not (current_platform.is_cuda() or current_platform.is_mps()): - raise RuntimeError("MiniMax H3 full-loop denoise requires CUDA or MPS") + if not ( + current_platform.is_cuda() + or current_platform.is_mps() + or current_platform.is_npu() + ): + raise RuntimeError( + "MiniMax H3 full-loop denoise requires CUDA, MPS, or Ascend NPU" + ) + device = current_platform.get_local_torch_device() sigmas_video = [float(v) for v in ctx.sigmas["video"]] self._maybe_enable_cache_dit_and_torch_compile( diff --git a/python/sglang/multimodal_gen/runtime/platforms/interface.py b/python/sglang/multimodal_gen/runtime/platforms/interface.py index a62478f98..2c7866b0d 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/interface.py +++ b/python/sglang/multimodal_gen/runtime/platforms/interface.py @@ -188,6 +188,10 @@ class Platform: """Stateless version of :func:`torch.cuda.is_available`.""" return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM, PlatformEnum.MUSA) + def is_device_type(self, device_type: str | None) -> bool: + """Return whether a device type belongs to this platform.""" + return device_type == self.device_type + @lru_cache(maxsize=1) def is_mps(self) -> bool: return self._enum == PlatformEnum.MPS diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index df84ff700..73d1b7bf7 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -953,6 +953,23 @@ class ServerArgs(DisaggServerArgsMixin): text_backend, ) self.component_attention_backends["text_encoder"] = "torch_sdpa" + from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import ( + MiniMaxH3PipelineConfig, + ) + + if ( + self.backend != Backend.DIFFUSERS + and isinstance(self.pipeline_config, MiniMaxH3PipelineConfig) + and self.attention_backend == "laser_attn" + and "text_encoder" not in self.component_attention_backends + ): + # Laser Attention is used only by the MiniMax-H3 transformer. + # SDPA is faster than Ascend FA for its Qwen3-VL text encoder. + logger.info( + "Automatically set torch_sdpa backend for the MiniMax H3 text " + "encoder; laser_attn applies to the transformer" + ) + self.component_attention_backends["text_encoder"] = "torch_sdpa" if self.ring_degree > 1: if ( diff --git a/python/sglang/multimodal_gen/runtime/utils/precision.py b/python/sglang/multimodal_gen/runtime/utils/precision.py index e61f5ede6..6397d781f 100644 --- a/python/sglang/multimodal_gen/runtime/utils/precision.py +++ b/python/sglang/multimodal_gen/runtime/utils/precision.py @@ -107,6 +107,14 @@ def autocast_enabled(dtype: torch.dtype, disable_autocast: bool) -> bool: ) +def autocast_enabled_for_device( + tensor: torch.Tensor, dtype: torch.dtype, disable_autocast: bool +) -> bool: + return tensor.device.type == current_platform.device_type and autocast_enabled( + dtype, disable_autocast + ) + + def autocast_context( dtype: torch.dtype, disable_autocast: bool, diff --git a/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py b/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py index 737328f4f..ef646f5f7 100644 --- a/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py +++ b/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py @@ -46,7 +46,7 @@ def _all_cases() -> list[DiffusionTestCase]: def _baseline_path() -> Path: import sglang.multimodal_gen.test.server.testcase_configs as cfg - return cfg.get_perf_baseline_path() + return cfg.get_perf_baseline_update_path() def _openai_client(port: int) -> OpenAI: diff --git a/python/sglang/multimodal_gen/test/server/ascend/perf_baselines_npu.json b/python/sglang/multimodal_gen/test/server/ascend/perf_baselines_npu.json index 0b2d604bc..d9fd7146e 100644 --- a/python/sglang/multimodal_gen/test/server/ascend/perf_baselines_npu.json +++ b/python/sglang/multimodal_gen/test/server/ascend/perf_baselines_npu.json @@ -7,702 +7,770 @@ "scenarios": { "flux_image_t2i_npu": { "stages_ms": { - "InputValidationStage": 0.06, - "TextEncodingStage": 468.6, - "TimestepPreparationStage": 41.44, - "LatentPreparationStage": 0.28, - "DenoisingStage": 19973.94, - "DecodingStage": 8.58 + "InputValidationStage": 0.08, + "TextEncodingStage": 367.9, + "LatentPreparationStage": 0.59, + "TimestepPreparationStage": 44.34, + "DenoisingStage": 20045.89, + "DecodingStage": 10.08 }, "denoise_step_ms": { - "0": 100.15, - "1": 95.5, - "2": 334.93, - "3": 413.4, - "4": 413.43, - "5": 413.33, - "6": 413.6, - "7": 413.2, - "8": 413.4, - "9": 413.15, - "10": 413.24, - "11": 413.37, - "12": 413.63, - "13": 413.39, - "14": 413.49, - "15": 413.69, - "16": 413.61, - "17": 413.38, - "18": 413.32, - "19": 413.3, - "20": 413.45, - "21": 413.49, - "22": 413.31, - "23": 413.14, - "24": 413.46, - "25": 413.31, - "26": 413.61, - "27": 413.58, - "28": 413.52, - "29": 413.37, - "30": 413.46, - "31": 413.83, - "32": 413.23, - "33": 413.49, - "34": 413.45, - "35": 413.41, - "36": 413.4, - "37": 413.46, - "38": 413.38, - "39": 413.43, - "40": 413.52, - "41": 413.56, - "42": 413.63, - "43": 413.46, - "44": 413.47, - "45": 413.38, - "46": 413.57, - "47": 413.37, - "48": 413.19, - "49": 413.38 + "0": 113.84, + "1": 104.01, + "2": 309.67, + "3": 415.06, + "4": 415.26, + "5": 415.09, + "6": 414.92, + "7": 415.08, + "8": 414.82, + "9": 415.01, + "10": 414.83, + "11": 414.94, + "12": 414.9, + "13": 414.97, + "14": 415.18, + "15": 414.89, + "16": 414.97, + "17": 414.98, + "18": 414.94, + "19": 415.08, + "20": 414.9, + "21": 414.76, + "22": 415.03, + "23": 414.87, + "24": 415.09, + "25": 414.76, + "26": 414.98, + "27": 414.84, + "28": 415.18, + "29": 415.09, + "30": 415.06, + "31": 415.05, + "32": 415.22, + "33": 414.94, + "34": 415.13, + "35": 414.84, + "36": 414.94, + "37": 414.88, + "38": 414.98, + "39": 415.26, + "40": 414.9, + "41": 415.06, + "42": 414.97, + "43": 414.98, + "44": 415.07, + "45": 414.87, + "46": 414.93, + "47": 414.95, + "48": 414.86, + "49": 415.12 }, - "expected_e2e_ms": 20670.75, - "expected_avg_denoise_ms": 399.24, - "expected_median_denoise_ms": 413.41 + "expected_e2e_ms": 20864.25, + "expected_avg_denoise_ms": 400.64, + "expected_median_denoise_ms": 414.97 }, "flux_2_image_t2i_2npu": { "stages_ms": { - "InputValidationStage": 0.08, - "TextEncodingStage": 363.09, + "InputValidationStage": 0.07, + "TextEncodingStage": 546.33, "ImageVAEEncodingStage": 0.01, - "LatentPreparationStage": 0.97, - "TimestepPreparationStage": 34.65, - "DenoisingStage": 45390.41, - "DecodingStage": 5.32 + "LatentPreparationStage": 0.82, + "TimestepPreparationStage": 52.32, + "DenoisingStage": 36757.04, + "DecodingStage": 12.81 }, "denoise_step_ms": { - "0": 84.23, - "1": 80.0, - "2": 874.6, - "3": 918.81, - "4": 900.4, - "5": 910.67, - "6": 903.36, - "7": 904.97, - "8": 906.84, - "9": 906.39, - "10": 904.99, - "11": 909.96, - "12": 901.67, - "13": 908.8, - "14": 902.93, - "15": 906.17, - "16": 906.67, - "17": 905.89, - "18": 906.9, - "19": 907.1, - "20": 905.31, - "21": 907.93, - "22": 903.68, - "23": 904.49, - "24": 905.73, - "25": 907.66, - "26": 906.71, - "27": 912.69, - "28": 901.4, - "29": 909.9, - "30": 901.72, - "31": 904.35, - "32": 905.61, - "33": 905.97, - "34": 906.5, - "35": 921.45, - "36": 892.58, - "37": 909.03, - "38": 903.82, - "39": 906.25, - "40": 905.18, - "41": 905.06, - "42": 906.21, - "43": 911.62, - "44": 901.04, - "45": 909.09, - "46": 904.29, - "47": 905.16, - "48": 907.35, - "49": 905.65 + "0": 90.03, + "1": 93.98, + "2": 654.17, + "3": 738.7, + "4": 724.59, + "5": 752.94, + "6": 726.53, + "7": 741.55, + "8": 736.39, + "9": 730.53, + "10": 730.78, + "11": 724.38, + "12": 753.27, + "13": 726.0, + "14": 741.98, + "15": 735.24, + "16": 728.22, + "17": 734.33, + "18": 724.57, + "19": 753.06, + "20": 727.5, + "21": 724.15, + "22": 752.58, + "23": 728.32, + "24": 733.56, + "25": 724.06, + "26": 751.71, + "27": 728.16, + "28": 724.48, + "29": 752.98, + "30": 726.86, + "31": 738.65, + "32": 734.82, + "33": 737.41, + "34": 728.72, + "35": 724.76, + "36": 752.39, + "37": 726.33, + "38": 741.46, + "39": 734.73, + "40": 732.74, + "41": 730.34, + "42": 724.81, + "43": 753.44, + "44": 727.17, + "45": 741.79, + "46": 735.44, + "47": 731.91, + "48": 730.3, + "49": 723.94 }, - "expected_e2e_ms": 46557.7, - "expected_avg_denoise_ms": 872.7, - "expected_median_denoise_ms": 905.81, + "expected_e2e_ms": 38031.14, + "expected_avg_denoise_ms": 707.34, + "expected_median_denoise_ms": 730.65, "estimated_full_test_time_s": 488.4 }, "wan2_1_t2v_1.3b_1_npu": { "stages_ms": { - "InputValidationStage": 0.06, - "TextEncodingStage": 2386.55, - "LatentPreparationStage": 0.2, - "TimestepPreparationStage": 3.03, - "DenoisingStage": 26240.89, - "DecodingStage": 720.28, + "InputValidationStage": 0.07, + "TextEncodingStage": 1621.03, + "LatentPreparationStage": 0.23, + "TimestepPreparationStage": 3.82, + "DenoisingStage": 25194.82, + "DecodingStage": 855.22, "per_frame_generation": null }, "denoise_step_ms": { - "0": 330.0, - "1": 303.37, - "2": 548.94, - "3": 542.02, - "4": 537.7, - "5": 537.26, - "6": 537.78, - "7": 537.27, - "8": 537.72, - "9": 537.65, - "10": 537.49, - "11": 537.41, - "12": 537.31, - "13": 537.43, - "14": 537.41, - "15": 537.51, - "16": 537.54, - "17": 537.56, - "18": 538.09, - "19": 537.09, - "20": 537.72, - "21": 537.56, - "22": 537.85, - "23": 537.54, - "24": 537.67, - "25": 537.66, - "26": 537.25, - "27": 537.71, - "28": 537.77, - "29": 537.16, - "30": 537.6, - "31": 537.58, - "32": 537.6, - "33": 537.51, - "34": 537.78, - "35": 537.43, - "36": 537.52, - "37": 537.59, - "38": 537.46, - "39": 537.77, - "40": 537.41, - "41": 538.11, - "42": 537.41, - "43": 537.18, - "44": 537.64, - "45": 537.32, - "46": 537.52, - "47": 537.52, - "48": 537.29, - "49": 547.88 + "0": 164.38, + "1": 160.33, + "2": 448.37, + "3": 524.41, + "4": 519.61, + "5": 519.48, + "6": 519.56, + "7": 519.57, + "8": 519.44, + "9": 519.56, + "10": 519.75, + "11": 519.39, + "12": 519.55, + "13": 519.58, + "14": 519.59, + "15": 519.45, + "16": 519.5, + "17": 519.26, + "18": 519.44, + "19": 519.58, + "20": 519.39, + "21": 519.6, + "22": 519.57, + "23": 519.34, + "24": 519.52, + "25": 519.6, + "26": 519.53, + "27": 519.39, + "28": 519.64, + "29": 519.59, + "30": 519.56, + "31": 519.49, + "32": 519.74, + "33": 519.44, + "34": 519.38, + "35": 519.67, + "36": 519.41, + "37": 519.45, + "38": 519.56, + "39": 519.63, + "40": 519.45, + "41": 519.64, + "42": 519.35, + "43": 519.46, + "44": 519.74, + "45": 519.57, + "46": 519.63, + "47": 519.54, + "48": 519.36, + "49": 514.61 }, - "expected_e2e_ms": 29360.56, - "expected_avg_denoise_ms": 524.67, - "expected_median_denoise_ms": 537.54, + "expected_e2e_ms": 28194.51, + "expected_avg_denoise_ms": 503.81, + "expected_median_denoise_ms": 519.54, "estimated_full_test_time_s": 157.8 }, "wan2_2_t2v_14b_w8a8_2npu": { "stages_ms": { - "InputValidationStage": 0.09, - "TextEncodingStage": 2789.3, - "LatentPreparationStage": 0.28, - "TimestepPreparationStage": 3.19, - "DenoisingStage": 187650.19, - "DecodingStage": 3491.88, + "InputValidationStage": 0.1, + "TextEncodingStage": 247.21, + "LatentPreparationStage": 0.37, + "TimestepPreparationStage": 5.99, + "DenoisingStage": 185484.28, + "DecodingStage": 4944.85, "per_frame_generation": null }, "denoise_step_ms": { - "0": 1415.84, - "1": 4801.54, - "2": 4781.57, - "3": 4770.19, - "4": 4808.86, - "5": 4737.55, - "6": 4774.02, - "7": 4774.85, - "8": 4773.25, - "9": 4775.33, - "10": 4771.31, - "11": 4773.07, - "12": 4773.18, - "13": 4772.77, - "14": 4773.68, - "15": 4771.91, - "16": 4776.16, - "17": 4773.88, - "18": 4769.87, - "19": 4772.1, - "20": 4774.55, - "21": 4772.54, - "22": 4772.39, - "23": 4776.14, - "24": 4772.84, - "25": 4772.82, - "26": 4789.34, - "27": 4792.58, - "28": 4807.52, - "29": 4740.66, - "30": 4773.46, - "31": 4774.19, - "32": 4772.8, - "33": 4774.84, - "34": 4772.52, - "35": 4773.77, - "36": 4775.43, - "37": 4771.58, - "38": 4772.68, - "39": 4769.1 + "0": 1342.44, + "1": 4743.7, + "2": 4689.71, + "3": 4673.59, + "4": 4676.37, + "5": 4678.01, + "6": 4674.08, + "7": 4673.95, + "8": 4679.56, + "9": 4677.72, + "10": 4672.86, + "11": 4673.39, + "12": 4680.77, + "13": 4678.05, + "14": 4673.85, + "15": 4675.7, + "16": 4675.7, + "17": 4675.73, + "18": 4678.71, + "19": 4675.67, + "20": 4675.67, + "21": 4677.41, + "22": 4679.51, + "23": 4670.53, + "24": 4675.93, + "25": 4675.95, + "26": 5277.29, + "27": 4727.11, + "28": 4675.35, + "29": 4677.12, + "30": 4675.91, + "31": 4671.27, + "32": 4678.1, + "33": 4676.66, + "34": 4671.35, + "35": 4671.56, + "36": 4679.16, + "37": 4712.36, + "38": 4635.81, + "39": 4663.42 }, - "expected_e2e_ms": 193947.19, - "expected_avg_denoise_ms": 4691.07, - "expected_median_denoise_ms": 4773.22, + "expected_e2e_ms": 191226.03, + "expected_avg_denoise_ms": 4610.43, + "expected_median_denoise_ms": 4675.92, "estimated_full_test_time_s": 987.8 }, "qwen_image_t2i_2npu": { "stages_ms": { - "InputValidationStage": 0.07, - "TextEncodingStage": 629.24, - "LatentPreparationStage": 0.69, - "TimestepPreparationStage": 35.29, - "DenoisingStage": 30529.83, - "DecodingStage": 428.21 + "InputValidationStage": 0.09, + "TextEncodingStage": 446.0, + "LatentPreparationStage": 0.46, + "TimestepPreparationStage": 40.32, + "DenoisingStage": 31128.09, + "DecodingStage": 17.48 }, "denoise_step_ms": { - "0": 477.43, - "1": 511.96, - "2": 607.78, - "3": 615.12, - "4": 616.29, - "5": 614.61, - "6": 623.04, - "7": 607.12, - "8": 615.32, - "9": 615.47, - "10": 616.93, - "11": 623.26, - "12": 607.12, - "13": 615.48, - "14": 615.07, - "15": 614.83, - "16": 623.18, - "17": 609.0, - "18": 614.8, - "19": 623.08, - "20": 607.64, - "21": 614.2, - "22": 615.58, - "23": 615.43, - "24": 623.59, - "25": 606.57, - "26": 616.02, - "27": 615.48, - "28": 615.76, - "29": 623.13, - "30": 608.73, - "31": 615.04, - "32": 616.08, - "33": 616.59, - "34": 623.77, - "35": 608.0, - "36": 616.1, - "37": 615.79, - "38": 615.34, - "39": 617.43, - "40": 610.99, - "41": 614.22, - "42": 623.27, - "43": 606.98, - "44": 615.87, - "45": 615.99, - "46": 614.66, - "47": 622.93, - "48": 607.97, - "49": 614.69 + "0": 631.0, + "1": 630.1, + "2": 636.84, + "3": 644.67, + "4": 642.02, + "5": 609.58, + "6": 622.69, + "7": 628.72, + "8": 645.31, + "9": 613.48, + "10": 643.7, + "11": 625.31, + "12": 630.16, + "13": 631.39, + "14": 611.59, + "15": 635.71, + "16": 619.34, + "17": 675.93, + "18": 640.39, + "19": 668.38, + "20": 660.69, + "21": 604.93, + "22": 610.72, + "23": 611.1, + "24": 611.11, + "25": 606.23, + "26": 608.16, + "27": 607.94, + "28": 606.08, + "29": 613.37, + "30": 605.09, + "31": 607.22, + "32": 609.66, + "33": 603.19, + "34": 603.01, + "35": 608.42, + "36": 607.2, + "37": 610.12, + "38": 612.63, + "39": 607.0, + "40": 604.73, + "41": 610.18, + "42": 612.14, + "43": 627.09, + "44": 627.51, + "45": 632.23, + "46": 627.24, + "47": 624.45, + "48": 625.84, + "49": 627.63 }, - "expected_e2e_ms": 34362.34, - "expected_avg_denoise_ms": 610.41, - "expected_median_denoise_ms": 615.39, + "expected_e2e_ms": 32007.87, + "expected_avg_denoise_ms": 622.38, + "expected_median_denoise_ms": 616.41, "estimated_full_test_time_s": 275.2 }, "ernie_image_t2i_1npu": { "stages_ms": { - "InputValidationStage": 0.07, - "PromptEnhancementStage": 8787.2, - "TextEncodingStage": 35.87, - "TimestepPreparationStage": 246.30, - "LatentPreparationStage": 0.2, - "DenoisingStage": 47471.93, - "DecodingStage": 42.24 + "InputValidationStage": 0.08, + "PromptEnhancementStage": 13176.7, + "TextEncodingStage": 92.18, + "TimestepPreparationStage": 196.16, + "LatentPreparationStage": 0.22, + "DenoisingStage": 36635.06, + "DecodingStage": 43.47 }, "denoise_step_ms": { - "0": 93.3, - "1": 229.78, - "2": 982.46, - "3": 984.1, - "4": 982.02, - "5": 982.42, - "6": 982.01, - "7": 982.12, - "8": 981.92, - "9": 982.08, - "10": 982.39, - "11": 982.14, - "12": 982.37, - "13": 982.06, - "14": 982.0, - "15": 982.53, - "16": 981.83, - "17": 982.41, - "18": 981.97, - "19": 981.93, - "20": 982.14, - "21": 981.71, - "22": 982.36, - "23": 981.97, - "24": 981.92, - "25": 982.12, - "26": 981.87, - "27": 982.38, - "28": 981.92, - "29": 981.95, - "30": 982.28, - "31": 981.94, - "32": 981.95, - "33": 982.2, - "34": 981.71, - "35": 982.29, - "36": 981.84, - "37": 982.2, - "38": 981.98, - "39": 981.93, - "40": 982.3, - "41": 982.27, - "42": 982.15, - "43": 981.96, - "44": 981.82, - "45": 982.11, - "46": 981.79, - "47": 982.17, - "48": 982.01, - "49": 981.89 + "0": 92.26, + "1": 89.66, + "2": 588.04, + "3": 765.01, + "4": 762.78, + "5": 763.23, + "6": 763.03, + "7": 763.1, + "8": 762.89, + "9": 762.46, + "10": 763.13, + "11": 762.77, + "12": 763.1, + "13": 762.74, + "14": 763.18, + "15": 763.33, + "16": 762.82, + "17": 762.91, + "18": 763.41, + "19": 762.79, + "20": 762.93, + "21": 762.88, + "22": 762.99, + "23": 762.89, + "24": 762.68, + "25": 762.97, + "26": 762.79, + "27": 763.43, + "28": 763.05, + "29": 762.61, + "30": 763.15, + "31": 762.63, + "32": 763.34, + "33": 762.99, + "34": 763.07, + "35": 763.06, + "36": 762.87, + "37": 762.71, + "38": 762.79, + "39": 762.72, + "40": 763.45, + "41": 762.61, + "42": 763.02, + "43": 762.9, + "44": 762.75, + "45": 762.9, + "46": 762.84, + "47": 763.17, + "48": 763.02, + "49": 762.87 }, - "expected_e2e_ms": 56379.07, - "expected_avg_denoise_ms": 949.3, - "expected_median_denoise_ms": 982.02 + "expected_e2e_ms": 50170.28, + "expected_avg_denoise_ms": 732.61, + "expected_median_denoise_ms": 762.9 }, "glm_image_t2i_1npu": { "stages_ms": { - "GlmImageAR": 69249.94, - "GlmImageBeforeDenoisingStage": 61.78, - "DenoisingStage": 17750.92, - "DecodingStage": 277.36 + "GlmImageAR": 69033.12, + "GlmImageBeforeDenoisingStage": 46.04, + "DenoisingStage": 18392.5, + "DecodingStage": 153.7 }, "denoise_step_ms": { - "0": 272.46, - "1": 269.76, - "2": 537.8, - "3": 617.26, - "4": 617.11, - "5": 617.04, - "6": 617.33, - "7": 617.06, - "8": 616.97, - "9": 617.11, - "10": 616.94, - "11": 617.21, - "12": 617.41, - "13": 617.01, - "14": 617.19, - "15": 617.25, - "16": 616.99, - "17": 616.97, - "18": 617.17, - "19": 616.96, - "20": 617.26, - "21": 617.14, - "22": 617.06, - "23": 617.37, - "24": 617.22, - "25": 617.28, - "26": 616.77, - "27": 617.25, - "28": 617.23, - "29": 616.99 + "0": 479.04, + "1": 619.64, + "2": 615.18, + "3": 614.72, + "4": 615.66, + "5": 617.28, + "6": 618.61, + "7": 616.26, + "8": 617.09, + "9": 615.29, + "10": 617.05, + "11": 622.87, + "12": 617.31, + "13": 617.3, + "14": 611.5, + "15": 617.8, + "16": 618.23, + "17": 593.69, + "18": 611.86, + "19": 627.08, + "20": 607.68, + "21": 613.96, + "22": 614.95, + "23": 616.17, + "24": 617.35, + "25": 645.87, + "26": 637.32, + "27": 656.2, + "28": 585.93, + "29": 608.5 }, - "expected_e2e_ms": 87867.77, - "expected_avg_denoise_ms": 591.42, - "expected_median_denoise_ms": 617.11 + "expected_e2e_ms": 87668.19, + "expected_avg_denoise_ms": 612.91, + "expected_median_denoise_ms": 616.66 }, "z_image_t2i_1npu": { "stages_ms": { - "InputValidationStage": 0.08, - "TextEncodingStage": 246.13, - "LatentPreparationStage": 0.23, - "TimestepPreparationStage": 29.16, - "DenoisingStage": 36649.71, - "DecodingStage": 6.28 + "InputValidationStage": 0.07, + "TextEncodingStage": 245.87, + "LatentPreparationStage": 0.24, + "TimestepPreparationStage": 36.5, + "DenoisingStage": 30118.46, + "DecodingStage": 10.42 }, "denoise_step_ms": { - "0": 732.97, - "1": 732.49, - "2": 732.47, - "3": 732.93, - "4": 732.5, - "5": 732.51, - "6": 732.18, - "7": 733.07, - "8": 733.17, - "9": 734.29, - "10": 734.49, - "11": 735.74, - "12": 733.51, - "13": 732.76, - "14": 733.39, - "15": 732.44, - "16": 731.72, - "17": 731.76, - "18": 732.76, - "19": 731.88, - "20": 732.23, - "21": 731.63, - "22": 731.59, - "23": 732.4, - "24": 732.18, - "25": 732.54, - "26": 732.08, - "27": 732.09, - "28": 732.08, - "29": 732.44, - "30": 732.18, - "31": 731.93, - "32": 732.05, - "33": 732.32, - "34": 733.97, - "35": 736.5, - "36": 735.63, - "37": 733.37, - "38": 734.09, - "39": 732.9, - "40": 732.78, - "41": 732.26, - "42": 731.74, - "43": 732.48, - "44": 732.14, - "45": 732.64, - "46": 733.53, - "47": 733.48, - "48": 732.59, - "49": 732.1 + "0": 602.8, + "1": 602.53, + "2": 602.53, + "3": 602.67, + "4": 602.88, + "5": 602.55, + "6": 602.46, + "7": 602.38, + "8": 602.34, + "9": 603.62, + "10": 602.32, + "11": 602.25, + "12": 602.1, + "13": 602.5, + "14": 603.08, + "15": 602.05, + "16": 603.99, + "17": 602.44, + "18": 601.92, + "19": 601.67, + "20": 601.98, + "21": 601.62, + "22": 601.95, + "23": 601.88, + "24": 602.81, + "25": 602.5, + "26": 602.09, + "27": 601.84, + "28": 601.83, + "29": 602.09, + "30": 602.2, + "31": 601.85, + "32": 602.12, + "33": 602.14, + "34": 602.29, + "35": 602.21, + "36": 602.11, + "37": 601.72, + "38": 602.14, + "39": 602.4, + "40": 601.82, + "41": 601.87, + "42": 601.92, + "43": 601.93, + "44": 601.85, + "45": 601.96, + "46": 601.77, + "47": 601.93, + "48": 602.14, + "49": 601.63 }, - "expected_e2e_ms": 36940.12, - "expected_avg_denoise_ms": 732.82, - "expected_median_denoise_ms": 732.5 + "expected_e2e_ms": 30601.77, + "expected_avg_denoise_ms": 602.23, + "expected_median_denoise_ms": 602.13 }, "flux_2_klein_4b_t2i_1npu": { "stages_ms": { "InputValidationStage": 0.07, - "TextEncodingStage": 232.96, + "TextEncodingStage": 242.43, "ImageVAEEncodingStage": 0.01, - "LatentPreparationStage": 0.57, - "TimestepPreparationStage": 29.26, - "DenoisingStage": 992.79, - "DecodingStage": 10.76 + "LatentPreparationStage": 0.78, + "TimestepPreparationStage": 50.26, + "DenoisingStage": 766.26, + "DecodingStage": 12.24 }, "denoise_step_ms": { - "0": 37.78, - "1": 35.77, - "2": 34.34, - "3": 34.91 + "0": 43.37, + "1": 40.48, + "2": 38.1, + "3": 38.14 }, - "expected_e2e_ms": 1317.11, - "expected_avg_denoise_ms": 35.7, - "expected_median_denoise_ms": 35.34 - }, + "expected_e2e_ms": 1262.89, + "expected_avg_denoise_ms": 40.02, + "expected_median_denoise_ms": 39.31 + }, "joyai_image_edit_ti2i_1npu": { "stages_ms": { - "InputValidationStage": 20.04, - "ImageEncodingStage": 1239.81, - "ImageVAEEncodingStage": 370.97, - "LatentPreparationStage": 0.33, - "TimestepPreparationStage": 34.12, - "DenoisingStage": 105471.23, - "DecodingStage": 2762.97 + "InputValidationStage": 20.83, + "ImageEncodingStage": 836.43, + "ImageVAEEncodingStage": 534.57, + "LatentPreparationStage": 0.41, + "TimestepPreparationStage": 40.44, + "DenoisingStage": 78620.8, + "DecodingStage": 2283.01 }, "denoise_step_ms": { - "0": 229.17, - "1": 2429.69, - "2": 2706.14, - "3": 2707.12, - "4": 2702.84, - "5": 2704.71, - "6": 2704.96, - "7": 2706.45, - "8": 2705.75, - "9": 2703.3, - "10": 2705.13, - "11": 2704.95, - "12": 2704.57, - "13": 2705.43, - "14": 2702.58, - "15": 2705.44, - "16": 2705.74, - "17": 2706.63, - "18": 2705.73, - "19": 2703.79, - "20": 2704.97, - "21": 2707.3, - "22": 2705.92, - "23": 2708.38, - "24": 2703.5, - "25": 2704.63, - "26": 2706.51, - "27": 2704.44, - "28": 2706.82, - "29": 2704.59, - "30": 2704.36, - "31": 2707.21, - "32": 2704.87, - "33": 2708.24, - "34": 2703.86, - "35": 2704.19, - "36": 2706.7, - "37": 2705.72, - "38": 2706.74, - "39": 2704.96 + "0": 252.88, + "1": 1522.3, + "2": 2021.55, + "3": 2021.94, + "4": 2021.89, + "5": 2022.14, + "6": 2020.7, + "7": 2022.28, + "8": 2022.39, + "9": 2022.2, + "10": 2021.88, + "11": 2022.1, + "12": 2022.25, + "13": 2022.43, + "14": 2021.87, + "15": 2022.84, + "16": 2021.7, + "17": 2021.33, + "18": 2022.23, + "19": 2022.06, + "20": 2023.67, + "21": 2021.51, + "22": 2021.99, + "23": 2022.06, + "24": 2021.69, + "25": 2022.59, + "26": 2021.9, + "27": 2022.22, + "28": 2022.49, + "29": 2023.81, + "30": 2022.23, + "31": 2021.59, + "32": 2021.99, + "33": 2023.51, + "34": 2021.61, + "35": 2021.73, + "36": 2021.79, + "37": 2021.86, + "38": 2021.93, + "39": 2023.11 }, - "expected_e2e_ms": 109909.47, - "expected_avg_denoise_ms": 2636.6, - "expected_median_denoise_ms": 2705.05, + "expected_e2e_ms": 82380.9, + "expected_avg_denoise_ms": 1965.41, + "expected_median_denoise_ms": 2021.99, "estimated_full_test_time_s": 239.7 - }, - "mova_360p_ti2va_2npu": { - "stages_ms": { - "InputValidationStage": 19.09, - "TextEncodingStage": 512.61, - "ImageVAEEncodingStage": 588.51, - "MOVALatentPreparationStage": 0.34, - "MOVATimestepPreparationStage": 0.52, - "MOVADenoisingStage": 91033.05, - "MOVADecodingStage": 586.86 - }, - "denoise_step_ms": { - "0": 1816.84, - "1": 1818.1, - "2": 1819.39, - "3": 1817.58, - "4": 1820.35, - "5": 1816.92, - "6": 1811.79, - "7": 1826.0, - "8": 1816.3, - "9": 1817.59, - "10": 1816.24, - "11": 1815.25, - "12": 1815.69, - "13": 1817.23, - "14": 1817.59, - "15": 1820.19, - "16": 1820.93, - "17": 1823.82, - "18": 1848.39, - "19": 1819.37, - "20": 1818.51, - "21": 1815.18, - "22": 1821.84, - "23": 1816.08, - "24": 1817.98, - "25": 1817.91, - "26": 1818.12, - "27": 1816.79, - "28": 1815.43, - "29": 1818.44, - "30": 1817.64, - "31": 1818.73, - "32": 1814.81, - "33": 1818.24, - "34": 1817.88, - "35": 1816.15, - "36": 1818.26, - "37": 1814.76, - "38": 1816.17, - "39": 1817.48, - "40": 1817.83, - "41": 1817.71, - "42": 1811.81, - "43": 1820.49, - "44": 1817.3, - "45": 1814.35, - "46": 1820.92, - "47": 1824.75, - "48": 1822.46, - "49": 1822.08 - }, - "expected_e2e_ms": 92840.23, - "expected_avg_denoise_ms": 1818.63, - "expected_median_denoise_ms": 1817.77, - "estimated_full_test_time_s": 175.6 }, - "ltx_2_ti2va_2npu": { + "mova_360p_ti2va_2npu": { "stages_ms": { - "InputValidationStage": 3.8, - "TextEncodingStage": 1029.19, - "LTX2TextConnectorStage": 654.62, - "LTX2SigmaPreparationStage": 0.19, - "TimestepPreparationStage": 33.52, - "LTX2AVLatentPreparationStage": 0.45, - "LTX2ImageEncodingStage": 93.32, - "LTX2AVDenoisingStage": 29672.06, - "LTX2AVDecodingStage": 1240.88 + "InputValidationStage": 7.63, + "TextEncodingStage": 93.72, + "ImageVAEEncodingStage": 724.12, + "MOVALatentPreparationStage": 0.39, + "MOVATimestepPreparationStage": 0.52, + "MOVADenoisingStage": 72336.37, + "MOVADecodingStage": 489.21 }, "denoise_step_ms": { - "0": 544.81, - "1": 746.36, - "2": 746.01, - "3": 774.46, - "4": 717.7, - "5": 749.83, - "6": 746.58, - "7": 784.74, - "8": 708.78, - "9": 746.72, - "10": 746.38, - "11": 784.14, - "12": 708.74, - "13": 746.85, - "14": 746.57, - "15": 751.14, - "16": 742.59, - "17": 747.19, - "18": 746.26, - "19": 750.58, - "20": 742.55, - "21": 746.34, - "22": 760.44, - "23": 736.36, - "24": 741.97, - "25": 746.3, - "26": 782.28, - "27": 714.3, - "28": 741.96, - "29": 746.25, - "30": 784.82, - "31": 711.85, - "32": 742.0, - "33": 746.24, - "34": 746.13, - "35": 750.84, - "36": 741.81, - "37": 746.2, - "38": 746.08, - "39": 752.12 + "0": 1447.88, + "1": 1392.62, + "2": 1390.13, + "3": 1391.76, + "4": 1386.24, + "5": 1389.36, + "6": 1389.82, + "7": 1388.06, + "8": 1389.99, + "9": 1391.89, + "10": 1387.23, + "11": 1384.75, + "12": 1388.97, + "13": 1387.58, + "14": 1387.33, + "15": 1387.33, + "16": 1385.9, + "17": 1394.24, + "18": 3845.53, + "19": 1387.23, + "20": 1392.08, + "21": 1389.13, + "22": 1392.05, + "23": 1390.72, + "24": 1389.82, + "25": 1389.69, + "26": 1388.77, + "27": 1388.54, + "28": 1388.65, + "29": 1387.31, + "30": 1391.51, + "31": 1387.64, + "32": 1388.7, + "33": 1387.99, + "34": 1388.24, + "35": 1393.49, + "36": 1388.14, + "37": 1393.94, + "38": 1390.18, + "39": 1390.89, + "40": 1388.67, + "41": 1388.2, + "42": 1387.87, + "43": 1392.25, + "44": 1386.83, + "45": 1387.82, + "46": 1389.25, + "47": 1388.19, + "48": 1386.4, + "49": 1399.95 }, - "expected_e2e_ms": 31484.37, - "expected_avg_denoise_ms": 741.58, - "expected_median_denoise_ms": 746.28, + "expected_e2e_ms": 74704.75, + "expected_avg_denoise_ms": 1439.74, + "expected_median_denoise_ms": 1388.87, + "estimated_full_test_time_s": 175.6 + }, + "ltx_2_ti2va_2npu": { + "stages_ms": { + "InputValidationStage": 3.42, + "TextEncodingStage": 537.51, + "LTX2TextConnectorStage": 696.11, + "LTX2SigmaPreparationStage": 0.19, + "TimestepPreparationStage": 99.86, + "LTX2AVLatentPreparationStage": 0.49, + "LTX2ImageEncodingStage": 83.25, + "LTX2AVDenoisingStage": 26633.36, + "LTX2AVDecodingStage": 1295.92 + }, + "denoise_step_ms": { + "0": 586.23, + "1": 637.14, + "2": 677.26, + "3": 733.65, + "4": 646.0, + "5": 670.33, + "6": 689.42, + "7": 663.27, + "8": 653.13, + "9": 635.83, + "10": 640.95, + "11": 663.12, + "12": 665.84, + "13": 661.13, + "14": 692.55, + "15": 669.63, + "16": 645.21, + "17": 692.89, + "18": 674.34, + "19": 708.68, + "20": 666.02, + "21": 666.63, + "22": 669.92, + "23": 664.52, + "24": 659.71, + "25": 660.71, + "26": 681.28, + "27": 635.74, + "28": 659.23, + "29": 612.98, + "30": 672.93, + "31": 634.69, + "32": 699.96, + "33": 654.52, + "34": 669.96, + "35": 652.5, + "36": 675.18, + "37": 672.33, + "38": 679.37, + "39": 653.36 + }, + "expected_e2e_ms": 30063.69, + "expected_avg_denoise_ms": 663.7, + "expected_median_denoise_ms": 665.18, "estimated_full_test_time_s": 255.2 + }, + "minimax_h3_t2va_2npu": { + "stages_ms": { + "InputValidationStage": 0.1, + "MiniMaxH3PartitionAdmissionStage": 0.06, + "MiniMaxH3TextEncodingStage": 134.59, + "MiniMaxH3VisualEncodingStage": 0.06, + "MiniMaxH3AudioEncodingStage": 0.11, + "MiniMaxH3LatentPreparationStage": 41.31, + "MiniMaxH3TimestepPreparationStage": 0.46, + "MiniMaxH3DenoisingStage": 134761.9, + "MiniMaxH3DecodingStage": 9507.3 + }, + "denoise_step_ms": { + "0": 98.14, + "1": 8576.23, + "2": 9407.25, + "3": 9378.5, + "4": 10544.9, + "5": 609.74, + "6": 608.21, + "7": 610.59, + "8": 224.33, + "9": 9771.07, + "10": 610.61, + "11": 614.43, + "12": 611.74, + "13": 100.7, + "14": 9919.46, + "15": 608.76, + "16": 617.32, + "17": 616.59, + "18": 98.9, + "19": 9912.09, + "20": 610.52, + "21": 611.63, + "22": 609.55, + "23": 227.0, + "24": 9785.64, + "25": 614.82, + "26": 612.64, + "27": 615.8, + "28": 97.7, + "29": 9918.15, + "30": 610.76, + "31": 613.17, + "32": 612.41, + "33": 99.4, + "34": 9909.12, + "35": 611.72, + "36": 615.54, + "37": 616.37, + "38": 95.61, + "39": 9905.96, + "40": 609.88, + "41": 614.73, + "42": 613.55, + "43": 94.84, + "44": 9919.13, + "45": 613.18, + "46": 615.76, + "47": 616.8, + "48": 97.59 + }, + "expected_e2e_ms": 155106.21, + "expected_avg_denoise_ms": 2749.56, + "expected_median_denoise_ms": 613.17, + "estimated_full_test_time_s": 320.0 } } } diff --git a/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py b/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py index fa5be7047..d3745d0f5 100644 --- a/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py +++ b/python/sglang/multimodal_gen/test/server/ascend/testcase_configs_npu.py @@ -27,6 +27,7 @@ JOYAI_IMAGE_EDIT_WEIGHTS_PATH = use_modelscope( ) LTX_2_WEIGHTS_PATH = use_modelscope("Lightricks/LTX-2") MOVA_360_WEIGHTS_PATH = use_modelscope("openmoss/MOVA-360p") +MINIMAX_H3_WEIGHTS_PATH = use_modelscope("MiniMax/MiniMax-H3") QWEN_IMAGE_WEIGHTS_PATH = use_modelscope("Qwen/Qwen-Image") WAN2_1_T2V_1_3B_DIFFUSERS_WEIGHTS_PATH = use_modelscope( "Wan-AI/Wan2.1-T2V-1.3B-Diffusers" @@ -145,7 +146,69 @@ TWO_NPU_CASES: list[DiffusionTestCase] = [ prompt=T2V_PROMPT, ), ), - # === Text+Image to Video+Audio (TI2V) + # === Text to Video+Audio (T2VA) + DiffusionTestCase( + "minimax_h3_t2va_2npu", + DiffusionServerArgs( + model_path=MINIMAX_H3_WEIGHTS_PATH, + modality="video", + num_gpus=2, + tp_size=2, + extras=[ + "--model-variant", + "fl2va", + "--dit-cpu-offload", + "false", + "--sp-degree", + "1", + "--attention-backend", + "laser_attn", + "--component-residency", + "text_encoder=layerwise-offload", + ], + env_vars={ + "SGLANG_CACHE_DIT_ENABLED": "true", + "SGLANG_CACHE_DIT_FN": "2", + "SGLANG_CACHE_DIT_BN": "1", + "SGLANG_CACHE_DIT_WARMUP": "4", + "SGLANG_CACHE_DIT_RDT": "0.4", + "SGLANG_CACHE_DIT_MC": "4", + "SGLANG_CACHE_DIT_TAYLORSEER": "true", + "SGLANG_CACHE_DIT_TS_ORDER": "2", + "HCCL_BUFFSIZE": "256", + }, + ), + DiffusionSamplingParams( + prompt=( + "At night, while their owner sleeps in a bedroom, three cats " + "march in loudly playing tiny brass instruments, then abruptly " + "file out." + ), + output_size="1344x768", + seconds=5, + output_format="mp4", + num_outputs_per_prompt=1, + extras={ + "task": "t2va", + "conditions": [], + "target": { + "short_edge": 768, + "aspect_ratio": "16:9", + "duration_seconds": 5.0, + }, + "num_inference_steps": 50, + "flow_shift": 12.0, + "audio_flow_shift": 3.0, + "seed": 1101, + }, + ), + run_perf_check=True, + run_consistency_check=True, + run_component_accuracy_check=False, + run_models_api_check=False, + run_t2v_input_reference_check=False, + ), + # === Text+Image to Video+Audio (TI2VA) DiffusionTestCase( "ltx_2_ti2va_2npu", DiffusionServerArgs( diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index eea8b0b88..d1f524daf 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -45,7 +45,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import ( PerformanceSummary, ScenarioConfig, get_model_task_type_for_server_args, - get_perf_baseline_path, + get_perf_baseline_update_path, ) from sglang.multimodal_gen.test.test_utils import ( SGL_TEST_FILES_CI_DATA_REVISION, @@ -246,7 +246,7 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext: logger.error( f'\n{"=" * 60}\n' f'Add "estimated_full_test_time_s" to scenario "{case.id}":\n\n' - f"File: {get_perf_baseline_path()}\n\n" + f"File: {get_perf_baseline_update_path()}\n\n" f' "{case.id}": {{\n' f" ...\n" f' "estimated_full_test_time_s": {_measured_full_time:.1f}\n' @@ -445,7 +445,7 @@ class DiffusionServerBase: self._dump_baseline_for_testcase(case, summary, missing_scenario) if missing_scenario: pytest.fail( - f"Testcase '{case.id}' not found in {get_perf_baseline_path()}" + f"Testcase '{case.id}' not found in {get_perf_baseline_update_path()}" ) return @@ -459,7 +459,7 @@ class DiffusionServerBase: self._dump_baseline_for_testcase(case, summary, missing_scenario) pytest.fail( f"Testcase '{case.id}' is missing a load/runtime peak VRAM " - f"baseline in {get_perf_baseline_path()}" + f"baseline in {get_perf_baseline_update_path()}" ) try: validator.validate_peak_vram( @@ -521,7 +521,9 @@ class DiffusionServerBase: scenario = BASELINE_CONFIG.scenarios.get(case.id) if scenario is None: - pytest.fail(f"Testcase '{case.id}' not found in {get_perf_baseline_path()}") + pytest.fail( + f"Testcase '{case.id}' not found in {get_perf_baseline_update_path()}" + ) validator = PerformanceValidator( scenario=scenario, @@ -544,7 +546,7 @@ class DiffusionServerBase: if scenario.load_peak_vram_mb is None or scenario.runtime_peak_vram_mb is None: pytest.fail( f"Testcase '{case.id}' is missing a load/runtime peak VRAM " - f"baseline in {get_perf_baseline_path()}; measured " + f"baseline in {get_perf_baseline_update_path()}; measured " f"load={summary.load_peak_vram_mb:.0f}MiB, " f"runtime={summary.runtime_peak_vram_mb:.0f}MiB" ) @@ -689,7 +691,7 @@ class DiffusionServerBase: ) action = "add" if missing_scenario else "update" output = f""" -{action} this baseline in the "scenarios" section of {get_perf_baseline_path()}: +{action} this baseline in the "scenarios" section of {get_perf_baseline_update_path()}: "{case.id}": {json.dumps(baseline, indent=4)} diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index 6c9fcac93..8cb695d3b 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -899,6 +899,14 @@ def get_perf_baseline_path(platform: str | None = None) -> Path: return PERF_BASELINE_DIR / PERF_BASELINE_FILE_BY_PLATFORM[baseline_platform] +def get_perf_baseline_update_path() -> Path: + if current_platform.is_npu(): + return Path(__file__).parent / "ascend" / "perf_baselines_npu.json" + if current_platform.is_musa(): + return Path(__file__).parent / "musa" / "perf_baselines_musa.json" + return get_perf_baseline_path() + + def _make_modelopt_ci_case( case_id: str, *, diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 79e91233b..b43a29fbb 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -45,7 +45,7 @@ SGL_TEST_FILES_CI_DATA_REVISION = "15b30030ef980756788ab40072f9223fe21a5526" # The NPU pin is kept as a separate branch so ascend GT can be bumped independently # when it's regenerated on its own cadence. if current_platform.is_npu(): - SGL_TEST_FILES_CI_DATA_REVISION = "8e3d717e65fb87339c2974382a092a731669f884" + SGL_TEST_FILES_CI_DATA_REVISION = "7df858ead07940ff4d9489230fa9f040dd186789" SGL_TEST_FILES_CONSISTENCY_GT_ROOT = ( "https://raw.githubusercontent.com/" diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_media.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_media.py index 546b9e25b..f1fdabe91 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_media.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_media.py @@ -13,11 +13,21 @@ import torch from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3 import ( + keyframe_encoding, material_io, reference_encoding, ) +def test_keyframe_rng_supports_cpu_and_default_device(): + initial_state = torch.random.get_rng_state() + + for device in (None, torch.device("cpu")): + with keyframe_encoding.minimax_h3_scoped_encode_rng(42, device): + assert torch.initial_seed() == 42 + torch.testing.assert_close(torch.random.get_rng_state(), initial_state) + + def test_ffprobe_falls_back_when_stream_side_data_is_unknown(monkeypatch): material_io._ffprobe_entries = None calls = [] diff --git a/scripts/ci/npu/npu_ci_install_dependency.sh b/scripts/ci/npu/npu_ci_install_dependency.sh index 10b9c1570..58cc158be 100755 --- a/scripts/ci/npu/npu_ci_install_dependency.sh +++ b/scripts/ci/npu/npu_ci_install_dependency.sh @@ -19,6 +19,7 @@ apt update -y && apt install -y \ clang \ locales \ ccache \ + ffmpeg \ libgl1-mesa-glx \ libgl1-mesa-dri \ ca-certificates \