diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 80516c27c6..0ebba3cde0 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -904,9 +904,14 @@ refiner falls back to dense FA inside the backend, and the text encoder and VAEs keep their own backends. Ulysses sequence parallelism is supported. See [Attention Backends](/docs/sglang-diffusion/attention_backends) for `VSA_sparsity`, `vsa_mode`, `vsa_dense_first_n_steps`, and -`vsa_dense_layers`. Every dense backend that runs on base H3 (`fa`, -`torch_sdpa`, ...) also runs on the FastH3 weights without VSA flags, and -`sglang generate` takes the same flags as `sglang serve`. +`vsa_dense_layers`. On CUDA, FastH3 selects VSA-H3 for the transformer when no +transformer backend is given, unless torch.compile, the breakable CUDA graph or +ring attention is on (VSA-H3 serves none of them). Every dense backend that +runs on base H3 also runs on the FastH3 weights, without the compression +branch: select it with `--component-attention-backends transformer=fa` (or +`torch_sdpa`, ...). On 4× H200 dense FA is about 25% slower end to end than +VSA-H3 (12.6 s against 10.0 s). `sglang generate` takes the same flags as +`sglang serve`. Measured latencies for the 4× GB300 recipe are in [FastH3 on GB300](#fasth3-on-gb300). diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py index 1aeba59e8c..26024254a1 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py @@ -28,6 +28,9 @@ from sglang.multimodal_gen.runtime.platforms import ( AttentionBackendEnum, current_platform, ) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) @dataclass @@ -355,8 +358,43 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig): "--model-variant does not apply. FL2VA and Ref2VA tasks were " "not distilled; use MiniMaxAI/MiniMax-H3 for those." ) + self._default_to_trained_sparse_attention(server_args) super().validate_server_args(server_args) + def _default_to_trained_sparse_attention(self, server_args) -> None: + # The checkpoint was distilled with VSA-H3; dense attention skips its + # trained compression gate and runs slower. + if self.resolve_transformer_attention_backend(server_args) is not None: + return + if not current_platform.is_cuda(): + return + from sglang.multimodal_gen.runtime.platforms.cuda import VSA_H3_CAPABILITIES + + capability = current_platform.get_device_capability() + if ( + capability is None + or (capability.major, capability.minor) not in VSA_H3_CAPABILITIES + ): + return + if ( + server_args.enable_torch_compile + or server_args.enable_breakable_cuda_graph + or int(server_args.ring_degree or 1) > 1 + ): + logger.info( + "FastH3 runs dense attention: VSA-H3 does not serve torch.compile, " + "the breakable CUDA graph or ring attention" + ) + return + server_args.set_automatic_component_attention_backend( + "transformer", AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3.name.lower() + ) + logger.info( + "Automatically set video_sparse_attn_h3 for the FastH3 transformer, the " + "attention it was distilled with; --component-attention-backends " + "transformer=fa selects dense attention" + ) + __all__ = ["FastH3PipelineConfig", "MiniMaxH3PipelineConfig"] diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py index 19cd076e75..a05fa7aa17 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -257,12 +257,14 @@ class _VideoSparseAttentionBackendResolver(_CudaAttentionBackendResolver): raise ImportError("Video Sparse Attention backend is not installed.") from e +# The vendored Triton tile-64 kernel is written against Hopper and Blackwell +# block-sparse geometry; older architectures fail closed. +VSA_H3_CAPABILITIES = frozenset({(9, 0), (10, 0), (10, 3)}) + + class _VideoSparseAttentionH3BackendResolver(_CudaAttentionBackendResolver): backend = AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3 - - # The vendored Triton tile-64 kernel is written against Hopper and - # Blackwell block-sparse geometry; older architectures fail closed. - supported_capabilities = {(9, 0), (10, 0), (10, 3)} + supported_capabilities = VSA_H3_CAPABILITIES @classmethod def resolve(cls, platform) -> str: diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 8661576ab7..3b8d9e739f 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -1444,6 +1444,13 @@ class ServerArgs(DisaggServerArgsMixin): self._requested_component_attention_backend_roles ) + def set_automatic_component_attention_backend( + self, component_name: str, backend: str + ) -> None: + """Select a component's backend on the user's behalf.""" + self.component_attention_backends[component_name] = backend + self._automatic_component_attention_backend_keys.add(component_name) + def is_component_attention_backend_automatic( self, component_name: str | None ) -> bool: diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index f0b8da7996..1c8f0a1b53 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -43,6 +43,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import ( LTX23PipelineConfig, ) from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import ( + FastH3PipelineConfig, MiniMaxH3PipelineConfig, ) from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import ( @@ -93,6 +94,7 @@ from sglang.multimodal_gen.runtime.platforms import ( AttentionBackendEnum, current_platform, ) +from sglang.multimodal_gen.runtime.platforms.interface import DeviceCapability from sglang.multimodal_gen.runtime.server_args import ( MAX_SCHEDULER_RPC_TIMEOUT_S, ServerArgs, @@ -400,6 +402,46 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase): self.assertFalse(args.is_component_attention_backend_automatic("text_encoder")) + def _fasth3_args(self, capability=(9, 0), **kwargs): + with patch( + "sglang.multimodal_gen.runtime.platforms.current_platform.get_device_capability", + return_value=DeviceCapability(*capability), + ): + return _from_dict_without_model_resolution( + {"model_path": "FastVideo/FastVideo-FastH3-8-Step-V2", **kwargs}, + pipeline_config=FastH3PipelineConfig(), + ) + + def test_fasth3_selects_vsa_h3_for_the_transformer(self): + for capability in ((9, 0), (10, 0), (10, 3)): + with self.subTest(capability=capability): + args = self._fasth3_args(capability) + self.assertEqual( + args.component_attention_backends, + {"transformer": "video_sparse_attn_h3"}, + ) + self.assertTrue( + args.is_component_attention_backend_automatic("transformer") + ) + + def test_fasth3_keeps_a_requested_or_incompatible_transformer_backend(self): + cases = ( + ({"component_attention_backends": {"transformer": "fa"}}, (9, 0)), + ({"attention_backend": "fa"}, (9, 0)), + ({"enable_torch_compile": True}, (9, 0)), + ({}, (8, 0)), + ) + for kwargs, capability in cases: + with self.subTest(kwargs=kwargs, capability=capability): + args = self._fasth3_args(capability, **kwargs) + self.assertEqual( + args.component_attention_backends, + kwargs.get("component_attention_backends", {}), + ) + self.assertFalse( + args.is_component_attention_backend_automatic("transformer") + ) + def test_invalid_component_attention_backend_raises(self): with self.assertRaises(ValueError): self._from_dict_without_model_resolution(