[diffusion] fix: select VSA-H3 for FastH3 when no transformer backend is given (#43690)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-11 23:05:25 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 46a97e5540
commit 97bfa5cc53
5 changed files with 101 additions and 7 deletions
@@ -28,6 +28,9 @@ from sglang.multimodal_gen.runtime.platforms import (
AttentionBackendEnum,
current_platform,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
@dataclass
@@ -355,8 +358,43 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig):
"--model-variant does not apply. FL2VA and Ref2VA tasks were "
"not distilled; use MiniMaxAI/MiniMax-H3 for those."
)
self._default_to_trained_sparse_attention(server_args)
super().validate_server_args(server_args)
def _default_to_trained_sparse_attention(self, server_args) -> None:
# The checkpoint was distilled with VSA-H3; dense attention skips its
# trained compression gate and runs slower.
if self.resolve_transformer_attention_backend(server_args) is not None:
return
if not current_platform.is_cuda():
return
from sglang.multimodal_gen.runtime.platforms.cuda import VSA_H3_CAPABILITIES
capability = current_platform.get_device_capability()
if (
capability is None
or (capability.major, capability.minor) not in VSA_H3_CAPABILITIES
):
return
if (
server_args.enable_torch_compile
or server_args.enable_breakable_cuda_graph
or int(server_args.ring_degree or 1) > 1
):
logger.info(
"FastH3 runs dense attention: VSA-H3 does not serve torch.compile, "
"the breakable CUDA graph or ring attention"
)
return
server_args.set_automatic_component_attention_backend(
"transformer", AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3.name.lower()
)
logger.info(
"Automatically set video_sparse_attn_h3 for the FastH3 transformer, the "
"attention it was distilled with; --component-attention-backends "
"transformer=fa selects dense attention"
)
__all__ = ["FastH3PipelineConfig", "MiniMaxH3PipelineConfig"]
@@ -257,12 +257,14 @@ class _VideoSparseAttentionBackendResolver(_CudaAttentionBackendResolver):
raise ImportError("Video Sparse Attention backend is not installed.") from e
# The vendored Triton tile-64 kernel is written against Hopper and Blackwell
# block-sparse geometry; older architectures fail closed.
VSA_H3_CAPABILITIES = frozenset({(9, 0), (10, 0), (10, 3)})
class _VideoSparseAttentionH3BackendResolver(_CudaAttentionBackendResolver):
backend = AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3
# The vendored Triton tile-64 kernel is written against Hopper and
# Blackwell block-sparse geometry; older architectures fail closed.
supported_capabilities = {(9, 0), (10, 0), (10, 3)}
supported_capabilities = VSA_H3_CAPABILITIES
@classmethod
def resolve(cls, platform) -> str:
@@ -1444,6 +1444,13 @@ class ServerArgs(DisaggServerArgsMixin):
self._requested_component_attention_backend_roles
)
def set_automatic_component_attention_backend(
self, component_name: str, backend: str
) -> None:
"""Select a component's backend on the user's behalf."""
self.component_attention_backends[component_name] = backend
self._automatic_component_attention_backend_keys.add(component_name)
def is_component_attention_backend_automatic(
self, component_name: str | None
) -> bool:
@@ -43,6 +43,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import (
LTX23PipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import (
FastH3PipelineConfig,
MiniMaxH3PipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import (
@@ -93,6 +94,7 @@ from sglang.multimodal_gen.runtime.platforms import (
AttentionBackendEnum,
current_platform,
)
from sglang.multimodal_gen.runtime.platforms.interface import DeviceCapability
from sglang.multimodal_gen.runtime.server_args import (
MAX_SCHEDULER_RPC_TIMEOUT_S,
ServerArgs,
@@ -400,6 +402,46 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
self.assertFalse(args.is_component_attention_backend_automatic("text_encoder"))
def _fasth3_args(self, capability=(9, 0), **kwargs):
with patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.get_device_capability",
return_value=DeviceCapability(*capability),
):
return _from_dict_without_model_resolution(
{"model_path": "FastVideo/FastVideo-FastH3-8-Step-V2", **kwargs},
pipeline_config=FastH3PipelineConfig(),
)
def test_fasth3_selects_vsa_h3_for_the_transformer(self):
for capability in ((9, 0), (10, 0), (10, 3)):
with self.subTest(capability=capability):
args = self._fasth3_args(capability)
self.assertEqual(
args.component_attention_backends,
{"transformer": "video_sparse_attn_h3"},
)
self.assertTrue(
args.is_component_attention_backend_automatic("transformer")
)
def test_fasth3_keeps_a_requested_or_incompatible_transformer_backend(self):
cases = (
({"component_attention_backends": {"transformer": "fa"}}, (9, 0)),
({"attention_backend": "fa"}, (9, 0)),
({"enable_torch_compile": True}, (9, 0)),
({}, (8, 0)),
)
for kwargs, capability in cases:
with self.subTest(kwargs=kwargs, capability=capability):
args = self._fasth3_args(capability, **kwargs)
self.assertEqual(
args.component_attention_backends,
kwargs.get("component_attention_backends", {}),
)
self.assertFalse(
args.is_component_attention_backend_automatic("transformer")
)
def test_invalid_component_attention_backend_raises(self):
with self.assertRaises(ValueError):
self._from_dict_without_model_resolution(