[diffusion] fix: select VSA-H3 for FastH3 when no transformer backend is given (#43690)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
46a97e5540
commit
97bfa5cc53
@@ -28,6 +28,9 @@ from sglang.multimodal_gen.runtime.platforms import (
|
||||
AttentionBackendEnum,
|
||||
current_platform,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -355,8 +358,43 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig):
|
||||
"--model-variant does not apply. FL2VA and Ref2VA tasks were "
|
||||
"not distilled; use MiniMaxAI/MiniMax-H3 for those."
|
||||
)
|
||||
self._default_to_trained_sparse_attention(server_args)
|
||||
super().validate_server_args(server_args)
|
||||
|
||||
def _default_to_trained_sparse_attention(self, server_args) -> None:
|
||||
# The checkpoint was distilled with VSA-H3; dense attention skips its
|
||||
# trained compression gate and runs slower.
|
||||
if self.resolve_transformer_attention_backend(server_args) is not None:
|
||||
return
|
||||
if not current_platform.is_cuda():
|
||||
return
|
||||
from sglang.multimodal_gen.runtime.platforms.cuda import VSA_H3_CAPABILITIES
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if (
|
||||
capability is None
|
||||
or (capability.major, capability.minor) not in VSA_H3_CAPABILITIES
|
||||
):
|
||||
return
|
||||
if (
|
||||
server_args.enable_torch_compile
|
||||
or server_args.enable_breakable_cuda_graph
|
||||
or int(server_args.ring_degree or 1) > 1
|
||||
):
|
||||
logger.info(
|
||||
"FastH3 runs dense attention: VSA-H3 does not serve torch.compile, "
|
||||
"the breakable CUDA graph or ring attention"
|
||||
)
|
||||
return
|
||||
server_args.set_automatic_component_attention_backend(
|
||||
"transformer", AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3.name.lower()
|
||||
)
|
||||
logger.info(
|
||||
"Automatically set video_sparse_attn_h3 for the FastH3 transformer, the "
|
||||
"attention it was distilled with; --component-attention-backends "
|
||||
"transformer=fa selects dense attention"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["FastH3PipelineConfig", "MiniMaxH3PipelineConfig"]
|
||||
|
||||
|
||||
@@ -257,12 +257,14 @@ class _VideoSparseAttentionBackendResolver(_CudaAttentionBackendResolver):
|
||||
raise ImportError("Video Sparse Attention backend is not installed.") from e
|
||||
|
||||
|
||||
# The vendored Triton tile-64 kernel is written against Hopper and Blackwell
|
||||
# block-sparse geometry; older architectures fail closed.
|
||||
VSA_H3_CAPABILITIES = frozenset({(9, 0), (10, 0), (10, 3)})
|
||||
|
||||
|
||||
class _VideoSparseAttentionH3BackendResolver(_CudaAttentionBackendResolver):
|
||||
backend = AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3
|
||||
|
||||
# The vendored Triton tile-64 kernel is written against Hopper and
|
||||
# Blackwell block-sparse geometry; older architectures fail closed.
|
||||
supported_capabilities = {(9, 0), (10, 0), (10, 3)}
|
||||
supported_capabilities = VSA_H3_CAPABILITIES
|
||||
|
||||
@classmethod
|
||||
def resolve(cls, platform) -> str:
|
||||
|
||||
@@ -1444,6 +1444,13 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
self._requested_component_attention_backend_roles
|
||||
)
|
||||
|
||||
def set_automatic_component_attention_backend(
|
||||
self, component_name: str, backend: str
|
||||
) -> None:
|
||||
"""Select a component's backend on the user's behalf."""
|
||||
self.component_attention_backends[component_name] = backend
|
||||
self._automatic_component_attention_backend_keys.add(component_name)
|
||||
|
||||
def is_component_attention_backend_automatic(
|
||||
self, component_name: str | None
|
||||
) -> bool:
|
||||
|
||||
@@ -43,6 +43,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import (
|
||||
LTX23PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import (
|
||||
FastH3PipelineConfig,
|
||||
MiniMaxH3PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import (
|
||||
@@ -93,6 +94,7 @@ from sglang.multimodal_gen.runtime.platforms import (
|
||||
AttentionBackendEnum,
|
||||
current_platform,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms.interface import DeviceCapability
|
||||
from sglang.multimodal_gen.runtime.server_args import (
|
||||
MAX_SCHEDULER_RPC_TIMEOUT_S,
|
||||
ServerArgs,
|
||||
@@ -400,6 +402,46 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
|
||||
|
||||
self.assertFalse(args.is_component_attention_backend_automatic("text_encoder"))
|
||||
|
||||
def _fasth3_args(self, capability=(9, 0), **kwargs):
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.get_device_capability",
|
||||
return_value=DeviceCapability(*capability),
|
||||
):
|
||||
return _from_dict_without_model_resolution(
|
||||
{"model_path": "FastVideo/FastVideo-FastH3-8-Step-V2", **kwargs},
|
||||
pipeline_config=FastH3PipelineConfig(),
|
||||
)
|
||||
|
||||
def test_fasth3_selects_vsa_h3_for_the_transformer(self):
|
||||
for capability in ((9, 0), (10, 0), (10, 3)):
|
||||
with self.subTest(capability=capability):
|
||||
args = self._fasth3_args(capability)
|
||||
self.assertEqual(
|
||||
args.component_attention_backends,
|
||||
{"transformer": "video_sparse_attn_h3"},
|
||||
)
|
||||
self.assertTrue(
|
||||
args.is_component_attention_backend_automatic("transformer")
|
||||
)
|
||||
|
||||
def test_fasth3_keeps_a_requested_or_incompatible_transformer_backend(self):
|
||||
cases = (
|
||||
({"component_attention_backends": {"transformer": "fa"}}, (9, 0)),
|
||||
({"attention_backend": "fa"}, (9, 0)),
|
||||
({"enable_torch_compile": True}, (9, 0)),
|
||||
({}, (8, 0)),
|
||||
)
|
||||
for kwargs, capability in cases:
|
||||
with self.subTest(kwargs=kwargs, capability=capability):
|
||||
args = self._fasth3_args(capability, **kwargs)
|
||||
self.assertEqual(
|
||||
args.component_attention_backends,
|
||||
kwargs.get("component_attention_backends", {}),
|
||||
)
|
||||
self.assertFalse(
|
||||
args.is_component_attention_backend_automatic("transformer")
|
||||
)
|
||||
|
||||
def test_invalid_component_attention_backend_raises(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self._from_dict_without_model_resolution(
|
||||
|
||||
Reference in New Issue
Block a user