[diffusion] feat: allow per-role attention backend override (#35310)

Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Aleksi Vesanto
2026-10-09 20:57:29 +08:00
committed by GitHub
co-authored by Mick Mick Qian
parent 348689da40
commit 67d063f63b
17 changed files with 756 additions and 72 deletions
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
"""Attention roles.
A "role" distinguishes the type of attention a layer performs (self-attention
vs cross-attention) so a different backend can be selected per role. The value
strings double as the role token accepted in ``--component-attention-backends``
keys, e.g. ``transformer.cross``; ``ServerArgs`` splits those entries out of
``component_attention_backends`` into ``component_attention_backend_roles`` so a
component lookup by name never sees a role-qualified key.
"""
import enum
class AttentionRole(str, enum.Enum):
SELF = "self"
CROSS = "cross"
# Separator between the component name and the role token in a
# ``--component-attention-backends`` key (e.g. ``transformer.cross``).
ROLE_KEY_SEPARATOR = "."
def make_component_role_key(component: str, role: AttentionRole) -> str:
"""Build a role-qualified config key from a component name and role."""
return f"{component}{ROLE_KEY_SEPARATOR}{role.value}"
def split_component_role_key(key: str) -> tuple[str, AttentionRole | None]:
"""Split a ``component`` or ``component.role`` config key.
Returns the component part and the parsed ``AttentionRole`` (or ``None`` when
the key carries no role suffix). Raises ``ValueError`` if a role suffix is
present but is not a valid role token.
"""
component_part, sep, role_part = key.rpartition(ROLE_KEY_SEPARATOR)
if not sep:
return key, None
try:
role = AttentionRole(role_part.strip().lower())
except ValueError:
valid_roles = sorted(r.value for r in AttentionRole)
raise ValueError(
f"Invalid attention role '{role_part}' in component key '{key}'. "
f"Available roles are: {valid_roles}"
) from None
return component_part, role
@@ -12,6 +12,10 @@ from typing import NamedTuple, cast
import torch
from sglang.multimodal_gen.configs.attention_roles import (
AttentionRole,
make_component_role_key,
)
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
AttentionBackend,
AttentionRequirements,
@@ -33,6 +37,11 @@ class ComponentAttnBackendContext(NamedTuple):
backend: AttentionBackendEnum | None
component_name: str | None
selected_backends: dict[str, str | None]
backend_by_role: dict[AttentionRole, AttentionBackendEnum]
# Same records as ``selected_backends``, but attributed to the role of the
# layer that made them. ``None`` collects selections made outside a layer,
# which belong to no single role.
selected_backends_by_role: dict[AttentionRole | None, dict[str, str | None]]
allow_global_backend_fallback: bool = False
require_backend_selection: bool = False
@@ -89,6 +98,17 @@ def claim_deferred_component_attn_backend() -> AttentionBackendEnum | None:
return context.backend
def get_component_forced_attn_backend_for_role(
role: AttentionRole,
) -> AttentionBackendEnum | None:
"""Role-specific component override (e.g. ``transformer.cross``).
Takes precedence over the component-wide backend when both are configured.
"""
context = get_component_attn_backend_context()
return context.backend_by_role.get(role) if context is not None else None
def get_component_attn_backend_name() -> str | None:
context = get_component_attn_backend_context()
return context.component_name if context is not None else None
@@ -99,19 +119,37 @@ def _component_allows_global_backend_fallback() -> bool:
return context is not None and context.allow_global_backend_fallback
def _record_component_attn_backend(backend_name: str, reason: str | None) -> bool:
def _merge_selection(
selections: dict[str, str | None], backend_name: str, reason: str | None
) -> None:
if backend_name not in selections:
selections[backend_name] = reason
elif reason is None:
# unrestricted selection must not be hidden by a later valid fallback
selections[backend_name] = None
def _record_component_attn_backend(
backend_name: str, reason: str | None, role: AttentionRole | None = None
) -> bool:
context = get_component_attn_backend_context()
if context is None or context.component_name is None:
return False
if backend_name not in context.selected_backends:
context.selected_backends[backend_name] = reason
elif reason is None:
# unrestricted selection must not be hidden by a later valid fallback
context.selected_backends[backend_name] = None
_merge_selection(context.selected_backends, backend_name, reason)
_merge_selection(
context.selected_backends_by_role.setdefault(role, {}), backend_name, reason
)
return True
def _format_component_attn_selections(selections: dict[str, str | None]) -> str:
return ", ".join(
f"{backend_name} ({reason})" if reason else backend_name
for backend_name, reason in selections.items()
)
def _log_component_attn_backend_summary(
context: ComponentAttnBackendContext | None,
) -> None:
@@ -122,50 +160,101 @@ def _log_component_attn_backend_summary(
):
return
backend_parts = []
for backend_name, reason in context.selected_backends.items():
if reason:
backend_parts.append(f"{backend_name} ({reason})")
else:
backend_parts.append(backend_name)
by_role = context.selected_backends_by_role
summary = ""
# Break the summary out per role only when a role override is configured, so
# it can be confirmed from the log without re-deriving which layers it
# reached. Components with no override keep the flat single-line format.
if context.backend_by_role:
parts = [
f"{role.value}={_format_component_attn_selections(by_role[role])}"
for role in AttentionRole
if by_role.get(role)
]
if by_role.get(None):
parts.append(_format_component_attn_selections(by_role[None]))
summary = "; ".join(parts)
if not summary:
summary = _format_component_attn_selections(context.selected_backends)
logger.info_once(
f"Attention backends for {context.component_name}: {', '.join(backend_parts)}"
logger.info_once(f"Attention backends for {context.component_name}: {summary}")
def _validate_selection_against_request(
selections: dict[str, str | None], requested_name: str, target: str
) -> None:
"""Check that ``target``'s layers honored ``requested_name``.
A divergence is tolerated only when every diverging selection carries a
reason, i.e. it went through an allowed fallback such as the dense
replacement for a sparse backend in cross-attention.
"""
unexplained = sorted(
backend_name
for backend_name, reason in selections.items()
if backend_name != requested_name and reason is None
)
if unexplained:
detail = (
f"also selected {', '.join(unexplained)} without an allowed fallback"
if requested_name in selections
else f"selected {', '.join(unexplained)} without an allowed fallback"
)
raise ComponentAttentionBackendNotAppliedError(
f"Attention backend '{requested_name}' was requested for {target}, "
f"but it {detail}"
)
def _validate_component_attn_backend_selection(
context: ComponentAttnBackendContext,
) -> None:
if not context.require_backend_selection:
component_name = context.component_name or "component"
# A role override is honored when the layers of that role selected it, or
# fell back for a recorded reason. Having no layers of that role at all
# means the override silently did nothing, which is the same failure the
# component-wide check below reports.
for role, backend in context.backend_by_role.items():
target = f"component '{make_component_role_key(component_name, role)}'"
selections = context.selected_backends_by_role.get(role)
requested_name = backend.name.lower()
if not selections:
raise ComponentAttentionBackendNotAppliedError(
f"Attention backend '{requested_name}' was requested for {target}, "
f"but the component constructed no {role.value}-attention layers"
)
_validate_selection_against_request(selections, requested_name, target)
if not context.require_backend_selection or context.backend is None:
return
requested_backend = context.backend
assert requested_backend is not None
requested_name = requested_backend.name.lower()
component_name = context.component_name or "component"
if requested_name not in context.selected_backends:
detail = (
"did not construct any SGLang-selectable attention layers"
if not context.selected_backends
else f"selected {', '.join(sorted(context.selected_backends))} instead"
)
raise ComponentAttentionBackendNotAppliedError(
f"Attention backend '{requested_name}' was requested for component "
f"'{component_name}', but it {detail}"
)
# Roles carrying their own override are out of the component-wide backend's
# scope, so judge it only on what is left.
residual: dict[str, str | None] = {}
for role, selections in context.selected_backends_by_role.items():
if role is not None and role in context.backend_by_role:
continue
for backend_name, reason in selections.items():
_merge_selection(residual, backend_name, reason)
unexplained = sorted(
backend_name
for backend_name, reason in context.selected_backends.items()
if backend_name != requested_name and reason is None
)
if unexplained:
requested_name = context.backend.name.lower()
target = f"component '{component_name}'"
if not residual:
if context.selected_backends_by_role:
# Every layer this component built was diverted by a role override,
# so the component-wide backend had nothing left to apply to.
return
raise ComponentAttentionBackendNotAppliedError(
f"Attention backend '{requested_name}' was requested for component "
f"'{component_name}', but it also selected "
f"{', '.join(unexplained)} without an allowed fallback"
f"Attention backend '{requested_name}' was requested for {target}, "
"but it did not construct any SGLang-selectable attention layers"
)
if requested_name not in residual:
raise ComponentAttentionBackendNotAppliedError(
f"Attention backend '{requested_name}' was requested for {target}, "
f"but it selected {', '.join(sorted(residual))} instead"
)
_validate_selection_against_request(residual, requested_name, target)
def get_attn_backend(
@@ -184,6 +273,7 @@ def get_attn_backend(
it is admitted when the platform resolves it and the backend satisfies the
layer's semantic requirements.
"""
attention_role = AttentionRole.CROSS if is_cross_attention else AttentionRole.SELF
requirements = attention_requirements or AttentionRequirements()
if supported_attention_backends is None:
be_tuple = tuple()
@@ -199,6 +289,11 @@ def get_attn_backend(
if selected_backend is None:
selected_backend = get_global_forced_attn_backend()
selection_is_explicit = selected_backend is not None
selected_from_role_override = False
if selected_backend is None:
selected_backend = get_component_forced_attn_backend_for_role(attention_role)
selection_is_explicit = selected_backend is not None
selected_from_role_override = selection_is_explicit
if selected_backend is None:
selected_backend = get_component_forced_attn_backend()
selection_is_explicit = selected_backend is not None
@@ -322,9 +417,22 @@ def get_attn_backend(
backend_name = attention_backend_cls.get_enum().name.lower()
reason = fallback_reason
if selected_from_role_override and fallback_reason is not None:
# An explicit role request that could not be used is allowed to fall
# back, but silently doing so would look like the override took effect.
component_name = get_component_attn_backend_name() or "component"
logger.warning_once(
f"Attention backend '{selected_backend.name.lower()}' was requested "
f"for '{make_component_role_key(component_name, attention_role)}' but "
f"could not be used ({fallback_reason}); using {backend_name} instead"
)
if reason is None and selected_from_role_override:
# Diverging from the component-wide backend is the point of a role
# override, so record it as an explained selection.
reason = f"{attention_role.value}-attention override"
if reason is None and backend_name == constraint_backend:
reason = "component constraint"
if not _record_component_attn_backend(backend_name, reason):
if not _record_component_attn_backend(backend_name, reason, attention_role):
reason_suffix = f" ({reason})" if reason else ""
logger.info_once(f"Using {backend_name} attention backend{reason_suffix}")
return attention_backend_cls
@@ -378,11 +486,12 @@ def _is_backend_supported(
def component_attn_backend_context_manager(
attn_backend: AttentionBackendEnum | None,
component_name: str | None = None,
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
allow_global_backend_fallback: bool = False,
require_backend_selection: bool | None = None,
require_component_backend_selection: bool | None = None,
) -> Generator[None, None, None]:
if attn_backend is None and component_name is None:
if attn_backend is None and component_name is None and not backend_by_role:
yield
return
@@ -395,11 +504,27 @@ def component_attn_backend_context_manager(
elif require_component_backend_selection is not None:
raise ValueError("Specify only one component backend selection requirement")
if backend_by_role:
# Announce the configured overrides up front; the summary logged on exit
# reports which layers each one actually reached.
role_parts = ", ".join(
f"{role.value}={backend.name.lower()}"
for role, backend in sorted(
backend_by_role.items(), key=lambda item: item[0].value
)
)
logger.info(
f"Per-role attention backend overrides for "
f"{component_name or 'component'}: {role_parts}"
)
token = component_attn_backend_context.set(
ComponentAttnBackendContext(
attn_backend,
component_name,
{},
dict(backend_by_role or {}),
{},
allow_global_backend_fallback,
require_backend_selection,
)
@@ -22,6 +22,7 @@ from transformers import (
)
from transformers.quantizers import AutoHfQuantizer
from sglang.multimodal_gen.configs.attention_roles import AttentionRole
from sglang.multimodal_gen.configs.models.base import ModelConfig
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.layers.attention.selector import (
@@ -55,7 +56,10 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_
)
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.platforms import (
AttentionBackendEnum,
current_platform,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
get_diffusers_component_config,
@@ -242,11 +246,13 @@ class ComponentLoader(ABC):
attn_backend: Any,
component_attn_name: str | None,
require_backend_selection: bool,
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
):
"""Build the attention-selection context used by this loader."""
return component_attn_backend_context_manager(
attn_backend,
component_name=component_attn_name,
backend_by_role=backend_by_role,
allow_global_backend_fallback=True,
require_backend_selection=require_backend_selection,
)
@@ -283,11 +289,13 @@ class ComponentLoader(ABC):
attn_backend: Any,
component_attn_name: str | None,
require_backend_selection: bool,
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
) -> AutoModel:
with self.component_attention_backend_context(
attn_backend,
component_attn_name,
require_backend_selection,
backend_by_role,
):
load_kwargs = self.customized_load_kwargs_for_component(
server_args, component_name
@@ -305,11 +313,13 @@ class ComponentLoader(ABC):
attn_backend: Any,
component_attn_name: str | None,
require_backend_selection: bool,
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
) -> AutoModel:
with self.component_attention_backend_context(
attn_backend,
component_attn_name,
require_backend_selection,
backend_by_role,
):
component = self.load_native(
component_model_path,
@@ -328,6 +338,9 @@ class ComponentLoader(ABC):
*,
component_attn_backend: Any = None,
component_attn_name: str | None = None,
component_backend_by_role: (
dict[AttentionRole, AttentionBackendEnum] | None
) = None,
allow_native_fallback: bool = True,
) -> tuple[AutoModel, float]:
"""
@@ -369,13 +382,26 @@ class ComponentLoader(ABC):
component_attn_backend.name.lower(),
matched_backend_key,
)
if (
component_backend_by_role is None
and get_component_attn_backend_context() is None
):
# A caller that resolved only the component-wide backend must not
# silently drop the role-qualified overrides that go with it.
component_backend_by_role = server_args.resolve_component_backend_by_role(
component_name, component_attn_name
)
requested_backend = (
server_args.requested_component_attention_backend(component_attn_name)
if component_attn_name is not None
else None
)
require_backend_selection = requested_backend is not None
if require_backend_selection and (
# A role override is as explicit as a component-wide one, so it carries
# the same requirement that the component actually apply it.
require_backend_selection = requested_backend is not None or bool(
component_backend_by_role
)
if requested_backend is not None and (
component_attn_backend is None
or component_attn_backend.name.lower() != requested_backend
):
@@ -391,6 +417,7 @@ class ComponentLoader(ABC):
component_attn_backend,
component_attn_name,
require_backend_selection,
component_backend_by_role,
)
source = "sgl-diffusion"
except (
@@ -455,6 +482,7 @@ class ComponentLoader(ABC):
component_attn_backend,
component_attn_name,
require_backend_selection,
component_backend_by_role,
)
source = "native"
logger.warning(
@@ -1003,12 +1031,14 @@ class GenericComponentLoader(ComponentLoader):
attn_backend: Any,
component_attn_name: str | None,
require_backend_selection: bool,
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
):
# An unknown out-of-tree component may itself be the primary transformer.
# Require it to opt into fallback through a registered component loader.
return component_attn_backend_context_manager(
attn_backend,
component_name=component_attn_name,
backend_by_role=backend_by_role,
allow_global_backend_fallback=False,
require_backend_selection=require_backend_selection,
)
@@ -1028,6 +1058,9 @@ class PipelineComponentLoader:
component_architecture: str | None = None,
component_attn_backend: Any = None,
component_attn_name: str | None = None,
component_backend_by_role: (
dict[AttentionRole, AttentionBackendEnum] | None
) = None,
component_type: str | None = None,
loader_cls: type[ComponentLoader] | None = None,
):
@@ -1059,6 +1092,7 @@ class PipelineComponentLoader:
transformers_or_diffusers,
component_attn_backend=component_attn_backend,
component_attn_name=component_attn_name,
component_backend_by_role=component_backend_by_role,
allow_native_fallback=loader_cls is None,
)
except Exception:
@@ -7,6 +7,7 @@ from typing import Any
import torch
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.configs.attention_roles import AttentionRole
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.layers.attention.selector import (
component_attn_backend_context_manager,
@@ -171,10 +172,12 @@ class TransformerLoader(OnlineQuantizationComponentLoader):
attn_backend,
component_attn_name: str | None,
require_backend_selection: bool,
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
):
return component_attn_backend_context_manager(
attn_backend,
component_name=component_attn_name,
backend_by_role=backend_by_role,
allow_global_backend_fallback=False,
require_backend_selection=require_backend_selection,
)
@@ -632,6 +632,9 @@ class ComposedPipelineBase(ABC):
module_name, load_module_name
)
)
backend_by_role = server_args.resolve_component_backend_by_role(
module_name, load_module_name
)
if attn_backend is not None:
logger.info(
"Using %s backend for component: %s",
@@ -648,6 +651,7 @@ class ComposedPipelineBase(ABC):
component_architecture=architecture,
component_attn_backend=attn_backend,
component_attn_name=matched_backend_key or module_name,
component_backend_by_role=backend_by_role,
)
self.memory_usages[module_name] = memory_usage
@@ -20,6 +20,10 @@ import addict
import yaml
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.configs.attention_roles import (
AttentionRole,
split_component_role_key,
)
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import (
LTX2PipelineConfig,
@@ -311,6 +315,15 @@ class ServerArgs(DisaggServerArgsMixin):
_requested_component_attention_backends: dict[str, str] | None = field(
default=None, repr=False, compare=False
)
# Role-qualified overrides (``<component>.<role>``) live in their own map so
# ``component_attention_backends`` stays a plain component -> backend dict for
# every consumer that looks a component up by name.
component_attention_backend_roles: dict[str, dict[str, str]] = field(
default_factory=dict
)
_requested_component_attention_backend_roles: dict[str, dict[str, str]] | None = (
field(default=None, repr=False, compare=False)
)
cache_dit_config: str | dict[str, Any] | None = (
None # cache-dit config for diffusers
)
@@ -1108,20 +1121,29 @@ class ServerArgs(DisaggServerArgsMixin):
def _adjust_attention_backend(self):
if self.attention_backend in ["fa3", "fa4"]:
self.attention_backend = "fa"
self.component_attention_backends = (
self._normalize_component_attention_backends(
self.component_attention_backends
self.component_attention_backends, self.component_attention_backend_roles = (
self._split_component_attention_backends(
self.component_attention_backends,
self.component_attention_backend_roles,
)
)
# Snapshot what the user asked for before the pipeline-specific
# adjustments below add any automatic entries of our own.
if self._requested_component_attention_backends is None:
self._requested_component_attention_backends = dict(
self.component_attention_backends
)
self._requested_component_attention_backend_roles = {
component: dict(entries)
for component, entries in self.component_attention_backend_roles.items()
}
else:
self._requested_component_attention_backends = (
self._normalize_component_attention_backends(
self._requested_component_attention_backends
)
(
self._requested_component_attention_backends,
self._requested_component_attention_backend_roles,
) = self._split_component_attention_backends(
self._requested_component_attention_backends,
self._requested_component_attention_backend_roles,
)
# attention_backend_config
@@ -1148,6 +1170,20 @@ class ServerArgs(DisaggServerArgsMixin):
text_backend,
)
self.component_attention_backends["text_encoder"] = "torch_sdpa"
# A role-qualified override would otherwise outrank the backend we
# just forced, so drop it for the same reason.
text_encoder_roles = self.component_attention_backend_roles.pop(
"text_encoder", None
)
if text_encoder_roles:
logger.warning(
"Ignoring per-role attention backend overrides (%s) for component "
"text_encoder to preserve LTX2 official attention semantics",
", ".join(
f"{role}={backend}"
for role, backend in sorted(text_encoder_roles.items())
),
)
from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import (
MiniMaxH3PipelineConfig,
)
@@ -1157,6 +1193,7 @@ class ServerArgs(DisaggServerArgsMixin):
and isinstance(self.pipeline_config, MiniMaxH3PipelineConfig)
and self.attention_backend == "laser_attn"
and "text_encoder" not in self.component_attention_backends
and "text_encoder" not in self.component_attention_backend_roles
):
# Laser Attention is used only by the MiniMax-H3 transformer.
# SDPA is faster than Ascend FA for its Qwen3-VL text encoder.
@@ -1335,19 +1372,50 @@ class ServerArgs(DisaggServerArgsMixin):
return result
@classmethod
def _normalize_component_attention_backends(
cls, value: dict[str, str] | str | None
) -> dict[str, str]:
def _split_component_attention_backends(
cls,
value: dict[str, str] | str | None,
roles: dict[str, dict[str, str]] | None = None,
) -> tuple[dict[str, str], dict[str, dict[str, str]]]:
"""Normalize raw entries into component-wide and role-qualified maps.
A key may be a bare ``component`` or a role-qualified ``component.role``.
The optional trailing ``.<role>`` selects the backend for one attention
role (self vs cross) and is validated against ``AttentionRole``; those
entries are routed into the returned role map instead of the flat one so
a component lookup by name never sees them. ``roles`` seeds the role map
for callers that already hold a normalized one.
"""
raw = cls._parse_component_attention_backend_map(value)
normalized: dict[str, str] = {}
normalized_roles: dict[str, dict[str, str]] = {
component: dict(entries) for component, entries in (roles or {}).items()
}
for component, backend in raw.items():
if not isinstance(component, str):
raise ValueError("Component attention backend key must be a string")
component_name = component.strip().replace("-", "_")
component_part, role = split_component_role_key(component.strip())
component_name = component_part.strip().replace("-", "_")
if not component_name:
raise ValueError("Component attention backend key must not be empty")
normalized[component_name] = cls._normalize_attention_backend_name(backend)
return normalized
backend_name = cls._normalize_attention_backend_name(backend)
if role is None:
normalized[component_name] = backend_name
else:
normalized_roles.setdefault(component_name, {})[role.value] = (
backend_name
)
return normalized, normalized_roles
@staticmethod
def _component_fallback_keys(component_name: str) -> list[str]:
key = component_name.replace("-", "_")
fallback_keys = [key]
if key.endswith("_2"):
# Secondary two-stage components inherit the base component backend
# unless explicitly overridden.
fallback_keys.append(key[:-2])
return fallback_keys
def resolve_component_attention_backend(
self, *component_names: str | None
@@ -1355,13 +1423,7 @@ class ServerArgs(DisaggServerArgsMixin):
for component_name in component_names:
if component_name is None:
continue
key = component_name.replace("-", "_")
fallback_keys = [key]
if key.endswith("_2"):
# Secondary two-stage components inherit the base component
# backend unless explicitly overridden.
fallback_keys.append(key[:-2])
for backend_key in fallback_keys:
for backend_key in self._component_fallback_keys(component_name):
backend = self.component_attention_backends.get(backend_key)
if backend is not None:
return AttentionBackendEnum[backend.upper()], backend_key
@@ -1372,7 +1434,9 @@ class ServerArgs(DisaggServerArgsMixin):
return self._requested_component_attention_backends.get(component_name)
def has_requested_component_attention_backends(self) -> bool:
return bool(self._requested_component_attention_backends)
return bool(self._requested_component_attention_backends) or bool(
self._requested_component_attention_backend_roles
)
def is_component_attention_backend_automatic(
self, component_name: str | None
@@ -1382,6 +1446,33 @@ class ServerArgs(DisaggServerArgsMixin):
and component_name in self._automatic_component_attention_backend_keys
)
def resolve_component_backend_by_role(
self, *component_names: str | None
) -> dict[AttentionRole, AttentionBackendEnum]:
"""Resolve the per-role backend overrides for a component.
For each role, tries ``<component>.<role>`` across the candidate names
(with the same ``_2`` two-stage fallback as the component-wide lookup).
Returns only the roles that have an explicit override configured.
"""
backend_by_role: dict[AttentionRole, AttentionBackendEnum] = {}
for role in AttentionRole:
for component_name in component_names:
if component_name is None:
continue
matched = False
for base_key in self._component_fallback_keys(component_name):
backend = self.component_attention_backend_roles.get(
base_key, {}
).get(role.value)
if backend is not None:
backend_by_role[role] = AttentionBackendEnum[backend.upper()]
matched = True
break
if matched:
break
return backend_by_role
def _adjust_warmup(self):
if self.warmup_mode is not None and self.warmup_mode not in WARMUP_MODES:
raise ValueError(
@@ -4,6 +4,7 @@ from unittest.mock import patch
import torch
from sglang.multimodal_gen.configs.attention_roles import AttentionRole
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
AttentionRequirements,
)
@@ -127,6 +128,8 @@ class TestAttentionBackendFallback(unittest.TestCase):
component_backend: AttentionBackendEnum | None = None,
allow_global_backend_fallback: bool = False,
server_args: object | None = None,
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
also_resolve_is_cross: tuple[bool, ...] = (),
):
if server_args is None:
server_args = _ServerArgs(backend.name.lower(), explicit=explicit)
@@ -148,11 +151,12 @@ class TestAttentionBackendFallback(unittest.TestCase):
component_attn_backend_context_manager(
component_backend,
component_name="text_encoder",
backend_by_role=backend_by_role,
allow_global_backend_fallback=allow_global_backend_fallback,
require_backend_selection=component_backend is not None,
),
):
return get_attn_backend(
resolved = get_attn_backend(
128,
torch.bfloat16,
supported_attention_backends=supported,
@@ -160,6 +164,18 @@ class TestAttentionBackendFallback(unittest.TestCase):
default_attention_backend=default_attention_backend,
is_cross_attention=is_cross_attention,
)
# Component-scoped validation runs on context exit, so a test that
# configures a role must build a layer of that role somewhere.
for extra_is_cross in also_resolve_is_cross:
get_attn_backend(
128,
torch.bfloat16,
supported_attention_backends=supported,
attention_requirements=attention_requirements,
default_attention_backend=default_attention_backend,
is_cross_attention=extra_is_cross,
)
return resolved
def test_implicit_platform_preference_falls_back(self):
backend = self._resolve(
@@ -331,6 +347,120 @@ class TestAttentionBackendFallback(unittest.TestCase):
self.assertIs(backend, _FakeSparseBackend)
def test_role_backend_selects_for_cross(self):
# Component-wide default is SDPA (implicit), but the cross role forces FA.
backend = self._resolve(
AttentionBackendEnum.TORCH_SDPA,
explicit=False,
is_cross_attention=True,
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
)
self.assertIs(backend, _FakeFABackend)
self.assertEqual(_FakePlatform.selected_backend, AttentionBackendEnum.FA)
def test_role_backend_not_applied_to_self(self):
# The cross role override must not leak into self-attention, which keeps
# the explicit SDPA selection.
backend = self._resolve(
AttentionBackendEnum.TORCH_SDPA,
explicit=True,
is_cross_attention=False,
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
also_resolve_is_cross=(True,),
)
self.assertIs(backend, _FakeSDPABackend)
def test_role_backend_requires_a_matching_layer(self):
# A role override on a component that builds no layers of that role is
# a silent no-op, which is the failure the component-wide check reports.
with self.assertRaisesRegex(
ComponentAttentionBackendNotAppliedError,
"constructed no cross-attention layers",
):
self._resolve(
AttentionBackendEnum.TORCH_SDPA,
explicit=True,
is_cross_attention=False,
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
)
def test_role_override_does_not_mask_component_wide_failure(self):
# A layer that pins its own backend still fails the component-wide
# requirement even when a role override happens to name that backend.
with (
patch(f"{_SELECTOR}.get_global_forced_attn_backend", return_value=None),
patch(
f"{_SELECTOR}.get_component_forced_attn_backend",
return_value=AttentionBackendEnum.AITER,
),
patch(
f"{_SELECTOR}.get_global_server_args",
return_value=_ServerArgs("aiter", explicit=False),
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform",
_FakePlatform,
),
patch(
f"{_SELECTOR}.resolve_name",
side_effect=_FAKE_BACKENDS.__getitem__,
),
self.assertRaisesRegex(
ComponentAttentionBackendNotAppliedError, "selected fa instead"
),
component_attn_backend_context_manager(
AttentionBackendEnum.AITER,
component_name="text_encoder",
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
require_backend_selection=True,
),
):
# Cross-attention honors the role override.
get_attn_backend(
128,
torch.bfloat16,
supported_attention_backends={AttentionBackendEnum.FA},
is_cross_attention=True,
)
# Self-attention pins FA for correctness, ignoring aiter entirely.
get_attn_backend(
128,
torch.bfloat16,
supported_attention_backends={AttentionBackendEnum.FA},
selected_attention_backend=AttentionBackendEnum.FA,
is_cross_attention=False,
)
def test_role_backend_overrides_component_wide(self):
# Role-qualified selection takes precedence over the component-wide backend.
backend = self._resolve(
AttentionBackendEnum.TORCH_SDPA,
explicit=False,
is_cross_attention=True,
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
component_backend=AttentionBackendEnum.TORCH_SDPA,
)
self.assertIs(backend, _FakeFABackend)
def test_role_sparse_backend_downgrades_for_cross(self):
# A sparse backend chosen via the cross role still downgrades to dense.
backend = self._resolve(
AttentionBackendEnum.LASER_ATTN,
explicit=False,
is_cross_attention=True,
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.LASER_ATTN},
)
self.assertIs(backend, _FakeFABackend)
class TestComponentAttentionBackendScope(unittest.TestCase):
def _load_with_policy(self, allow_global_backend_fallback: bool):
@@ -347,10 +477,12 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
attn_backend,
component_attn_name: str | None,
require_backend_selection: bool,
backend_by_role=None,
):
return component_attn_backend_context_manager(
attn_backend,
component_name=component_attn_name,
backend_by_role=backend_by_role,
allow_global_backend_fallback=allow_global_backend_fallback,
require_backend_selection=require_backend_selection,
)
@@ -361,6 +493,10 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
component_weights_paths = {}
pipeline_config = SimpleNamespace(native_only_components=())
@staticmethod
def resolve_component_backend_by_role(*_component_names):
return {}
@staticmethod
def requested_component_attention_backend(_component_name):
return None
@@ -453,6 +589,10 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
component_weights_paths = {}
pipeline_config = SimpleNamespace(native_only_components=())
@staticmethod
def resolve_component_backend_by_role(*_component_names):
return {}
@staticmethod
def requested_component_attention_backend(_component_name):
return "fa"
@@ -520,6 +660,10 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
component_weights_paths = {}
pipeline_config = SimpleNamespace(native_only_components=())
@staticmethod
def resolve_component_backend_by_role(*_component_names):
return {}
@staticmethod
def requested_component_attention_backend(_component_name):
return "fa"
@@ -572,6 +716,10 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
component_weights_paths = {}
pipeline_config = SimpleNamespace(native_only_components=())
@staticmethod
def resolve_component_backend_by_role(*_component_names):
return {}
@staticmethod
def requested_component_attention_backend(_component_name):
return None
@@ -181,6 +181,7 @@ class TestComponentLoaderIdentity(unittest.TestCase):
component_paths={},
component_direct_gpu_weight_loading=set(),
resolve_component_attention_backend=lambda *_names: (None, None),
resolve_component_backend_by_role=lambda *_names: {},
comfyui_mode=False,
)
@@ -203,6 +204,7 @@ class TestComponentLoaderIdentity(unittest.TestCase):
component_architecture="LTX2DurationHeadModel",
component_attn_backend=None,
component_attn_name="auxiliary_head",
component_backend_by_role={},
)
def test_skipped_alias_keeps_exact_override_and_structural_config(self):
@@ -68,6 +68,7 @@ def _loader_server_args(component_weights_paths, *, fsdp_requested=False):
component_weights_paths=component_weights_paths,
pipeline_config=SimpleNamespace(native_only_components=()),
resolve_component_attention_backend=lambda _name: (None, None),
resolve_component_backend_by_role=lambda *_names: {},
requested_component_attention_backend=lambda _name: None,
should_direct_gpu_weight_load_component=lambda _name: False,
should_start_component_on_cpu=lambda _name: True,
@@ -47,6 +47,7 @@ class TestImageEncoderQuantizationAdmission(unittest.TestCase):
component_precisions={},
encoder_parallel="replicate",
resolve_component_attention_backend=lambda _name: (None, None),
resolve_component_backend_by_role=lambda *_names: {},
requested_component_attention_backend=lambda _name: None,
should_direct_gpu_weight_load_component=lambda _name: False,
should_use_fsdp_for_component=lambda _name: False,
@@ -268,6 +269,7 @@ class TestImageEncoderNativeLoading(unittest.TestCase):
native_only_components=(),
),
resolve_component_attention_backend=lambda _name: (None, None),
resolve_component_backend_by_role=lambda *_names: {},
requested_component_attention_backend=lambda _name: None,
explicit_residency_mode=lambda _name: None,
require_component_resident=mock.Mock(),
@@ -200,6 +200,9 @@ class FakeServerArgs:
def resolve_component_attention_backend(self, *names):
return None, None
def resolve_component_backend_by_role(self, *names):
return {}
def _write_component(directory, config, tensors):
directory.mkdir(parents=True, exist_ok=True)
@@ -284,10 +284,12 @@ def test_real_model_index_is_accepted_by_the_module_loading_path(
loader_cls=None,
component_attn_backend=None,
component_attn_name=None,
component_backend_by_role=None,
):
assert component_type == component_name
assert loader_cls is None and component_attn_backend is None
assert component_attn_name == component_name
assert component_backend_by_role == {}
requested[component_name] = (transformers_or_diffusers, component_architecture)
return SimpleNamespace(name=component_name), 0.0
@@ -302,6 +304,7 @@ def test_real_model_index_is_accepted_by_the_module_loading_path(
component_direct_gpu_weight_loading={},
pipeline_config=Kandinsky6SRPipelineConfig(),
resolve_component_attention_backend=lambda *names: (None, None),
resolve_component_backend_by_role=lambda *names: {},
)
# the state ``ComposedPipelineBase.__init__`` sets up before it calls load_modules
pipeline = object.__new__(Kandinsky6SRPipeline)
@@ -1,5 +1,6 @@
import json
import os
import subprocess
import sys
import tempfile
import unittest
@@ -9,6 +10,7 @@ from unittest.mock import Mock, patch
import zmq
from sglang.cli.utils import get_is_diffusion_model
from sglang.multimodal_gen.configs.attention_roles import AttentionRole
from sglang.multimodal_gen.configs.models.fsdp import (
is_module_list_entry,
is_module_list_entry_in,
@@ -87,7 +89,10 @@ from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
from sglang.multimodal_gen.runtime.pipelines.minimax_h3_pipeline import (
MiniMaxH3Pipeline,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.platforms import (
AttentionBackendEnum,
current_platform,
)
from sglang.multimodal_gen.runtime.server_args import (
MAX_SCHEDULER_RPC_TIMEOUT_S,
ServerArgs,
@@ -148,6 +153,23 @@ def _from_dict_without_model_resolution(
return ServerArgs.from_dict(kwargs)
class TestServerArgsColdImport(unittest.TestCase):
def test_public_entry_points_import_in_fresh_processes(self):
modules = (
"sglang.multimodal_gen.runtime.server_args",
"sglang.multimodal_gen.test.runner.diffusion_suite_runner",
)
for module in modules:
with self.subTest(module=module):
result = subprocess.run(
[sys.executable, "-c", f"import {module}"],
capture_output=True,
text=True,
timeout=60,
)
self.assertEqual(result.returncode, 0, result.stderr)
class TestPlatformLifecycleHooks(unittest.TestCase):
def test_server_args_applies_platform_defaults(self):
with patch.object(
@@ -421,6 +443,66 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
server_args.component_attention_backends, {"text_encoder": "torch_sdpa"}
)
def test_role_qualified_component_keys_are_normalized(self):
args = self._from_dict_without_model_resolution(
{
"model_path": "/data/my-model",
"component_attention_backends": (
"transformer.self=sage_attn,transformer.cross=fa3"
),
}
)
# Role-qualified entries must not leak into the flat component map, or
# every consumer that looks a component up by name would see them.
self.assertEqual(args.component_attention_backends, {})
self.assertEqual(
args.component_attention_backend_roles,
{"transformer": {"self": "sage_attn", "cross": "fa"}},
)
def test_role_and_component_wide_keys_are_split(self):
args = self._from_dict_without_model_resolution(
{
"model_path": "/data/my-model",
"component_attention_backends": (
"text_encoder=torch_sdpa,transformer=fa,transformer.cross=fa3"
),
}
)
self.assertEqual(
args.component_attention_backends,
{"text_encoder": "torch_sdpa", "transformer": "fa"},
)
self.assertEqual(
args.component_attention_backend_roles,
{"transformer": {"cross": "fa"}},
)
def test_role_qualified_component_normalizes_hyphenated_component(self):
args = self._from_dict_without_model_resolution(
{
"model_path": "/data/my-model",
"component_attention_backends": "text-encoder.cross=torch_sdpa",
}
)
self.assertEqual(args.component_attention_backends, {})
self.assertEqual(
args.component_attention_backend_roles,
{"text_encoder": {"cross": "torch_sdpa"}},
)
def test_invalid_component_attention_role_raises(self):
with self.assertRaises(ValueError):
self._from_dict_without_model_resolution(
{
"model_path": "/data/my-model",
"component_attention_backends": {"transformer.bogus": "fa"},
}
)
def test_dynamic_component_precision_cli_args(self):
parser = FlexibleArgumentParser()
ServerArgs.add_cli_args(parser)
@@ -443,6 +525,99 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
self.assertEqual(server_args.component_precisions, {"text_encoder_2": "fp32"})
def test_dynamic_role_qualified_cli_args(self):
parser = FlexibleArgumentParser()
ServerArgs.add_cli_args(parser)
argv = [
"--model-path",
"/fake",
"--component-attention-backends.transformer.cross=fa",
]
with (
patch.object(sys, "argv", ["sglang"] + argv),
patch.object(
PipelineConfig, "from_kwargs", return_value=QwenImagePipelineConfig()
),
_mock_cuda_platform(),
):
args, unknown_args = parser.parse_known_args(argv)
server_args = ServerArgs.from_cli_args(args, unknown_args)
self.assertEqual(server_args.component_attention_backends, {})
self.assertEqual(
server_args.component_attention_backend_roles,
{"transformer": {"cross": "fa"}},
)
def test_resolve_component_backend_by_role_returns_overrides(self):
args = self._from_dict_without_model_resolution(
{
"model_path": "/data/my-model",
"component_attention_backends": {
"transformer.self": "sage_attn",
"transformer.cross": "fa",
},
}
)
backend_by_role = args.resolve_component_backend_by_role("transformer")
self.assertEqual(
backend_by_role,
{
AttentionRole.SELF: AttentionBackendEnum.SAGE_ATTN,
AttentionRole.CROSS: AttentionBackendEnum.FA,
},
)
def test_resolve_component_backend_by_role_only_returns_configured_roles(self):
args = self._from_dict_without_model_resolution(
{
"model_path": "/data/my-model",
"component_attention_backends": {"transformer.cross": "fa"},
}
)
backend_by_role = args.resolve_component_backend_by_role("transformer")
self.assertEqual(
backend_by_role, {AttentionRole.CROSS: AttentionBackendEnum.FA}
)
def test_resolve_component_backend_by_role_two_stage_fallback(self):
args = self._from_dict_without_model_resolution(
{
"model_path": "/data/my-model",
"component_attention_backends": {"transformer.cross": "fa"},
}
)
# transformer_2 has no explicit role override, so it inherits the base
# transformer.cross entry via the two-stage fallback.
backend_by_role = args.resolve_component_backend_by_role("transformer_2")
self.assertEqual(
backend_by_role, {AttentionRole.CROSS: AttentionBackendEnum.FA}
)
def test_resolve_component_backend_by_role_prefers_two_stage_override(self):
args = self._from_dict_without_model_resolution(
{
"model_path": "/data/my-model",
"component_attention_backends": {
"transformer.cross": "fa",
"transformer_2.cross": "torch_sdpa",
},
}
)
backend_by_role = args.resolve_component_backend_by_role("transformer_2")
self.assertEqual(
backend_by_role, {AttentionRole.CROSS: AttentionBackendEnum.TORCH_SDPA}
)
def test_layerwise_offload_components_imply_layerwise(self):
args = self._from_dict_without_model_resolution(
{
@@ -32,6 +32,7 @@ class TestTransformerLoaderFallbackAdmission(unittest.TestCase):
"dp_size": 1,
"use_fsdp_inference": False,
"resolve_component_attention_backend": mock.Mock(return_value=(None, None)),
"resolve_component_backend_by_role": mock.Mock(return_value={}),
"requested_component_attention_backend": mock.Mock(return_value=None),
"should_direct_gpu_weight_load_component": mock.Mock(return_value=False),
"should_use_fsdp_for_component": mock.Mock(return_value=fsdp_requested),
@@ -66,6 +66,9 @@ class _FakeServerArgs:
def resolve_component_attention_backend(self, _component_name):
return None, None
def resolve_component_backend_by_role(self, *_component_names):
return {}
def requested_component_attention_backend(self, _component_name):
return None