[diffusion] feat: allow per-role attention backend override (#35310)
Co-authored-by: Mick <mickjagger19@icloud.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
co-authored by
Mick
Mick Qian
parent
348689da40
commit
67d063f63b
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Attention roles.
|
||||
|
||||
A "role" distinguishes the type of attention a layer performs (self-attention
|
||||
vs cross-attention) so a different backend can be selected per role. The value
|
||||
strings double as the role token accepted in ``--component-attention-backends``
|
||||
keys, e.g. ``transformer.cross``; ``ServerArgs`` splits those entries out of
|
||||
``component_attention_backends`` into ``component_attention_backend_roles`` so a
|
||||
component lookup by name never sees a role-qualified key.
|
||||
"""
|
||||
|
||||
import enum
|
||||
|
||||
|
||||
class AttentionRole(str, enum.Enum):
|
||||
SELF = "self"
|
||||
CROSS = "cross"
|
||||
|
||||
|
||||
# Separator between the component name and the role token in a
|
||||
# ``--component-attention-backends`` key (e.g. ``transformer.cross``).
|
||||
ROLE_KEY_SEPARATOR = "."
|
||||
|
||||
|
||||
def make_component_role_key(component: str, role: AttentionRole) -> str:
|
||||
"""Build a role-qualified config key from a component name and role."""
|
||||
return f"{component}{ROLE_KEY_SEPARATOR}{role.value}"
|
||||
|
||||
|
||||
def split_component_role_key(key: str) -> tuple[str, AttentionRole | None]:
|
||||
"""Split a ``component`` or ``component.role`` config key.
|
||||
|
||||
Returns the component part and the parsed ``AttentionRole`` (or ``None`` when
|
||||
the key carries no role suffix). Raises ``ValueError`` if a role suffix is
|
||||
present but is not a valid role token.
|
||||
"""
|
||||
component_part, sep, role_part = key.rpartition(ROLE_KEY_SEPARATOR)
|
||||
if not sep:
|
||||
return key, None
|
||||
try:
|
||||
role = AttentionRole(role_part.strip().lower())
|
||||
except ValueError:
|
||||
valid_roles = sorted(r.value for r in AttentionRole)
|
||||
raise ValueError(
|
||||
f"Invalid attention role '{role_part}' in component key '{key}'. "
|
||||
f"Available roles are: {valid_roles}"
|
||||
) from None
|
||||
return component_part, role
|
||||
@@ -12,6 +12,10 @@ from typing import NamedTuple, cast
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.attention_roles import (
|
||||
AttentionRole,
|
||||
make_component_role_key,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
|
||||
AttentionBackend,
|
||||
AttentionRequirements,
|
||||
@@ -33,6 +37,11 @@ class ComponentAttnBackendContext(NamedTuple):
|
||||
backend: AttentionBackendEnum | None
|
||||
component_name: str | None
|
||||
selected_backends: dict[str, str | None]
|
||||
backend_by_role: dict[AttentionRole, AttentionBackendEnum]
|
||||
# Same records as ``selected_backends``, but attributed to the role of the
|
||||
# layer that made them. ``None`` collects selections made outside a layer,
|
||||
# which belong to no single role.
|
||||
selected_backends_by_role: dict[AttentionRole | None, dict[str, str | None]]
|
||||
allow_global_backend_fallback: bool = False
|
||||
require_backend_selection: bool = False
|
||||
|
||||
@@ -89,6 +98,17 @@ def claim_deferred_component_attn_backend() -> AttentionBackendEnum | None:
|
||||
return context.backend
|
||||
|
||||
|
||||
def get_component_forced_attn_backend_for_role(
|
||||
role: AttentionRole,
|
||||
) -> AttentionBackendEnum | None:
|
||||
"""Role-specific component override (e.g. ``transformer.cross``).
|
||||
|
||||
Takes precedence over the component-wide backend when both are configured.
|
||||
"""
|
||||
context = get_component_attn_backend_context()
|
||||
return context.backend_by_role.get(role) if context is not None else None
|
||||
|
||||
|
||||
def get_component_attn_backend_name() -> str | None:
|
||||
context = get_component_attn_backend_context()
|
||||
return context.component_name if context is not None else None
|
||||
@@ -99,19 +119,37 @@ def _component_allows_global_backend_fallback() -> bool:
|
||||
return context is not None and context.allow_global_backend_fallback
|
||||
|
||||
|
||||
def _record_component_attn_backend(backend_name: str, reason: str | None) -> bool:
|
||||
def _merge_selection(
|
||||
selections: dict[str, str | None], backend_name: str, reason: str | None
|
||||
) -> None:
|
||||
if backend_name not in selections:
|
||||
selections[backend_name] = reason
|
||||
elif reason is None:
|
||||
# unrestricted selection must not be hidden by a later valid fallback
|
||||
selections[backend_name] = None
|
||||
|
||||
|
||||
def _record_component_attn_backend(
|
||||
backend_name: str, reason: str | None, role: AttentionRole | None = None
|
||||
) -> bool:
|
||||
context = get_component_attn_backend_context()
|
||||
if context is None or context.component_name is None:
|
||||
return False
|
||||
|
||||
if backend_name not in context.selected_backends:
|
||||
context.selected_backends[backend_name] = reason
|
||||
elif reason is None:
|
||||
# unrestricted selection must not be hidden by a later valid fallback
|
||||
context.selected_backends[backend_name] = None
|
||||
_merge_selection(context.selected_backends, backend_name, reason)
|
||||
_merge_selection(
|
||||
context.selected_backends_by_role.setdefault(role, {}), backend_name, reason
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _format_component_attn_selections(selections: dict[str, str | None]) -> str:
|
||||
return ", ".join(
|
||||
f"{backend_name} ({reason})" if reason else backend_name
|
||||
for backend_name, reason in selections.items()
|
||||
)
|
||||
|
||||
|
||||
def _log_component_attn_backend_summary(
|
||||
context: ComponentAttnBackendContext | None,
|
||||
) -> None:
|
||||
@@ -122,50 +160,101 @@ def _log_component_attn_backend_summary(
|
||||
):
|
||||
return
|
||||
|
||||
backend_parts = []
|
||||
for backend_name, reason in context.selected_backends.items():
|
||||
if reason:
|
||||
backend_parts.append(f"{backend_name} ({reason})")
|
||||
else:
|
||||
backend_parts.append(backend_name)
|
||||
by_role = context.selected_backends_by_role
|
||||
summary = ""
|
||||
# Break the summary out per role only when a role override is configured, so
|
||||
# it can be confirmed from the log without re-deriving which layers it
|
||||
# reached. Components with no override keep the flat single-line format.
|
||||
if context.backend_by_role:
|
||||
parts = [
|
||||
f"{role.value}={_format_component_attn_selections(by_role[role])}"
|
||||
for role in AttentionRole
|
||||
if by_role.get(role)
|
||||
]
|
||||
if by_role.get(None):
|
||||
parts.append(_format_component_attn_selections(by_role[None]))
|
||||
summary = "; ".join(parts)
|
||||
if not summary:
|
||||
summary = _format_component_attn_selections(context.selected_backends)
|
||||
|
||||
logger.info_once(
|
||||
f"Attention backends for {context.component_name}: {', '.join(backend_parts)}"
|
||||
logger.info_once(f"Attention backends for {context.component_name}: {summary}")
|
||||
|
||||
|
||||
def _validate_selection_against_request(
|
||||
selections: dict[str, str | None], requested_name: str, target: str
|
||||
) -> None:
|
||||
"""Check that ``target``'s layers honored ``requested_name``.
|
||||
|
||||
A divergence is tolerated only when every diverging selection carries a
|
||||
reason, i.e. it went through an allowed fallback such as the dense
|
||||
replacement for a sparse backend in cross-attention.
|
||||
"""
|
||||
unexplained = sorted(
|
||||
backend_name
|
||||
for backend_name, reason in selections.items()
|
||||
if backend_name != requested_name and reason is None
|
||||
)
|
||||
if unexplained:
|
||||
detail = (
|
||||
f"also selected {', '.join(unexplained)} without an allowed fallback"
|
||||
if requested_name in selections
|
||||
else f"selected {', '.join(unexplained)} without an allowed fallback"
|
||||
)
|
||||
raise ComponentAttentionBackendNotAppliedError(
|
||||
f"Attention backend '{requested_name}' was requested for {target}, "
|
||||
f"but it {detail}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_component_attn_backend_selection(
|
||||
context: ComponentAttnBackendContext,
|
||||
) -> None:
|
||||
if not context.require_backend_selection:
|
||||
component_name = context.component_name or "component"
|
||||
|
||||
# A role override is honored when the layers of that role selected it, or
|
||||
# fell back for a recorded reason. Having no layers of that role at all
|
||||
# means the override silently did nothing, which is the same failure the
|
||||
# component-wide check below reports.
|
||||
for role, backend in context.backend_by_role.items():
|
||||
target = f"component '{make_component_role_key(component_name, role)}'"
|
||||
selections = context.selected_backends_by_role.get(role)
|
||||
requested_name = backend.name.lower()
|
||||
if not selections:
|
||||
raise ComponentAttentionBackendNotAppliedError(
|
||||
f"Attention backend '{requested_name}' was requested for {target}, "
|
||||
f"but the component constructed no {role.value}-attention layers"
|
||||
)
|
||||
_validate_selection_against_request(selections, requested_name, target)
|
||||
|
||||
if not context.require_backend_selection or context.backend is None:
|
||||
return
|
||||
|
||||
requested_backend = context.backend
|
||||
assert requested_backend is not None
|
||||
requested_name = requested_backend.name.lower()
|
||||
component_name = context.component_name or "component"
|
||||
if requested_name not in context.selected_backends:
|
||||
detail = (
|
||||
"did not construct any SGLang-selectable attention layers"
|
||||
if not context.selected_backends
|
||||
else f"selected {', '.join(sorted(context.selected_backends))} instead"
|
||||
)
|
||||
raise ComponentAttentionBackendNotAppliedError(
|
||||
f"Attention backend '{requested_name}' was requested for component "
|
||||
f"'{component_name}', but it {detail}"
|
||||
)
|
||||
# Roles carrying their own override are out of the component-wide backend's
|
||||
# scope, so judge it only on what is left.
|
||||
residual: dict[str, str | None] = {}
|
||||
for role, selections in context.selected_backends_by_role.items():
|
||||
if role is not None and role in context.backend_by_role:
|
||||
continue
|
||||
for backend_name, reason in selections.items():
|
||||
_merge_selection(residual, backend_name, reason)
|
||||
|
||||
unexplained = sorted(
|
||||
backend_name
|
||||
for backend_name, reason in context.selected_backends.items()
|
||||
if backend_name != requested_name and reason is None
|
||||
)
|
||||
if unexplained:
|
||||
requested_name = context.backend.name.lower()
|
||||
target = f"component '{component_name}'"
|
||||
if not residual:
|
||||
if context.selected_backends_by_role:
|
||||
# Every layer this component built was diverted by a role override,
|
||||
# so the component-wide backend had nothing left to apply to.
|
||||
return
|
||||
raise ComponentAttentionBackendNotAppliedError(
|
||||
f"Attention backend '{requested_name}' was requested for component "
|
||||
f"'{component_name}', but it also selected "
|
||||
f"{', '.join(unexplained)} without an allowed fallback"
|
||||
f"Attention backend '{requested_name}' was requested for {target}, "
|
||||
"but it did not construct any SGLang-selectable attention layers"
|
||||
)
|
||||
if requested_name not in residual:
|
||||
raise ComponentAttentionBackendNotAppliedError(
|
||||
f"Attention backend '{requested_name}' was requested for {target}, "
|
||||
f"but it selected {', '.join(sorted(residual))} instead"
|
||||
)
|
||||
_validate_selection_against_request(residual, requested_name, target)
|
||||
|
||||
|
||||
def get_attn_backend(
|
||||
@@ -184,6 +273,7 @@ def get_attn_backend(
|
||||
it is admitted when the platform resolves it and the backend satisfies the
|
||||
layer's semantic requirements.
|
||||
"""
|
||||
attention_role = AttentionRole.CROSS if is_cross_attention else AttentionRole.SELF
|
||||
requirements = attention_requirements or AttentionRequirements()
|
||||
if supported_attention_backends is None:
|
||||
be_tuple = tuple()
|
||||
@@ -199,6 +289,11 @@ def get_attn_backend(
|
||||
if selected_backend is None:
|
||||
selected_backend = get_global_forced_attn_backend()
|
||||
selection_is_explicit = selected_backend is not None
|
||||
selected_from_role_override = False
|
||||
if selected_backend is None:
|
||||
selected_backend = get_component_forced_attn_backend_for_role(attention_role)
|
||||
selection_is_explicit = selected_backend is not None
|
||||
selected_from_role_override = selection_is_explicit
|
||||
if selected_backend is None:
|
||||
selected_backend = get_component_forced_attn_backend()
|
||||
selection_is_explicit = selected_backend is not None
|
||||
@@ -322,9 +417,22 @@ def get_attn_backend(
|
||||
|
||||
backend_name = attention_backend_cls.get_enum().name.lower()
|
||||
reason = fallback_reason
|
||||
if selected_from_role_override and fallback_reason is not None:
|
||||
# An explicit role request that could not be used is allowed to fall
|
||||
# back, but silently doing so would look like the override took effect.
|
||||
component_name = get_component_attn_backend_name() or "component"
|
||||
logger.warning_once(
|
||||
f"Attention backend '{selected_backend.name.lower()}' was requested "
|
||||
f"for '{make_component_role_key(component_name, attention_role)}' but "
|
||||
f"could not be used ({fallback_reason}); using {backend_name} instead"
|
||||
)
|
||||
if reason is None and selected_from_role_override:
|
||||
# Diverging from the component-wide backend is the point of a role
|
||||
# override, so record it as an explained selection.
|
||||
reason = f"{attention_role.value}-attention override"
|
||||
if reason is None and backend_name == constraint_backend:
|
||||
reason = "component constraint"
|
||||
if not _record_component_attn_backend(backend_name, reason):
|
||||
if not _record_component_attn_backend(backend_name, reason, attention_role):
|
||||
reason_suffix = f" ({reason})" if reason else ""
|
||||
logger.info_once(f"Using {backend_name} attention backend{reason_suffix}")
|
||||
return attention_backend_cls
|
||||
@@ -378,11 +486,12 @@ def _is_backend_supported(
|
||||
def component_attn_backend_context_manager(
|
||||
attn_backend: AttentionBackendEnum | None,
|
||||
component_name: str | None = None,
|
||||
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
|
||||
allow_global_backend_fallback: bool = False,
|
||||
require_backend_selection: bool | None = None,
|
||||
require_component_backend_selection: bool | None = None,
|
||||
) -> Generator[None, None, None]:
|
||||
if attn_backend is None and component_name is None:
|
||||
if attn_backend is None and component_name is None and not backend_by_role:
|
||||
yield
|
||||
return
|
||||
|
||||
@@ -395,11 +504,27 @@ def component_attn_backend_context_manager(
|
||||
elif require_component_backend_selection is not None:
|
||||
raise ValueError("Specify only one component backend selection requirement")
|
||||
|
||||
if backend_by_role:
|
||||
# Announce the configured overrides up front; the summary logged on exit
|
||||
# reports which layers each one actually reached.
|
||||
role_parts = ", ".join(
|
||||
f"{role.value}={backend.name.lower()}"
|
||||
for role, backend in sorted(
|
||||
backend_by_role.items(), key=lambda item: item[0].value
|
||||
)
|
||||
)
|
||||
logger.info(
|
||||
f"Per-role attention backend overrides for "
|
||||
f"{component_name or 'component'}: {role_parts}"
|
||||
)
|
||||
|
||||
token = component_attn_backend_context.set(
|
||||
ComponentAttnBackendContext(
|
||||
attn_backend,
|
||||
component_name,
|
||||
{},
|
||||
dict(backend_by_role or {}),
|
||||
{},
|
||||
allow_global_backend_fallback,
|
||||
require_backend_selection,
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ from transformers import (
|
||||
)
|
||||
from transformers.quantizers import AutoHfQuantizer
|
||||
|
||||
from sglang.multimodal_gen.configs.attention_roles import AttentionRole
|
||||
from sglang.multimodal_gen.configs.models.base import ModelConfig
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
@@ -55,7 +56,10 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
|
||||
from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.platforms import (
|
||||
AttentionBackendEnum,
|
||||
current_platform,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
|
||||
get_diffusers_component_config,
|
||||
@@ -242,11 +246,13 @@ class ComponentLoader(ABC):
|
||||
attn_backend: Any,
|
||||
component_attn_name: str | None,
|
||||
require_backend_selection: bool,
|
||||
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
|
||||
):
|
||||
"""Build the attention-selection context used by this loader."""
|
||||
return component_attn_backend_context_manager(
|
||||
attn_backend,
|
||||
component_name=component_attn_name,
|
||||
backend_by_role=backend_by_role,
|
||||
allow_global_backend_fallback=True,
|
||||
require_backend_selection=require_backend_selection,
|
||||
)
|
||||
@@ -283,11 +289,13 @@ class ComponentLoader(ABC):
|
||||
attn_backend: Any,
|
||||
component_attn_name: str | None,
|
||||
require_backend_selection: bool,
|
||||
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
|
||||
) -> AutoModel:
|
||||
with self.component_attention_backend_context(
|
||||
attn_backend,
|
||||
component_attn_name,
|
||||
require_backend_selection,
|
||||
backend_by_role,
|
||||
):
|
||||
load_kwargs = self.customized_load_kwargs_for_component(
|
||||
server_args, component_name
|
||||
@@ -305,11 +313,13 @@ class ComponentLoader(ABC):
|
||||
attn_backend: Any,
|
||||
component_attn_name: str | None,
|
||||
require_backend_selection: bool,
|
||||
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
|
||||
) -> AutoModel:
|
||||
with self.component_attention_backend_context(
|
||||
attn_backend,
|
||||
component_attn_name,
|
||||
require_backend_selection,
|
||||
backend_by_role,
|
||||
):
|
||||
component = self.load_native(
|
||||
component_model_path,
|
||||
@@ -328,6 +338,9 @@ class ComponentLoader(ABC):
|
||||
*,
|
||||
component_attn_backend: Any = None,
|
||||
component_attn_name: str | None = None,
|
||||
component_backend_by_role: (
|
||||
dict[AttentionRole, AttentionBackendEnum] | None
|
||||
) = None,
|
||||
allow_native_fallback: bool = True,
|
||||
) -> tuple[AutoModel, float]:
|
||||
"""
|
||||
@@ -369,13 +382,26 @@ class ComponentLoader(ABC):
|
||||
component_attn_backend.name.lower(),
|
||||
matched_backend_key,
|
||||
)
|
||||
if (
|
||||
component_backend_by_role is None
|
||||
and get_component_attn_backend_context() is None
|
||||
):
|
||||
# A caller that resolved only the component-wide backend must not
|
||||
# silently drop the role-qualified overrides that go with it.
|
||||
component_backend_by_role = server_args.resolve_component_backend_by_role(
|
||||
component_name, component_attn_name
|
||||
)
|
||||
requested_backend = (
|
||||
server_args.requested_component_attention_backend(component_attn_name)
|
||||
if component_attn_name is not None
|
||||
else None
|
||||
)
|
||||
require_backend_selection = requested_backend is not None
|
||||
if require_backend_selection and (
|
||||
# A role override is as explicit as a component-wide one, so it carries
|
||||
# the same requirement that the component actually apply it.
|
||||
require_backend_selection = requested_backend is not None or bool(
|
||||
component_backend_by_role
|
||||
)
|
||||
if requested_backend is not None and (
|
||||
component_attn_backend is None
|
||||
or component_attn_backend.name.lower() != requested_backend
|
||||
):
|
||||
@@ -391,6 +417,7 @@ class ComponentLoader(ABC):
|
||||
component_attn_backend,
|
||||
component_attn_name,
|
||||
require_backend_selection,
|
||||
component_backend_by_role,
|
||||
)
|
||||
source = "sgl-diffusion"
|
||||
except (
|
||||
@@ -455,6 +482,7 @@ class ComponentLoader(ABC):
|
||||
component_attn_backend,
|
||||
component_attn_name,
|
||||
require_backend_selection,
|
||||
component_backend_by_role,
|
||||
)
|
||||
source = "native"
|
||||
logger.warning(
|
||||
@@ -1003,12 +1031,14 @@ class GenericComponentLoader(ComponentLoader):
|
||||
attn_backend: Any,
|
||||
component_attn_name: str | None,
|
||||
require_backend_selection: bool,
|
||||
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
|
||||
):
|
||||
# An unknown out-of-tree component may itself be the primary transformer.
|
||||
# Require it to opt into fallback through a registered component loader.
|
||||
return component_attn_backend_context_manager(
|
||||
attn_backend,
|
||||
component_name=component_attn_name,
|
||||
backend_by_role=backend_by_role,
|
||||
allow_global_backend_fallback=False,
|
||||
require_backend_selection=require_backend_selection,
|
||||
)
|
||||
@@ -1028,6 +1058,9 @@ class PipelineComponentLoader:
|
||||
component_architecture: str | None = None,
|
||||
component_attn_backend: Any = None,
|
||||
component_attn_name: str | None = None,
|
||||
component_backend_by_role: (
|
||||
dict[AttentionRole, AttentionBackendEnum] | None
|
||||
) = None,
|
||||
component_type: str | None = None,
|
||||
loader_cls: type[ComponentLoader] | None = None,
|
||||
):
|
||||
@@ -1059,6 +1092,7 @@ class PipelineComponentLoader:
|
||||
transformers_or_diffusers,
|
||||
component_attn_backend=component_attn_backend,
|
||||
component_attn_name=component_attn_name,
|
||||
component_backend_by_role=component_backend_by_role,
|
||||
allow_native_fallback=loader_cls is None,
|
||||
)
|
||||
except Exception:
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.configs.attention_roles import AttentionRole
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
component_attn_backend_context_manager,
|
||||
@@ -171,10 +172,12 @@ class TransformerLoader(OnlineQuantizationComponentLoader):
|
||||
attn_backend,
|
||||
component_attn_name: str | None,
|
||||
require_backend_selection: bool,
|
||||
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
|
||||
):
|
||||
return component_attn_backend_context_manager(
|
||||
attn_backend,
|
||||
component_name=component_attn_name,
|
||||
backend_by_role=backend_by_role,
|
||||
allow_global_backend_fallback=False,
|
||||
require_backend_selection=require_backend_selection,
|
||||
)
|
||||
|
||||
@@ -632,6 +632,9 @@ class ComposedPipelineBase(ABC):
|
||||
module_name, load_module_name
|
||||
)
|
||||
)
|
||||
backend_by_role = server_args.resolve_component_backend_by_role(
|
||||
module_name, load_module_name
|
||||
)
|
||||
if attn_backend is not None:
|
||||
logger.info(
|
||||
"Using %s backend for component: %s",
|
||||
@@ -648,6 +651,7 @@ class ComposedPipelineBase(ABC):
|
||||
component_architecture=architecture,
|
||||
component_attn_backend=attn_backend,
|
||||
component_attn_name=matched_backend_key or module_name,
|
||||
component_backend_by_role=backend_by_role,
|
||||
)
|
||||
|
||||
self.memory_usages[module_name] = memory_usage
|
||||
|
||||
@@ -20,6 +20,10 @@ import addict
|
||||
import yaml
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.configs.attention_roles import (
|
||||
AttentionRole,
|
||||
split_component_role_key,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import (
|
||||
LTX2PipelineConfig,
|
||||
@@ -311,6 +315,15 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
_requested_component_attention_backends: dict[str, str] | None = field(
|
||||
default=None, repr=False, compare=False
|
||||
)
|
||||
# Role-qualified overrides (``<component>.<role>``) live in their own map so
|
||||
# ``component_attention_backends`` stays a plain component -> backend dict for
|
||||
# every consumer that looks a component up by name.
|
||||
component_attention_backend_roles: dict[str, dict[str, str]] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
_requested_component_attention_backend_roles: dict[str, dict[str, str]] | None = (
|
||||
field(default=None, repr=False, compare=False)
|
||||
)
|
||||
cache_dit_config: str | dict[str, Any] | None = (
|
||||
None # cache-dit config for diffusers
|
||||
)
|
||||
@@ -1108,20 +1121,29 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
def _adjust_attention_backend(self):
|
||||
if self.attention_backend in ["fa3", "fa4"]:
|
||||
self.attention_backend = "fa"
|
||||
self.component_attention_backends = (
|
||||
self._normalize_component_attention_backends(
|
||||
self.component_attention_backends
|
||||
self.component_attention_backends, self.component_attention_backend_roles = (
|
||||
self._split_component_attention_backends(
|
||||
self.component_attention_backends,
|
||||
self.component_attention_backend_roles,
|
||||
)
|
||||
)
|
||||
# Snapshot what the user asked for before the pipeline-specific
|
||||
# adjustments below add any automatic entries of our own.
|
||||
if self._requested_component_attention_backends is None:
|
||||
self._requested_component_attention_backends = dict(
|
||||
self.component_attention_backends
|
||||
)
|
||||
self._requested_component_attention_backend_roles = {
|
||||
component: dict(entries)
|
||||
for component, entries in self.component_attention_backend_roles.items()
|
||||
}
|
||||
else:
|
||||
self._requested_component_attention_backends = (
|
||||
self._normalize_component_attention_backends(
|
||||
self._requested_component_attention_backends
|
||||
)
|
||||
(
|
||||
self._requested_component_attention_backends,
|
||||
self._requested_component_attention_backend_roles,
|
||||
) = self._split_component_attention_backends(
|
||||
self._requested_component_attention_backends,
|
||||
self._requested_component_attention_backend_roles,
|
||||
)
|
||||
|
||||
# attention_backend_config
|
||||
@@ -1148,6 +1170,20 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
text_backend,
|
||||
)
|
||||
self.component_attention_backends["text_encoder"] = "torch_sdpa"
|
||||
# A role-qualified override would otherwise outrank the backend we
|
||||
# just forced, so drop it for the same reason.
|
||||
text_encoder_roles = self.component_attention_backend_roles.pop(
|
||||
"text_encoder", None
|
||||
)
|
||||
if text_encoder_roles:
|
||||
logger.warning(
|
||||
"Ignoring per-role attention backend overrides (%s) for component "
|
||||
"text_encoder to preserve LTX2 official attention semantics",
|
||||
", ".join(
|
||||
f"{role}={backend}"
|
||||
for role, backend in sorted(text_encoder_roles.items())
|
||||
),
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import (
|
||||
MiniMaxH3PipelineConfig,
|
||||
)
|
||||
@@ -1157,6 +1193,7 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
and isinstance(self.pipeline_config, MiniMaxH3PipelineConfig)
|
||||
and self.attention_backend == "laser_attn"
|
||||
and "text_encoder" not in self.component_attention_backends
|
||||
and "text_encoder" not in self.component_attention_backend_roles
|
||||
):
|
||||
# Laser Attention is used only by the MiniMax-H3 transformer.
|
||||
# SDPA is faster than Ascend FA for its Qwen3-VL text encoder.
|
||||
@@ -1335,19 +1372,50 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def _normalize_component_attention_backends(
|
||||
cls, value: dict[str, str] | str | None
|
||||
) -> dict[str, str]:
|
||||
def _split_component_attention_backends(
|
||||
cls,
|
||||
value: dict[str, str] | str | None,
|
||||
roles: dict[str, dict[str, str]] | None = None,
|
||||
) -> tuple[dict[str, str], dict[str, dict[str, str]]]:
|
||||
"""Normalize raw entries into component-wide and role-qualified maps.
|
||||
|
||||
A key may be a bare ``component`` or a role-qualified ``component.role``.
|
||||
The optional trailing ``.<role>`` selects the backend for one attention
|
||||
role (self vs cross) and is validated against ``AttentionRole``; those
|
||||
entries are routed into the returned role map instead of the flat one so
|
||||
a component lookup by name never sees them. ``roles`` seeds the role map
|
||||
for callers that already hold a normalized one.
|
||||
"""
|
||||
raw = cls._parse_component_attention_backend_map(value)
|
||||
normalized: dict[str, str] = {}
|
||||
normalized_roles: dict[str, dict[str, str]] = {
|
||||
component: dict(entries) for component, entries in (roles or {}).items()
|
||||
}
|
||||
for component, backend in raw.items():
|
||||
if not isinstance(component, str):
|
||||
raise ValueError("Component attention backend key must be a string")
|
||||
component_name = component.strip().replace("-", "_")
|
||||
component_part, role = split_component_role_key(component.strip())
|
||||
component_name = component_part.strip().replace("-", "_")
|
||||
if not component_name:
|
||||
raise ValueError("Component attention backend key must not be empty")
|
||||
normalized[component_name] = cls._normalize_attention_backend_name(backend)
|
||||
return normalized
|
||||
backend_name = cls._normalize_attention_backend_name(backend)
|
||||
if role is None:
|
||||
normalized[component_name] = backend_name
|
||||
else:
|
||||
normalized_roles.setdefault(component_name, {})[role.value] = (
|
||||
backend_name
|
||||
)
|
||||
return normalized, normalized_roles
|
||||
|
||||
@staticmethod
|
||||
def _component_fallback_keys(component_name: str) -> list[str]:
|
||||
key = component_name.replace("-", "_")
|
||||
fallback_keys = [key]
|
||||
if key.endswith("_2"):
|
||||
# Secondary two-stage components inherit the base component backend
|
||||
# unless explicitly overridden.
|
||||
fallback_keys.append(key[:-2])
|
||||
return fallback_keys
|
||||
|
||||
def resolve_component_attention_backend(
|
||||
self, *component_names: str | None
|
||||
@@ -1355,13 +1423,7 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
for component_name in component_names:
|
||||
if component_name is None:
|
||||
continue
|
||||
key = component_name.replace("-", "_")
|
||||
fallback_keys = [key]
|
||||
if key.endswith("_2"):
|
||||
# Secondary two-stage components inherit the base component
|
||||
# backend unless explicitly overridden.
|
||||
fallback_keys.append(key[:-2])
|
||||
for backend_key in fallback_keys:
|
||||
for backend_key in self._component_fallback_keys(component_name):
|
||||
backend = self.component_attention_backends.get(backend_key)
|
||||
if backend is not None:
|
||||
return AttentionBackendEnum[backend.upper()], backend_key
|
||||
@@ -1372,7 +1434,9 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
return self._requested_component_attention_backends.get(component_name)
|
||||
|
||||
def has_requested_component_attention_backends(self) -> bool:
|
||||
return bool(self._requested_component_attention_backends)
|
||||
return bool(self._requested_component_attention_backends) or bool(
|
||||
self._requested_component_attention_backend_roles
|
||||
)
|
||||
|
||||
def is_component_attention_backend_automatic(
|
||||
self, component_name: str | None
|
||||
@@ -1382,6 +1446,33 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
and component_name in self._automatic_component_attention_backend_keys
|
||||
)
|
||||
|
||||
def resolve_component_backend_by_role(
|
||||
self, *component_names: str | None
|
||||
) -> dict[AttentionRole, AttentionBackendEnum]:
|
||||
"""Resolve the per-role backend overrides for a component.
|
||||
|
||||
For each role, tries ``<component>.<role>`` across the candidate names
|
||||
(with the same ``_2`` two-stage fallback as the component-wide lookup).
|
||||
Returns only the roles that have an explicit override configured.
|
||||
"""
|
||||
backend_by_role: dict[AttentionRole, AttentionBackendEnum] = {}
|
||||
for role in AttentionRole:
|
||||
for component_name in component_names:
|
||||
if component_name is None:
|
||||
continue
|
||||
matched = False
|
||||
for base_key in self._component_fallback_keys(component_name):
|
||||
backend = self.component_attention_backend_roles.get(
|
||||
base_key, {}
|
||||
).get(role.value)
|
||||
if backend is not None:
|
||||
backend_by_role[role] = AttentionBackendEnum[backend.upper()]
|
||||
matched = True
|
||||
break
|
||||
if matched:
|
||||
break
|
||||
return backend_by_role
|
||||
|
||||
def _adjust_warmup(self):
|
||||
if self.warmup_mode is not None and self.warmup_mode not in WARMUP_MODES:
|
||||
raise ValueError(
|
||||
|
||||
@@ -4,6 +4,7 @@ from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.attention_roles import AttentionRole
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
|
||||
AttentionRequirements,
|
||||
)
|
||||
@@ -127,6 +128,8 @@ class TestAttentionBackendFallback(unittest.TestCase):
|
||||
component_backend: AttentionBackendEnum | None = None,
|
||||
allow_global_backend_fallback: bool = False,
|
||||
server_args: object | None = None,
|
||||
backend_by_role: dict[AttentionRole, AttentionBackendEnum] | None = None,
|
||||
also_resolve_is_cross: tuple[bool, ...] = (),
|
||||
):
|
||||
if server_args is None:
|
||||
server_args = _ServerArgs(backend.name.lower(), explicit=explicit)
|
||||
@@ -148,11 +151,12 @@ class TestAttentionBackendFallback(unittest.TestCase):
|
||||
component_attn_backend_context_manager(
|
||||
component_backend,
|
||||
component_name="text_encoder",
|
||||
backend_by_role=backend_by_role,
|
||||
allow_global_backend_fallback=allow_global_backend_fallback,
|
||||
require_backend_selection=component_backend is not None,
|
||||
),
|
||||
):
|
||||
return get_attn_backend(
|
||||
resolved = get_attn_backend(
|
||||
128,
|
||||
torch.bfloat16,
|
||||
supported_attention_backends=supported,
|
||||
@@ -160,6 +164,18 @@ class TestAttentionBackendFallback(unittest.TestCase):
|
||||
default_attention_backend=default_attention_backend,
|
||||
is_cross_attention=is_cross_attention,
|
||||
)
|
||||
# Component-scoped validation runs on context exit, so a test that
|
||||
# configures a role must build a layer of that role somewhere.
|
||||
for extra_is_cross in also_resolve_is_cross:
|
||||
get_attn_backend(
|
||||
128,
|
||||
torch.bfloat16,
|
||||
supported_attention_backends=supported,
|
||||
attention_requirements=attention_requirements,
|
||||
default_attention_backend=default_attention_backend,
|
||||
is_cross_attention=extra_is_cross,
|
||||
)
|
||||
return resolved
|
||||
|
||||
def test_implicit_platform_preference_falls_back(self):
|
||||
backend = self._resolve(
|
||||
@@ -331,6 +347,120 @@ class TestAttentionBackendFallback(unittest.TestCase):
|
||||
|
||||
self.assertIs(backend, _FakeSparseBackend)
|
||||
|
||||
def test_role_backend_selects_for_cross(self):
|
||||
# Component-wide default is SDPA (implicit), but the cross role forces FA.
|
||||
backend = self._resolve(
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
explicit=False,
|
||||
is_cross_attention=True,
|
||||
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
|
||||
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
|
||||
)
|
||||
|
||||
self.assertIs(backend, _FakeFABackend)
|
||||
self.assertEqual(_FakePlatform.selected_backend, AttentionBackendEnum.FA)
|
||||
|
||||
def test_role_backend_not_applied_to_self(self):
|
||||
# The cross role override must not leak into self-attention, which keeps
|
||||
# the explicit SDPA selection.
|
||||
backend = self._resolve(
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
explicit=True,
|
||||
is_cross_attention=False,
|
||||
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
|
||||
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
|
||||
also_resolve_is_cross=(True,),
|
||||
)
|
||||
|
||||
self.assertIs(backend, _FakeSDPABackend)
|
||||
|
||||
def test_role_backend_requires_a_matching_layer(self):
|
||||
# A role override on a component that builds no layers of that role is
|
||||
# a silent no-op, which is the failure the component-wide check reports.
|
||||
with self.assertRaisesRegex(
|
||||
ComponentAttentionBackendNotAppliedError,
|
||||
"constructed no cross-attention layers",
|
||||
):
|
||||
self._resolve(
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
explicit=True,
|
||||
is_cross_attention=False,
|
||||
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
|
||||
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
|
||||
)
|
||||
|
||||
def test_role_override_does_not_mask_component_wide_failure(self):
|
||||
# A layer that pins its own backend still fails the component-wide
|
||||
# requirement even when a role override happens to name that backend.
|
||||
with (
|
||||
patch(f"{_SELECTOR}.get_global_forced_attn_backend", return_value=None),
|
||||
patch(
|
||||
f"{_SELECTOR}.get_component_forced_attn_backend",
|
||||
return_value=AttentionBackendEnum.AITER,
|
||||
),
|
||||
patch(
|
||||
f"{_SELECTOR}.get_global_server_args",
|
||||
return_value=_ServerArgs("aiter", explicit=False),
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform",
|
||||
_FakePlatform,
|
||||
),
|
||||
patch(
|
||||
f"{_SELECTOR}.resolve_name",
|
||||
side_effect=_FAKE_BACKENDS.__getitem__,
|
||||
),
|
||||
self.assertRaisesRegex(
|
||||
ComponentAttentionBackendNotAppliedError, "selected fa instead"
|
||||
),
|
||||
component_attn_backend_context_manager(
|
||||
AttentionBackendEnum.AITER,
|
||||
component_name="text_encoder",
|
||||
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
|
||||
require_backend_selection=True,
|
||||
),
|
||||
):
|
||||
# Cross-attention honors the role override.
|
||||
get_attn_backend(
|
||||
128,
|
||||
torch.bfloat16,
|
||||
supported_attention_backends={AttentionBackendEnum.FA},
|
||||
is_cross_attention=True,
|
||||
)
|
||||
# Self-attention pins FA for correctness, ignoring aiter entirely.
|
||||
get_attn_backend(
|
||||
128,
|
||||
torch.bfloat16,
|
||||
supported_attention_backends={AttentionBackendEnum.FA},
|
||||
selected_attention_backend=AttentionBackendEnum.FA,
|
||||
is_cross_attention=False,
|
||||
)
|
||||
|
||||
def test_role_backend_overrides_component_wide(self):
|
||||
# Role-qualified selection takes precedence over the component-wide backend.
|
||||
backend = self._resolve(
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
explicit=False,
|
||||
is_cross_attention=True,
|
||||
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
|
||||
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.FA},
|
||||
component_backend=AttentionBackendEnum.TORCH_SDPA,
|
||||
)
|
||||
|
||||
self.assertIs(backend, _FakeFABackend)
|
||||
|
||||
def test_role_sparse_backend_downgrades_for_cross(self):
|
||||
# A sparse backend chosen via the cross role still downgrades to dense.
|
||||
backend = self._resolve(
|
||||
AttentionBackendEnum.LASER_ATTN,
|
||||
explicit=False,
|
||||
is_cross_attention=True,
|
||||
supported={AttentionBackendEnum.FA, AttentionBackendEnum.TORCH_SDPA},
|
||||
backend_by_role={AttentionRole.CROSS: AttentionBackendEnum.LASER_ATTN},
|
||||
)
|
||||
|
||||
self.assertIs(backend, _FakeFABackend)
|
||||
|
||||
|
||||
class TestComponentAttentionBackendScope(unittest.TestCase):
|
||||
def _load_with_policy(self, allow_global_backend_fallback: bool):
|
||||
@@ -347,10 +477,12 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
|
||||
attn_backend,
|
||||
component_attn_name: str | None,
|
||||
require_backend_selection: bool,
|
||||
backend_by_role=None,
|
||||
):
|
||||
return component_attn_backend_context_manager(
|
||||
attn_backend,
|
||||
component_name=component_attn_name,
|
||||
backend_by_role=backend_by_role,
|
||||
allow_global_backend_fallback=allow_global_backend_fallback,
|
||||
require_backend_selection=require_backend_selection,
|
||||
)
|
||||
@@ -361,6 +493,10 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
|
||||
component_weights_paths = {}
|
||||
pipeline_config = SimpleNamespace(native_only_components=())
|
||||
|
||||
@staticmethod
|
||||
def resolve_component_backend_by_role(*_component_names):
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def requested_component_attention_backend(_component_name):
|
||||
return None
|
||||
@@ -453,6 +589,10 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
|
||||
component_weights_paths = {}
|
||||
pipeline_config = SimpleNamespace(native_only_components=())
|
||||
|
||||
@staticmethod
|
||||
def resolve_component_backend_by_role(*_component_names):
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def requested_component_attention_backend(_component_name):
|
||||
return "fa"
|
||||
@@ -520,6 +660,10 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
|
||||
component_weights_paths = {}
|
||||
pipeline_config = SimpleNamespace(native_only_components=())
|
||||
|
||||
@staticmethod
|
||||
def resolve_component_backend_by_role(*_component_names):
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def requested_component_attention_backend(_component_name):
|
||||
return "fa"
|
||||
@@ -572,6 +716,10 @@ class TestComponentAttentionBackendScope(unittest.TestCase):
|
||||
component_weights_paths = {}
|
||||
pipeline_config = SimpleNamespace(native_only_components=())
|
||||
|
||||
@staticmethod
|
||||
def resolve_component_backend_by_role(*_component_names):
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def requested_component_attention_backend(_component_name):
|
||||
return None
|
||||
|
||||
@@ -181,6 +181,7 @@ class TestComponentLoaderIdentity(unittest.TestCase):
|
||||
component_paths={},
|
||||
component_direct_gpu_weight_loading=set(),
|
||||
resolve_component_attention_backend=lambda *_names: (None, None),
|
||||
resolve_component_backend_by_role=lambda *_names: {},
|
||||
comfyui_mode=False,
|
||||
)
|
||||
|
||||
@@ -203,6 +204,7 @@ class TestComponentLoaderIdentity(unittest.TestCase):
|
||||
component_architecture="LTX2DurationHeadModel",
|
||||
component_attn_backend=None,
|
||||
component_attn_name="auxiliary_head",
|
||||
component_backend_by_role={},
|
||||
)
|
||||
|
||||
def test_skipped_alias_keeps_exact_override_and_structural_config(self):
|
||||
|
||||
@@ -68,6 +68,7 @@ def _loader_server_args(component_weights_paths, *, fsdp_requested=False):
|
||||
component_weights_paths=component_weights_paths,
|
||||
pipeline_config=SimpleNamespace(native_only_components=()),
|
||||
resolve_component_attention_backend=lambda _name: (None, None),
|
||||
resolve_component_backend_by_role=lambda *_names: {},
|
||||
requested_component_attention_backend=lambda _name: None,
|
||||
should_direct_gpu_weight_load_component=lambda _name: False,
|
||||
should_start_component_on_cpu=lambda _name: True,
|
||||
|
||||
@@ -47,6 +47,7 @@ class TestImageEncoderQuantizationAdmission(unittest.TestCase):
|
||||
component_precisions={},
|
||||
encoder_parallel="replicate",
|
||||
resolve_component_attention_backend=lambda _name: (None, None),
|
||||
resolve_component_backend_by_role=lambda *_names: {},
|
||||
requested_component_attention_backend=lambda _name: None,
|
||||
should_direct_gpu_weight_load_component=lambda _name: False,
|
||||
should_use_fsdp_for_component=lambda _name: False,
|
||||
@@ -268,6 +269,7 @@ class TestImageEncoderNativeLoading(unittest.TestCase):
|
||||
native_only_components=(),
|
||||
),
|
||||
resolve_component_attention_backend=lambda _name: (None, None),
|
||||
resolve_component_backend_by_role=lambda *_names: {},
|
||||
requested_component_attention_backend=lambda _name: None,
|
||||
explicit_residency_mode=lambda _name: None,
|
||||
require_component_resident=mock.Mock(),
|
||||
|
||||
@@ -200,6 +200,9 @@ class FakeServerArgs:
|
||||
def resolve_component_attention_backend(self, *names):
|
||||
return None, None
|
||||
|
||||
def resolve_component_backend_by_role(self, *names):
|
||||
return {}
|
||||
|
||||
|
||||
def _write_component(directory, config, tensors):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -284,10 +284,12 @@ def test_real_model_index_is_accepted_by_the_module_loading_path(
|
||||
loader_cls=None,
|
||||
component_attn_backend=None,
|
||||
component_attn_name=None,
|
||||
component_backend_by_role=None,
|
||||
):
|
||||
assert component_type == component_name
|
||||
assert loader_cls is None and component_attn_backend is None
|
||||
assert component_attn_name == component_name
|
||||
assert component_backend_by_role == {}
|
||||
requested[component_name] = (transformers_or_diffusers, component_architecture)
|
||||
return SimpleNamespace(name=component_name), 0.0
|
||||
|
||||
@@ -302,6 +304,7 @@ def test_real_model_index_is_accepted_by_the_module_loading_path(
|
||||
component_direct_gpu_weight_loading={},
|
||||
pipeline_config=Kandinsky6SRPipelineConfig(),
|
||||
resolve_component_attention_backend=lambda *names: (None, None),
|
||||
resolve_component_backend_by_role=lambda *names: {},
|
||||
)
|
||||
# the state ``ComposedPipelineBase.__init__`` sets up before it calls load_modules
|
||||
pipeline = object.__new__(Kandinsky6SRPipeline)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -9,6 +10,7 @@ from unittest.mock import Mock, patch
|
||||
import zmq
|
||||
|
||||
from sglang.cli.utils import get_is_diffusion_model
|
||||
from sglang.multimodal_gen.configs.attention_roles import AttentionRole
|
||||
from sglang.multimodal_gen.configs.models.fsdp import (
|
||||
is_module_list_entry,
|
||||
is_module_list_entry_in,
|
||||
@@ -87,7 +89,10 @@ from sglang.multimodal_gen.runtime.models.dits.qwen_image import (
|
||||
from sglang.multimodal_gen.runtime.pipelines.minimax_h3_pipeline import (
|
||||
MiniMaxH3Pipeline,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.platforms import (
|
||||
AttentionBackendEnum,
|
||||
current_platform,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import (
|
||||
MAX_SCHEDULER_RPC_TIMEOUT_S,
|
||||
ServerArgs,
|
||||
@@ -148,6 +153,23 @@ def _from_dict_without_model_resolution(
|
||||
return ServerArgs.from_dict(kwargs)
|
||||
|
||||
|
||||
class TestServerArgsColdImport(unittest.TestCase):
|
||||
def test_public_entry_points_import_in_fresh_processes(self):
|
||||
modules = (
|
||||
"sglang.multimodal_gen.runtime.server_args",
|
||||
"sglang.multimodal_gen.test.runner.diffusion_suite_runner",
|
||||
)
|
||||
for module in modules:
|
||||
with self.subTest(module=module):
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", f"import {module}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
|
||||
class TestPlatformLifecycleHooks(unittest.TestCase):
|
||||
def test_server_args_applies_platform_defaults(self):
|
||||
with patch.object(
|
||||
@@ -421,6 +443,66 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
|
||||
server_args.component_attention_backends, {"text_encoder": "torch_sdpa"}
|
||||
)
|
||||
|
||||
def test_role_qualified_component_keys_are_normalized(self):
|
||||
args = self._from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/data/my-model",
|
||||
"component_attention_backends": (
|
||||
"transformer.self=sage_attn,transformer.cross=fa3"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
# Role-qualified entries must not leak into the flat component map, or
|
||||
# every consumer that looks a component up by name would see them.
|
||||
self.assertEqual(args.component_attention_backends, {})
|
||||
self.assertEqual(
|
||||
args.component_attention_backend_roles,
|
||||
{"transformer": {"self": "sage_attn", "cross": "fa"}},
|
||||
)
|
||||
|
||||
def test_role_and_component_wide_keys_are_split(self):
|
||||
args = self._from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/data/my-model",
|
||||
"component_attention_backends": (
|
||||
"text_encoder=torch_sdpa,transformer=fa,transformer.cross=fa3"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
args.component_attention_backends,
|
||||
{"text_encoder": "torch_sdpa", "transformer": "fa"},
|
||||
)
|
||||
self.assertEqual(
|
||||
args.component_attention_backend_roles,
|
||||
{"transformer": {"cross": "fa"}},
|
||||
)
|
||||
|
||||
def test_role_qualified_component_normalizes_hyphenated_component(self):
|
||||
args = self._from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/data/my-model",
|
||||
"component_attention_backends": "text-encoder.cross=torch_sdpa",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(args.component_attention_backends, {})
|
||||
self.assertEqual(
|
||||
args.component_attention_backend_roles,
|
||||
{"text_encoder": {"cross": "torch_sdpa"}},
|
||||
)
|
||||
|
||||
def test_invalid_component_attention_role_raises(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self._from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/data/my-model",
|
||||
"component_attention_backends": {"transformer.bogus": "fa"},
|
||||
}
|
||||
)
|
||||
|
||||
def test_dynamic_component_precision_cli_args(self):
|
||||
parser = FlexibleArgumentParser()
|
||||
ServerArgs.add_cli_args(parser)
|
||||
@@ -443,6 +525,99 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
|
||||
|
||||
self.assertEqual(server_args.component_precisions, {"text_encoder_2": "fp32"})
|
||||
|
||||
def test_dynamic_role_qualified_cli_args(self):
|
||||
parser = FlexibleArgumentParser()
|
||||
ServerArgs.add_cli_args(parser)
|
||||
argv = [
|
||||
"--model-path",
|
||||
"/fake",
|
||||
"--component-attention-backends.transformer.cross=fa",
|
||||
]
|
||||
|
||||
with (
|
||||
patch.object(sys, "argv", ["sglang"] + argv),
|
||||
patch.object(
|
||||
PipelineConfig, "from_kwargs", return_value=QwenImagePipelineConfig()
|
||||
),
|
||||
_mock_cuda_platform(),
|
||||
):
|
||||
args, unknown_args = parser.parse_known_args(argv)
|
||||
server_args = ServerArgs.from_cli_args(args, unknown_args)
|
||||
|
||||
self.assertEqual(server_args.component_attention_backends, {})
|
||||
self.assertEqual(
|
||||
server_args.component_attention_backend_roles,
|
||||
{"transformer": {"cross": "fa"}},
|
||||
)
|
||||
|
||||
def test_resolve_component_backend_by_role_returns_overrides(self):
|
||||
args = self._from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/data/my-model",
|
||||
"component_attention_backends": {
|
||||
"transformer.self": "sage_attn",
|
||||
"transformer.cross": "fa",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
backend_by_role = args.resolve_component_backend_by_role("transformer")
|
||||
|
||||
self.assertEqual(
|
||||
backend_by_role,
|
||||
{
|
||||
AttentionRole.SELF: AttentionBackendEnum.SAGE_ATTN,
|
||||
AttentionRole.CROSS: AttentionBackendEnum.FA,
|
||||
},
|
||||
)
|
||||
|
||||
def test_resolve_component_backend_by_role_only_returns_configured_roles(self):
|
||||
args = self._from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/data/my-model",
|
||||
"component_attention_backends": {"transformer.cross": "fa"},
|
||||
}
|
||||
)
|
||||
|
||||
backend_by_role = args.resolve_component_backend_by_role("transformer")
|
||||
|
||||
self.assertEqual(
|
||||
backend_by_role, {AttentionRole.CROSS: AttentionBackendEnum.FA}
|
||||
)
|
||||
|
||||
def test_resolve_component_backend_by_role_two_stage_fallback(self):
|
||||
args = self._from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/data/my-model",
|
||||
"component_attention_backends": {"transformer.cross": "fa"},
|
||||
}
|
||||
)
|
||||
|
||||
# transformer_2 has no explicit role override, so it inherits the base
|
||||
# transformer.cross entry via the two-stage fallback.
|
||||
backend_by_role = args.resolve_component_backend_by_role("transformer_2")
|
||||
|
||||
self.assertEqual(
|
||||
backend_by_role, {AttentionRole.CROSS: AttentionBackendEnum.FA}
|
||||
)
|
||||
|
||||
def test_resolve_component_backend_by_role_prefers_two_stage_override(self):
|
||||
args = self._from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/data/my-model",
|
||||
"component_attention_backends": {
|
||||
"transformer.cross": "fa",
|
||||
"transformer_2.cross": "torch_sdpa",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
backend_by_role = args.resolve_component_backend_by_role("transformer_2")
|
||||
|
||||
self.assertEqual(
|
||||
backend_by_role, {AttentionRole.CROSS: AttentionBackendEnum.TORCH_SDPA}
|
||||
)
|
||||
|
||||
def test_layerwise_offload_components_imply_layerwise(self):
|
||||
args = self._from_dict_without_model_resolution(
|
||||
{
|
||||
|
||||
@@ -32,6 +32,7 @@ class TestTransformerLoaderFallbackAdmission(unittest.TestCase):
|
||||
"dp_size": 1,
|
||||
"use_fsdp_inference": False,
|
||||
"resolve_component_attention_backend": mock.Mock(return_value=(None, None)),
|
||||
"resolve_component_backend_by_role": mock.Mock(return_value={}),
|
||||
"requested_component_attention_backend": mock.Mock(return_value=None),
|
||||
"should_direct_gpu_weight_load_component": mock.Mock(return_value=False),
|
||||
"should_use_fsdp_for_component": mock.Mock(return_value=fsdp_requested),
|
||||
|
||||
@@ -66,6 +66,9 @@ class _FakeServerArgs:
|
||||
def resolve_component_attention_backend(self, _component_name):
|
||||
return None, None
|
||||
|
||||
def resolve_component_backend_by_role(self, *_component_names):
|
||||
return {}
|
||||
|
||||
def requested_component_attention_backend(self, _component_name):
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user