[diffusion] Support LLaDA-Image BF16 generation and editing (#37907)

This commit is contained in:
Richard Wang
2026-10-11 02:10:51 +00:00
committed by GitHub
parent e4b423256e
commit d60e0c8570
28 changed files with 3118 additions and 28 deletions
@@ -14,6 +14,7 @@ from sglang.multimodal_gen.configs.models.dits.lingbot_video_moe import (
from sglang.multimodal_gen.configs.models.dits.lingbot_world import (
LingBotWorldVideoConfig,
)
from sglang.multimodal_gen.configs.models.dits.llada_image import LLaDAImageDitConfig
from sglang.multimodal_gen.configs.models.dits.longlive2 import LongLive2VideoConfig
from sglang.multimodal_gen.configs.models.dits.minimax_h3 import MiniMaxH3DiTConfig
from sglang.multimodal_gen.configs.models.dits.mova_audio import MOVAAudioConfig
@@ -33,6 +34,7 @@ __all__ = [
"Ideogram4DistilledDiTConfig",
"LingBotWorldVideoConfig",
"LingBotVideoMoEConfig",
"LLaDAImageDitConfig",
"LongLive2VideoConfig",
"MiniMaxH3DiTConfig",
"WanAnimate2Config",
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass, field
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
SEQUENCE_MULTIPLE = 32
def editing_rope_rows(
caption_tokens: int, image_tokens: list[int], sigvq_tokens: int
) -> int:
# Must match LLaDAImageTransformer2DModel._prepare_editing_sequences: two
# padded caption copies and the padded image grids precede the SigVQ tokens.
def padded(length: int) -> int:
return length + (-length) % SEQUENCE_MULTIPLE
return (
1
+ 2 * padded(caption_tokens)
+ sum(padded(tokens) for tokens in image_tokens)
+ sigvq_tokens
)
@dataclass
class LLaDAImageArchConfig(DiTArchConfig):
all_patch_size: tuple[int, ...] = (1,)
all_f_patch_size: tuple[int, ...] = (1,)
axes_dims: tuple[int, ...] = (32, 48, 48)
axes_lens: tuple[int, ...] = (32768, 1024, 1024)
cap_feat_dim: int = 2560
dim: int = 3840
in_channels: int = 128
n_heads: int = 30
n_layers: int = 30
n_refiner_layers: int = 2
norm_eps: float = 1e-5
qk_norm: bool = True
rope_theta: float = 256.0
semantic_feat_dim: int = 4096
t_scale: float = 1000.0
_supported_attention_backends: set[AttentionBackendEnum] = field(
default_factory=lambda: {
AttentionBackendEnum.FA,
AttentionBackendEnum.TORCH_SDPA,
}
)
param_names_mapping: dict = field(
default_factory=lambda: {
r"(.*)\.attention\.to_q\.weight$": (r"\1.attention.to_qkv.weight", 0, 3),
r"(.*)\.attention\.to_k\.weight$": (r"\1.attention.to_qkv.weight", 1, 3),
r"(.*)\.attention\.to_v\.weight$": (r"\1.attention.to_qkv.weight", 2, 3),
r"(.*)\.feed_forward\.w1\.weight$": (r"\1.feed_forward.w13.weight", 0, 2),
r"(.*)\.feed_forward\.w3\.weight$": (r"\1.feed_forward.w13.weight", 1, 2),
}
)
def __post_init__(self) -> None:
super().__post_init__()
self.hidden_size = self.dim
self.num_attention_heads = self.n_heads
self.num_channels_latents = self.in_channels
@dataclass
class LLaDAImageDitConfig(DiTConfig):
arch_config: LLaDAImageArchConfig = field(default_factory=LLaDAImageArchConfig)
prefix: str = "llada_image"
@@ -42,6 +42,9 @@ from sglang.multimodal_gen.configs.pipeline_configs.lingbot_world import (
LingBotWorldCausalDMDConfig,
LingBotWorldV2CausalDMDConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.llada_image import (
LLaDAImagePipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import (
LTX2PipelineConfig,
LTX23PipelineConfig,
@@ -83,6 +86,7 @@ __all__ = [
"HunyuanImage3PipelineConfig",
"Ideogram4PipelineConfig",
"Ideogram4DistilledPipelineConfig",
"LLaDAImagePipelineConfig",
"FluxPipelineConfig",
"Flux2PipelineConfig",
"Flux2KleinPipelineConfig",
@@ -0,0 +1,224 @@
# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass, field
import numpy as np
import torch
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.configs.models.dits.llada_image import (
LLaDAImageDitConfig,
editing_rope_rows,
)
from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ModelTaskType,
SpatialImagePipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.flux import _unpatchify_latents
def flux2_vae_bn_stats(vae, like: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
mean = vae.bn.running_mean.view(1, -1, 1, 1).to(like)
variance = vae.bn.running_var.view(1, -1, 1, 1)
std = torch.sqrt(variance + vae.config.arch_config.batch_norm_eps).to(like)
return mean, std
@dataclass
class LLaDAImagePipelineConfig(SpatialImagePipelineConfig):
task_type: ModelTaskType = ModelTaskType.TI2I
should_use_guidance: bool = False
dit_config: LLaDAImageDitConfig = field(default_factory=LLaDAImageDitConfig)
vae_config: Flux2VAEConfig = field(default_factory=Flux2VAEConfig)
vae_precision: str = "bf16"
vae_tiling: bool = False
vae_sp: bool = False
text_encoder_configs: tuple = ()
text_encoder_precisions: tuple[str, ...] = ()
preprocess_text_funcs: tuple = ()
postprocess_text_funcs: tuple = ()
latent_scale_factor: int = 16
# Editing feeds the source to SigVQ at half size, and its patch is 16 pixels.
editing_size_multiple: int = 32
# The embedded text worker admits 8192 prefill tokens shared by the two
# CFG sequences and each sequence appends 256 query tokens.
max_request_text_tokens: int = 3584
def prepare_sigmas(self, sigmas, num_inference_steps):
if sigmas is not None:
return sigmas
schedule = np.linspace(0.001, 1.0, num_inference_steps + 1)[:-1]
schedule = (1 - (1 - schedule**1.17) ** 0.8) ** 1.1
return (1 - schedule).tolist()
def prepare_latent_shape(self, batch, batch_size, num_frames):
del num_frames
self.validate_output_size(
batch.width, batch.height, editing=batch.condition_image is not None
)
return (
batch_size,
self.dit_config.num_channels_latents,
batch.height // self.latent_scale_factor,
batch.width // self.latent_scale_factor,
)
def prepare_calculated_size(self, image):
del image
return None
def calculate_condition_image_size(self, image, width, height):
del image, width, height
return None
def validate_server_args(self, server_args) -> None:
super().validate_server_args(server_args)
if any(
value != 1
for value in (
server_args.num_gpus,
server_args.sp_degree,
server_args.tp_size,
server_args.dp_size,
server_args.cfg_parallel_degree,
)
):
raise ValueError("LLaDA-Image currently supports single-GPU serving")
if server_args.quantization is not None:
raise ValueError("LLaDA-Image currently supports BF16 checkpoints")
if envs.SGLANG_CACHE_DIT_ENABLED:
raise ValueError("LLaDA-Image does not support cache-dit")
if server_args.text_encoder_cpu_offload:
if server_args.is_arg_explicitly_set("text_encoder_cpu_offload"):
raise ValueError("LLaDA-Image requires a resident text encoder")
server_args.text_encoder_cpu_offload = False
for attribute in ("layerwise_offload_components", "cpu_offload_components"):
components = getattr(server_args, attribute)
if components is not None and "text_encoder" in components:
if server_args.is_arg_explicitly_set(attribute):
raise ValueError("LLaDA-Image requires a resident text encoder")
setattr(
server_args,
attribute,
[name for name in components if name != "text_encoder"],
)
if server_args.explicit_residency_mode("text_encoder") not in (
None,
"resident",
):
raise ValueError("LLaDA-Image requires a resident text encoder")
def supports_disaggregation(self) -> bool:
return False
def validate_output_size(self, width: int, height: int, editing: bool) -> None:
multiple = self.editing_size_multiple if editing else self.latent_scale_factor
if width % multiple != 0 or height % multiple != 0:
task = "editing" if editing else "generation"
raise ValueError(
f"LLaDA-Image {task} width and height must be divisible by "
f"{multiple}, got {width}x{height}"
)
self._validate_spatial_rope_bounds(width, height)
if editing:
# Lower bound without the prompt, the DiT checks the exact length.
image_tokens = (width // self.latent_scale_factor) * (
height // self.latent_scale_factor
)
sigvq_tokens = (width // self.editing_size_multiple) * (
height // self.editing_size_multiple
)
rows = editing_rope_rows(0, [image_tokens, image_tokens], sigvq_tokens)
limit = self.dit_config.arch_config.axes_lens[0]
if rows > limit:
raise ValueError(
f"LLaDA-Image editing at {width}x{height} needs at least {rows} "
f"sequence positions, above the model limit of {limit}"
)
def _validate_spatial_rope_bounds(self, width: int, height: int) -> None:
# The served DiT uses patch size 1.
axes_lens = self.dit_config.arch_config.axes_lens
max_height = axes_lens[1] * self.latent_scale_factor
max_width = axes_lens[2] * self.latent_scale_factor
if height > max_height or width > max_width:
raise ValueError(
f"LLaDA-Image output size {width}x{height} exceeds spatial RoPE "
f"bounds of {max_width}x{max_height} pixels"
)
@staticmethod
def _cond_kwargs(batch, device, dtype, negative: bool) -> dict:
def to_device(values):
if values is None:
return None
return [value.to(device=device, dtype=dtype) for value in values]
image_embeds = batch.image_embeds
if image_embeds:
image_embeds = to_device(image_embeds)
if negative:
# The unconditional edit keeps the source latents without SigVQ.
empty = image_embeds[0].new_zeros((0, image_embeds[0].shape[-1]))
image_embeds = [empty] * batch.batch_size
return {
"encoder_hidden_states_image": image_embeds,
"source_latents": to_device(batch.source_latents),
}
def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype):
return self._cond_kwargs(batch, device, dtype, negative=False)
def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype):
return self._cond_kwargs(batch, device, dtype, negative=True)
def get_decode_scale_and_shift(self, device, dtype, vae):
del device, dtype, vae
return 1.0, None
def preprocess_decoding(self, latents, server_args=None, vae=None):
vae_parameter = next(vae.parameters())
latents = latents.to(device=vae_parameter.device, dtype=vae_parameter.dtype)
mean, std = flux2_vae_bn_stats(vae, latents)
return _unpatchify_latents(latents * std + mean)
def post_denoising_loop(self, latents, batch):
del batch
return latents
def register():
from sglang.multimodal_gen.configs.sample.llada_image import (
LLaDAImageSamplingParams,
LLaDAImageTurboSamplingParams,
)
from sglang.multimodal_gen.registry import register_configs
# Base and Turbo checkpoints share every class name, so Turbo registers
# first and the first matching detector wins.
register_configs(
sampling_param_cls=LLaDAImageTurboSamplingParams,
pipeline_config_cls=LLaDAImagePipelineConfig,
hf_model_paths=["inclusionAI/LLaDA-Image-Turbo"],
model_detectors=[
lambda hf_id: "llada-image" in hf_id.lower() and "turbo" in hf_id.lower()
],
)
register_configs(
sampling_param_cls=LLaDAImageSamplingParams,
pipeline_config_cls=LLaDAImagePipelineConfig,
hf_model_paths=["inclusionAI/LLaDA-Image"],
model_detectors=[
lambda hf_id: (
(
"lladaimagepipeline" in hf_id.lower()
or "llada-image" in hf_id.lower()
)
and "turbo" not in hf_id.lower()
)
],
)
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass
from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType,
SamplingParams,
)
@dataclass
class LLaDAImageSamplingParams(SamplingParams):
data_type: DataType = DataType.IMAGE
negative_prompt: str | None = None
# Base checkpoint recipe from the model card.
num_inference_steps: int = 50
num_frames: int = 1
height: int = 1024
width: int = 1024
guidance_scale: float = 5.0
max_sequence_length: int = 2048
def _validate_with_pipeline_config(self, pipeline_config):
super()._validate_with_pipeline_config(pipeline_config)
if self.enable_cache_dit or self.cache_dit_params is not None:
raise ValueError("LLaDA-Image does not support cache-dit")
if self.max_sequence_length is not None and not (
0 < self.max_sequence_length <= pipeline_config.max_request_text_tokens
):
raise ValueError(
"LLaDA-Image max_sequence_length must be between 1 and 3584"
)
pipeline_config.validate_output_size(
self.width, self.height, editing=bool(self.image_path)
)
@dataclass
class LLaDAImageTurboSamplingParams(LLaDAImageSamplingParams):
# The distilled Turbo checkpoint samples in 4 steps without CFG.
num_inference_steps: int = 4
guidance_scale: float = 1.0
@@ -248,6 +248,14 @@ def _get_response_resize(
return f"{width}x{height}"
def _build_request_sampling_params(request_id: str, **kwargs):
# Request validation errors are client errors, as in the video API.
try:
return build_sampling_params(request_id, **kwargs)
except (TypeError, ValueError) as e:
raise HTTPException(status_code=400, detail=str(e)) from e
@router.post("/generations", response_model=ImageResponse)
async def generations(
request: ImageGenerationsRequest,
@@ -266,7 +274,7 @@ async def generations(
)
with temp_dir_if_disabled(server_args.output_path) as output_dir:
sampling = build_sampling_params(
sampling = _build_request_sampling_params(
request_id,
prompt=prompt,
task_type=request.task_type,
@@ -465,7 +473,7 @@ async def edits(
task="image_edit",
image_paths=input_paths,
)
sampling = build_sampling_params(
sampling = _build_request_sampling_params(
request_id,
prompt=prompt,
task_type=task_type,
@@ -26,6 +26,10 @@ from sglang.kernels.ops.layernorm.norm import (
can_use_fused_inplace_qknorm,
fused_inplace_qknorm,
)
from sglang.kernels.ops.layernorm.rmsnorm_hf import (
is_supported_rmsnorm_hf_hidden_size,
rmsnorm_hf,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -102,11 +106,13 @@ class RMSNorm(CustomOp):
eps: float = 1e-6,
dtype: torch.dtype = torch.float32,
var_hidden_size: Optional[int] = None,
cast_x_before_out_mul: bool = False,
) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
self.hidden_size = hidden_size
self.cast_x_before_out_mul = cast_x_before_out_mul
self.variance_size_override = (
None if var_hidden_size == hidden_size else var_hidden_size
)
@@ -126,6 +132,22 @@ class RMSNorm(CustomOp):
residual: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
shape = x.shape
if self.cast_x_before_out_mul:
if residual is not None or self.variance_size_override is not None:
return self.forward_native(x, residual)
x_2d = x.contiguous().reshape(-1, shape[-1])
if (
x_2d.dtype in (torch.float16, torch.bfloat16)
and self.weight.dtype == x_2d.dtype
and is_supported_rmsnorm_hf_hidden_size(x_2d.shape[-1])
):
return rmsnorm_hf(
x_2d,
self.weight.data,
self.variance_epsilon,
).view(shape)
return self.forward_native(x)
x = x.reshape(-1, shape[-1])
if residual is not None:
residual_shape = residual.shape
@@ -200,10 +222,13 @@ class RMSNorm(CustomOp):
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
x = x * torch.rsqrt(variance + self.variance_epsilon)
weight = self.weight
if x.device.type == "mps" and weight.dtype != x.dtype:
weight = weight.to(dtype=x.dtype)
x = (x * weight).to(orig_dtype)
if self.cast_x_before_out_mul:
x = self.weight * x.to(orig_dtype)
else:
weight = self.weight
if x.device.type == "mps" and weight.dtype != x.dtype:
weight = weight.to(dtype=x.dtype)
x = (x * weight).to(orig_dtype)
if residual is None:
return x
else:
File diff suppressed because it is too large Load Diff
@@ -29,6 +29,7 @@ import torch
from diffusers.configuration_utils import ConfigMixin, register_to_config
from diffusers.schedulers.scheduling_utils import SchedulerMixin
from diffusers.utils import BaseOutput
from diffusers.utils.torch_utils import randn_tensor
from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import (
@@ -92,6 +93,9 @@ class FlowMatchEulerDiscreteScheduler(
Whether to use exponential sigmas for step sizes in the noise schedule during sampling.
use_beta_sigmas (`bool`, defaults to False):
Whether to use beta sigmas for step sizes in the noise schedule during sampling.
use_uniform_sigmas (`bool`, defaults to False):
Whether to use a uniform pre-shift sigma grid that excludes the terminal zero. The terminal zero is
appended after shifting.
time_shift_type (`str`, defaults to "exponential"):
The type of dynamic resolution-dependent timestep shifting to apply. Either "exponential" or "linear".
stochastic_sampling (`bool`, defaults to False):
@@ -116,6 +120,7 @@ class FlowMatchEulerDiscreteScheduler(
use_karras_sigmas: bool | None = False,
use_exponential_sigmas: bool | None = False,
use_beta_sigmas: bool | None = False,
use_uniform_sigmas: bool | None = False,
time_shift_type: str = "exponential",
stochastic_sampling: bool = False,
):
@@ -125,12 +130,14 @@ class FlowMatchEulerDiscreteScheduler(
self.config.use_beta_sigmas,
self.config.use_exponential_sigmas,
self.config.use_karras_sigmas,
self.config.use_uniform_sigmas,
]
)
> 1
):
raise ValueError(
"Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, `config.use_karras_sigmas` can be used."
"Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, "
"`config.use_karras_sigmas`, `config.use_uniform_sigmas` can be used."
)
if time_shift_type not in {"exponential", "linear"}:
raise ValueError(
@@ -341,13 +348,18 @@ class FlowMatchEulerDiscreteScheduler(
sigmas_array: np.ndarray
if sigmas is None:
if timesteps_array is None:
timesteps_array = np.linspace(
self._sigma_to_t(self.sigma_max),
self._sigma_to_t(self.sigma_min),
num_inference_steps,
)
sigmas_array = timesteps_array / self.config.num_train_timesteps
if timesteps_array is None and self.config.use_uniform_sigmas:
sigmas_array = np.linspace(
1.0, 0.0, num_inference_steps + 1, dtype=np.float32
)[:-1]
else:
if timesteps_array is None:
timesteps_array = np.linspace(
self._sigma_to_t(self.sigma_max),
self._sigma_to_t(self.sigma_min),
num_inference_steps,
)
sigmas_array = timesteps_array / self.config.num_train_timesteps
else:
sigmas_array = np.array(sigmas).astype(np.float32)
num_inference_steps = len(sigmas_array)
@@ -451,7 +463,7 @@ class FlowMatchEulerDiscreteScheduler(
s_tmin: float = 0.0,
s_tmax: float = float("inf"),
s_noise: float = 1.0,
generator: torch.Generator | None = None,
generator: torch.Generator | list[torch.Generator] | None = None,
per_token_timesteps: torch.Tensor | None = None,
batch=None,
return_dict: bool = True,
@@ -472,8 +484,8 @@ class FlowMatchEulerDiscreteScheduler(
s_tmax (`float`):
s_noise (`float`, defaults to 1.0):
Scaling factor for noise added to the sample.
generator (`torch.Generator`, *optional*):
A random number generator.
generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
A random number generator, or one generator per batch item.
per_token_timesteps (`torch.Tensor`, *optional*):
The timesteps for each token in the sample.
return_dict (`bool`):
@@ -532,7 +544,7 @@ class FlowMatchEulerDiscreteScheduler(
else:
if self.config.stochastic_sampling:
x0 = sample - current_sigma * model_output
noise = torch.randn(
noise = randn_tensor(
sample.shape,
generator=generator,
device=sample.device,
@@ -0,0 +1,122 @@
# SPDX-License-Identifier: Apache-2.0
from typing import ClassVar
from diffusers.image_processor import VaeImageProcessor
from diffusers.utils.torch_utils import randn_tensor
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
PlainStateDictComponentLoader,
)
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
ComposedPipelineBase,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages import (
DecodingStage,
DenoisingStage,
InputValidationStage,
LatentPreparationStage,
TimestepPreparationStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.llada_image.conditioning import (
LLaDAImageTextConditioningStage,
LLaDAImageTextEncoderRunner,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.llada_image.source import (
LLaDAImageSourceImageConditioningStage,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
class LLaDAImageLatentPreparationStage(LatentPreparationStage):
def forward(self, batch, server_args):
if batch.latents is None:
shape = server_args.pipeline_config.prepare_latent_shape(
batch, batch.batch_size, batch.num_frames
)
model_dtype = next(self.transformer.parameters()).dtype
batch.latents = randn_tensor(
shape,
generator=batch.generator,
device=get_local_torch_device(),
dtype=model_dtype,
).float()
batch = super().forward(batch, server_args)
return batch
class LLaDAImagePipeline(ComposedPipelineBase):
pipeline_name = "LLaDAImagePipeline"
component_loaders = {
"queryformer": PlainStateDictComponentLoader,
"text_projection": PlainStateDictComponentLoader,
"sigvq": PlainStateDictComponentLoader,
}
_required_config_modules: ClassVar[list[str]] = [
"queryformer",
"text_projection",
"sigvq",
"tokenizer",
"transformer",
"scheduler",
"vae",
]
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
text_runner = LLaDAImageTextEncoderRunner(
model_root=self.model_path,
queryformer=self.get_module("queryformer"),
text_projection=self.get_module("text_projection"),
tokenizer=self.get_module("tokenizer"),
server_args=server_args,
)
self.add_module("text_encoder", text_runner)
self.add_stage(InputValidationStage(), "input_validation_stage")
self.add_stage(
LLaDAImageTextConditioningStage(text_runner),
"text_conditioning_stage",
)
self.add_stage(
LLaDAImageSourceImageConditioningStage(
sigvq=self.get_module("sigvq"),
vae=self.get_module("vae"),
image_processor=VaeImageProcessor(
vae_scale_factor=server_args.pipeline_config.latent_scale_factor
),
),
"source_image_conditioning_stage",
)
self.add_stage(
TimestepPreparationStage(self.get_module("scheduler")),
"timestep_preparation_stage",
)
self.add_stage(
LLaDAImageLatentPreparationStage(
scheduler=self.get_module("scheduler"),
transformer=self.get_module("transformer"),
),
"latent_preparation_stage",
)
self.add_stage(
DenoisingStage(
transformer=self.get_module("transformer"),
scheduler=self.get_module("scheduler"),
vae=self.get_module("vae"),
pipeline=self,
),
"denoising_stage",
)
self.add_stage(
DecodingStage(
vae=self.get_module("vae"), pipeline=self, component_name="vae"
),
"decoding_stage",
)
EntryClass = [LLaDAImagePipeline]
@@ -135,6 +135,7 @@ class Req:
# Latent tensors
latents: torch.Tensor | None = None
source_latents: list[torch.Tensor] | None = None
y: torch.Tensor | None = None
# Flux-2
latent_ids: torch.Tensor | None = None
@@ -0,0 +1 @@
# SPDX-License-Identifier: Apache-2.0
@@ -0,0 +1,408 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import json
from array import array
from pathlib import Path
import torch
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
prepare_diffusers_component_path_for_loading,
)
# srt group globals the diffusion runtime mirrors from its own TP group. srt's
# initialize_model_parallel refuses to run while any of them is set.
_MIRRORED_SRT_GROUPS = ("_TP", "_ATTN_TP", "_MOE_TP", "_MOE_EP")
def _resolve_llada_image_component_path(
model_root: str,
server_args: ServerArgs,
component_name: str,
) -> str:
override_path = server_args.component_paths.get(component_name)
if override_path is not None:
return prepare_diffusers_component_path_for_loading(override_path)
return str(Path(model_root) / component_name)
def format_llada_image_prompt(prompt: str | None) -> str:
if prompt is None:
return "<role>HUMAN</role> Generate an image.\n<role>ASSISTANT</role>\n<IMAGE1>"
return (
f"<role>HUMAN</role> Generate an image: {prompt.strip()}\n"
"<role>ASSISTANT</role>\n<IMAGE1>"
)
class LLaDAImageTextEncoderRunner:
"""One-shot LLaDA2 prefill runner used to produce diffusion cap features."""
def __init__(
self,
model_root: str,
queryformer,
text_projection,
tokenizer,
server_args: ServerArgs,
) -> None:
import sglang.srt.distributed.parallel_state as srt_parallel_state
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.distributed.bootstrap import (
init_layer_runtime,
init_parallel_runtime,
)
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.unified_cache.components import ComponentType
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.runtime_context import SpawnRanks, create_context, use_context
from sglang.srt.server_args import ServerArgs as SRTServerArgs
self.queryformer = queryformer
self.text_projection = text_projection
self.tokenizer = tokenizer
text_encoder_path = _resolve_llada_image_component_path(
model_root, server_args, "text_encoder"
)
text_encoder_config = json.loads(
(Path(text_encoder_path) / "config.json").read_text()
)
if "quantization_config" in text_encoder_config or (
text_encoder_config.get("llada_fp8_experts", {}).get("enabled")
):
raise ValueError(
"LLaDA-Image serves only the BF16 checkpoints, such as "
"inclusionAI/LLaDA-Image, and this text encoder is quantized"
)
tokenizer_path = _resolve_llada_image_component_path(
model_root, server_args, "tokenizer"
)
device = get_local_torch_device()
gpu_id = device.index
nccl_port = server_args.nccl_port or 29500
srt_args = SRTServerArgs(
model_path=text_encoder_path,
tokenizer_path=tokenizer_path,
trust_remote_code=server_args.trust_remote_code,
revision=server_args.revision,
skip_tokenizer_init=True,
dtype="bfloat16",
tp_size=1,
attention_backend="llada2_cfg_flashinfer",
disable_cuda_graph=True,
disable_radix_cache=True,
chunked_prefill_size=-1,
max_prefill_tokens=8192,
max_total_tokens=8192,
max_running_requests=2,
)
self.runtime_context = create_context(
srt_args,
role="scheduler",
ranks=SpawnRanks(world_rank=0, gpu_id=gpu_id),
)
with use_context(self.runtime_context):
# The diffusion runtime mirrors its TP group into the srt globals,
# the MoE groups included. Detach the groups without clearing the
# encoder context's ranks, then restore the mirror and keep the
# encoder groups for forwards.
saved_groups = {
name: getattr(srt_parallel_state, name) for name in _MIRRORED_SRT_GROUPS
}
try:
for name in _MIRRORED_SRT_GROUPS:
setattr(srt_parallel_state, name, None)
init_parallel_runtime(
server_args=srt_args,
device=device.type,
dist_port=nccl_port,
)
init_layer_runtime(model_config=ModelConfig.from_server_args(srt_args))
self.worker = TpModelWorker(
server_args=srt_args,
gpu_id=gpu_id,
nccl_port=nccl_port,
)
# Run the post-construction init phases the srt scheduler drives.
self.worker.alloc_memory_pool()
self.worker.init_attention_backends()
self.worker.init_cuda_graphs()
self.encoder_groups = {
name: getattr(srt_parallel_state, name)
for name in _MIRRORED_SRT_GROUPS
}
finally:
for name, group in saved_groups.items():
setattr(srt_parallel_state, name, group)
self.server_args = srt_args
self.model_runner = self.worker.model_runner
self.page_size = self.model_runner.page_size
self.req_to_token_pool, self.token_to_kv_pool_allocator = (
self.worker.get_memory_pool()
)
# The radix-disabled cache srt builds for a full-attention model.
self.tree_cache = UnifiedRadixCache(
CacheInitParams(
disable=True,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
page_size=self.page_size,
tree_components=(ComponentType.FULL,),
)
)
def _prepare_input_embeds(
self,
input_ids: torch.Tensor,
sequence_lengths: list[int],
text_lengths: list[int],
component_context,
) -> torch.Tensor:
embeddings = self.model_runner.model.get_input_embeddings()
inputs = []
offset = 0
with component_context(
component_name="queryformer", module=self.queryformer
) as queryformer:
assert queryformer is not None
queryformer_dtype = queryformer.dtype
for sequence_length, text_length in zip(
sequence_lengths, text_lengths, strict=True
):
text_ids = input_ids[offset : offset + text_length]
text_embeds = embeddings(text_ids)
text_mask = torch.ones(
(1, text_length), dtype=torch.bool, device=text_embeds.device
)
query_embeds = queryformer(
text_embeds.unsqueeze(0).to(dtype=queryformer_dtype),
text_mask,
)
inputs.append(
torch.cat(
[text_embeds, query_embeds.squeeze(0).to(text_embeds.dtype)],
dim=0,
)
)
offset += sequence_length
return torch.cat(inputs, dim=0)
@torch.no_grad()
def encode(self, prompts: list[str], max_sequence_length: int, component_context):
import sglang.multimodal_gen.runtime.distributed.parallel_state as mm_parallel_state
import sglang.srt.distributed.parallel_state as srt_parallel_state
from sglang.srt.runtime_context import use_context
# Restore the encoder context inside the diffusion TP scope.
with (
mm_parallel_state.use_tensor_parallel_group(self.encoder_groups["_TP"]),
use_context(self.runtime_context),
):
# The TP scope points _TP and _ATTN_TP at one group. Use every
# encoder group the srt layers read during the forward.
swapped = [name for name in _MIRRORED_SRT_GROUPS if name != "_TP"]
saved_groups = {name: getattr(srt_parallel_state, name) for name in swapped}
try:
for name in swapped:
setattr(srt_parallel_state, name, self.encoder_groups[name])
return self._encode_impl(
prompts, max_sequence_length, component_context=component_context
)
finally:
for name, group in saved_groups.items():
setattr(srt_parallel_state, name, group)
def _encode_impl(
self, prompts: list[str], max_sequence_length: int, component_context
):
from sglang.srt.managers.overlap_utils import resolve_forward_inputs
from sglang.srt.managers.schedule_batch import Req as SRTReq
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder
from sglang.srt.mem_cache.common import release_kv_cache
from sglang.srt.model_executor.forward_batch_info import (
ForwardBatch,
ForwardMode,
)
from sglang.srt.sampling.sampling_params import (
SamplingParams as SRTSamplingParams,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
tokenized = self.tokenizer(
prompts,
add_special_tokens=True,
padding=False,
truncation=True,
max_length=max_sequence_length,
).input_ids
query_count = int(self.queryformer.config.num_queries)
reqs = []
text_lengths = []
sequence_lengths = []
for index, text_input_ids in enumerate(tokenized):
sampling_params = SRTSamplingParams(max_new_tokens=1, temperature=0.0)
sampling_params.normalize(None)
sequence_ids = list(text_input_ids) + [0] * query_count
req = SRTReq(
rid=f"llada-image-cap-{index}",
origin_input_text="",
origin_input_ids=array("q", sequence_ids),
sampling_params=sampling_params,
vocab_size=self.worker.model_config.vocab_size,
eos_token_ids=set(),
return_hidden_states=True,
dllm_config=None,
)
req.skip_radix_cache_insert = True
reqs.append(req)
text_lengths.append(len(text_input_ids))
sequence_lengths.append(len(sequence_ids))
chunked_prefill_size = self.server_args.chunked_prefill_size
if chunked_prefill_size <= 0:
chunked_prefill_size = None
adder = PrefillAdder(
self.page_size,
self.tree_cache,
self.token_to_kv_pool_allocator,
None,
0.5,
self.server_args.max_prefill_tokens,
chunked_prefill_size,
prefill_max_requests=len(reqs),
dllm_config=None,
)
for req in reqs:
req.init_next_round_input(self.tree_cache)
result = adder.add_one_req(
req, has_chunked_req=False, truncation_align_size=None
)
if req not in adder.can_run_list:
raise RuntimeError(
"Insufficient prefill capacity for LLaDA-Image conditioning"
)
if result != AddReqResult.CONTINUE:
break
if len(adder.can_run_list) != len(reqs):
raise RuntimeError("Could not batch all LLaDA-Image conditioning prompts")
batch = ScheduleBatch.init_new(
reqs=adder.can_run_list,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
tree_cache=self.tree_cache,
model_config=self.worker.model_config,
enable_overlap=False,
spec_algorithm=SpeculativeAlgorithm.NONE,
dllm_config=None,
)
try:
batch.prepare_for_extend()
batch.forward_mode = ForwardMode.EXTEND
# The FutureMap argument is only consulted on decode/spec paths.
resolve_forward_inputs(batch, None)
forward_batch = ForwardBatch.init_new(
batch,
self.model_runner,
return_hidden_states_before_norm=False,
)
forward_batch.input_embeds = self._prepare_input_embeds(
forward_batch.input_ids,
sequence_lengths,
text_lengths,
component_context=component_context,
)
forward_batch.llada_image_conditioning_text_lens_cpu = text_lengths
model_output = self.model_runner.forward(forward_batch=forward_batch)
if not self.model_runner.attn_backend.conditioning_mask_active:
raise RuntimeError(
"LLaDA-Image conditioning ran without its block attention mask "
"because the forward batch lost the conditioning text lengths"
)
hidden_states = model_output.logits_output.hidden_states
projected = []
offset = 0
with component_context(
component_name="text_projection", module=self.text_projection
) as text_projection:
assert text_projection is not None
projection_dtype = text_projection.dtype
for sequence_length in sequence_lengths:
output = text_projection(
hidden_states[offset : offset + sequence_length]
.unsqueeze(0)
.to(dtype=projection_dtype)
)
projected.append(output.squeeze(0))
offset += sequence_length
return projected
finally:
for req in reqs:
if not req.kv.is_kv_released:
release_kv_cache(req, self.tree_cache, checkpoint=False)
elif req.kv.holds_kv:
# KV alloc failed after the row grab. Freeing the bare row
# avoids the release_kv_cache lifecycle assert.
self.req_to_token_pool.free(req)
class LLaDAImageTextConditioningStage(PipelineStage):
def __init__(self, runner: LLaDAImageTextEncoderRunner) -> None:
super().__init__()
self.runner = runner
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
del server_args
stage_name = self._component_stage_name(stage_name)
return [
ComponentUse(stage_name, "queryformer"),
ComponentUse(stage_name, "text_projection"),
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
if not isinstance(batch.prompt, str):
raise TypeError("LLaDA-Image currently supports one prompt per request")
do_cfg = batch.guidance_scale > 1.0
prompts = [format_llada_image_prompt(batch.prompt)]
if do_cfg:
prompts.append(format_llada_image_prompt(batch.negative_prompt))
outputs = self.runner.encode(
prompts,
max_sequence_length=batch.max_sequence_length or 2048,
component_context=self.use_declared_component,
)
masks = [
torch.ones(output.shape[0], dtype=torch.bool, device=output.device)
for output in outputs
]
output_count = batch.num_outputs_per_prompt
batch.prompt_embeds = [outputs[0]] * output_count
batch.prompt_attention_mask = [masks[0]] * output_count
batch.do_classifier_free_guidance = do_cfg
if do_cfg:
batch.negative_prompt_embeds = [outputs[1]] * output_count
batch.negative_attention_mask = [masks[1]] * output_count
else:
batch.negative_prompt_embeds = []
batch.negative_attention_mask = []
return batch
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import torch
import torch.nn.functional as F
from sglang.multimodal_gen.configs.pipeline_configs.flux import _patchify_latents
from sglang.multimodal_gen.configs.pipeline_configs.llada_image import (
flux2_vae_bn_stats,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.server_args import ServerArgs
def _module_device_dtype(module: torch.nn.Module) -> tuple[torch.device, torch.dtype]:
parameter = next(module.parameters())
return parameter.device, parameter.dtype
class LLaDAImageSourceImageConditioningStage(PipelineStage):
def __init__(self, sigvq, vae, image_processor) -> None:
super().__init__()
self.sigvq = sigvq
self.vae = vae
self.image_processor = image_processor
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
del server_args
stage_name = self._component_stage_name(stage_name)
return [
ComponentUse(stage_name, "sigvq"),
ComponentUse(stage_name, "vae"),
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
del server_args
if batch.condition_image is None:
batch.source_latents = None
return batch
source_images = (
batch.condition_image
if isinstance(batch.condition_image, list)
else [batch.condition_image]
)
if len(source_images) != 1:
raise ValueError("LLaDA-Image editing requires exactly one source image")
image = self.image_processor.preprocess(
source_images[0],
height=batch.height,
width=batch.width,
)
if batch.batch_size > 1:
image = image.repeat(batch.batch_size, 1, 1, 1)
sigvq_pixels = F.interpolate(
image.float(),
size=(batch.height // 2, batch.width // 2),
mode="bilinear",
align_corners=False,
)
with self.use_declared_component(
component_name="sigvq", module=self.sigvq
) as sigvq:
assert sigvq is not None
sigvq_device, sigvq_dtype = _module_device_dtype(sigvq)
semantic_features = sigvq(
sigvq_pixels.to(device=sigvq_device, dtype=sigvq_dtype)
)
with self.use_declared_component(component_name="vae", module=self.vae) as vae:
assert vae is not None
vae_device, vae_dtype = _module_device_dtype(vae)
posterior = vae.encode(image.to(device=vae_device, dtype=vae_dtype))
source_latents = _patchify_latents(posterior.mode())
mean, std = flux2_vae_bn_stats(vae, source_latents)
source_latents = (source_latents - mean) / std
batch.image_embeds = list(semantic_features.unbind(dim=0))
batch.source_latents = [
latent.unsqueeze(1) for latent in source_latents.unbind(dim=0)
]
return batch
@@ -98,7 +98,13 @@ class TimestepPreparationStage(PipelineStage):
sigmas = batch.sigmas
n_tokens = batch.n_tokens
sigmas = server_args.pipeline_config.prepare_sigmas(sigmas, num_inference_steps)
use_uniform_sigmas = getattr(
getattr(scheduler, "config", None), "use_uniform_sigmas", False
)
if sigmas is not None or not use_uniform_sigmas:
sigmas = server_args.pipeline_config.prepare_sigmas(
sigmas, num_inference_steps
)
batch.sigmas = sigmas
# Prepare extra kwargs for set_timesteps
@@ -562,6 +562,50 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
),
]
_LLADA_IMAGE_SERVER_ARGS = DiffusionServerArgs(
model_path="inclusionAI/LLaDA-Image",
modality="image",
num_gpus=1,
tp_size=1,
ulysses_degree=1,
ring_degree=1,
cfg_parallel=False,
extras=[
"--trust-remote-code",
"--revision",
"e4e2703f410f7ddb6ee8d6b09dac6a8ec5093039",
],
)
_LLADA_IMAGE_SAMPLING = {"num_inference_steps": 50, "guidance_scale": 5.0, "seed": 42}
if not current_platform.is_hip() and not current_platform.is_xpu():
ONE_GPU_CASES += [
DiffusionTestCase(
"llada_image_bf16_t2i",
_LLADA_IMAGE_SERVER_ARGS,
DiffusionSamplingParams(
prompt="A red fox sitting on a snowy hill",
output_size="512x512",
extras=_LLADA_IMAGE_SAMPLING,
),
run_perf_check=False,
run_consistency_check=False,
run_component_accuracy_check=False,
),
DiffusionTestCase(
"llada_image_bf16_edit",
_LLADA_IMAGE_SERVER_ARGS,
DiffusionSamplingParams(
prompt="Convert 2D style to 3D style",
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
output_size="512x512",
extras=_LLADA_IMAGE_SAMPLING,
),
run_perf_check=False,
run_consistency_check=False,
run_component_accuracy_check=False,
),
]
# Skip hunyuan3d on AMD: marching_cubes surface extraction produces invalid SDF on ROCm.
if not current_platform.is_hip():
ONE_GPU_CASES.append(
@@ -3578,6 +3578,152 @@
"runtime_peak_allocated_mb": 25745.0,
"estimated_full_test_time_s": 404.0,
"warmup_peak_vram_mb": 14598.0
},
"llada_image_bf16_t2i": {
"stages_ms": {
"InputValidationStage": 0.07,
"LLaDAImageTextConditioningStage": 36.79,
"LLaDAImageSourceImageConditioningStage": 0.05,
"TimestepPreparationStage": 1.94,
"LLaDAImageLatentPreparationStage": 0.2,
"DenoisingStage": 4052.86,
"DecodingStage": 40.77
},
"denoise_step_ms": {
"0": 77.02,
"1": 78.74,
"2": 83.82,
"3": 81.04,
"4": 80.83,
"5": 81.33,
"6": 80.96,
"7": 81.2,
"8": 80.97,
"9": 81.26,
"10": 80.74,
"11": 81.22,
"12": 80.93,
"13": 81.19,
"14": 80.91,
"15": 81.32,
"16": 80.96,
"17": 82.78,
"18": 81.03,
"19": 81.29,
"20": 83.72,
"21": 81.44,
"22": 80.94,
"23": 81.4,
"24": 80.98,
"25": 81.26,
"26": 80.87,
"27": 81.42,
"28": 81.16,
"29": 81.41,
"30": 81.13,
"31": 81.58,
"32": 81.1,
"33": 81.47,
"34": 81.13,
"35": 81.34,
"36": 81.12,
"37": 81.37,
"38": 81.12,
"39": 81.42,
"40": 81.09,
"41": 81.6,
"42": 92.45,
"43": 83.16,
"44": 81.2,
"45": 81.45,
"46": 81.32,
"47": 81.65,
"48": 81.22,
"49": 81.53
},
"expected_e2e_ms": 4214.9,
"expected_load_ms": 51491.39,
"expected_avg_denoise_ms": 80.99,
"expected_median_denoise_ms": 81.06,
"load_peak_vram_mb": 48046.0,
"runtime_peak_vram_mb": 50028.0,
"load_peak_allocated_mb": 47799.72,
"runtime_peak_allocated_mb": 48224.27,
"estimated_full_test_time_s": 76.8,
"warmup_peak_vram_mb": 50028.0
},
"llada_image_bf16_edit": {
"stages_ms": {
"InputValidationStage": 10.58,
"LLaDAImageTextConditioningStage": 37.47,
"LLaDAImageSourceImageConditioningStage": 70.63,
"TimestepPreparationStage": 1.28,
"LLaDAImageLatentPreparationStage": 0.21,
"DenoisingStage": 8880.41,
"DecodingStage": 55.64
},
"denoise_step_ms": {
"0": 156.23,
"1": 177.2,
"2": 177.21,
"3": 177.7,
"4": 177.58,
"5": 179.87,
"6": 178.99,
"7": 180.97,
"8": 177.51,
"9": 177.87,
"10": 177.94,
"11": 178.05,
"12": 184.78,
"13": 178.68,
"14": 178.51,
"15": 178.74,
"16": 177.99,
"17": 179.74,
"18": 179.31,
"19": 179.54,
"20": 178.13,
"21": 178.0,
"22": 179.02,
"23": 178.76,
"24": 180.81,
"25": 179.14,
"26": 178.3,
"27": 178.71,
"28": 178.18,
"29": 178.26,
"30": 179.02,
"31": 179.39,
"32": 180.16,
"33": 179.42,
"34": 179.84,
"35": 179.17,
"36": 179.35,
"37": 178.6,
"38": 179.37,
"39": 179.67,
"40": 178.92,
"41": 179.36,
"42": 179.29,
"43": 179.02,
"44": 179.62,
"45": 178.28,
"46": 179.62,
"47": 178.49,
"48": 178.7,
"49": 180.12
},
"expected_e2e_ms": 9060.3,
"expected_load_ms": 48754.68,
"expected_avg_denoise_ms": 177.56,
"expected_median_denoise_ms": 177.89,
"load_peak_vram_mb": 48046.0,
"runtime_peak_vram_mb": 50028.0,
"load_peak_allocated_mb": 47799.72,
"runtime_peak_allocated_mb": 48227.33,
"estimated_full_test_time_s": 78.9,
"warmup_peak_vram_mb": 50028.0
}
}
}
@@ -0,0 +1,69 @@
# SPDX-License-Identifier: Apache-2.0
import math
import unittest
from sglang.multimodal_gen.configs.models.dits.llada_image import (
LLaDAImageArchConfig,
editing_rope_rows,
)
from sglang.multimodal_gen.configs.pipeline_configs.llada_image import (
LLaDAImagePipelineConfig,
)
from sglang.multimodal_gen.configs.sample.llada_image import (
LLaDAImageSamplingParams,
LLaDAImageTurboSamplingParams,
)
from sglang.multimodal_gen.registry import _get_config_info
class TestLLaDAImageConfig(unittest.TestCase):
def test_base_and_turbo_checkpoints_get_their_own_defaults(self):
"""A default Base request must not run the 4-step Turbo recipe."""
_get_config_info.cache_clear()
cases = {
"inclusionAI/LLaDA-Image": (LLaDAImageSamplingParams, 50, 5.0),
"inclusionAI/LLaDA-Image-Turbo": (LLaDAImageTurboSamplingParams, 4, 1.0),
"inclusionAI/LLaDA-Image-Turbo-FP8": (
LLaDAImageTurboSamplingParams,
4,
1.0,
),
}
for model_id, (sampling_cls, steps, guidance) in cases.items():
with self.subTest(model_id=model_id):
info = _get_config_info(model_id)
self.assertIs(info.sampling_param_cls, sampling_cls)
params = info.sampling_param_cls()
self.assertEqual(params.num_inference_steps, steps)
self.assertEqual(params.guidance_scale, guidance)
def test_editing_size_limit_follows_sequence_rope_table(self):
"""Editing sizes past the sequence RoPE table must be caught on the host."""
# A short prompt of 20 tokens plus the 256 conditioning query tokens.
caption_tokens = 20 + 256
limit = LLaDAImageArchConfig().axes_lens[0]
def rows(side: int) -> int:
image_tokens = (side // 16) ** 2
sigvq_tokens = math.ceil(side / 32) ** 2
return editing_rope_rows(
caption_tokens, [image_tokens, image_tokens], sigvq_tokens
)
self.assertLessEqual(rows(1904), limit)
self.assertGreater(rows(1920), limit)
def test_unservable_editing_sizes_are_rejected_at_validation(self):
"""Edits off the SigVQ grid or past the RoPE table fail before any GPU work."""
config = LLaDAImagePipelineConfig()
config.validate_output_size(1008, 1008, editing=False)
config.validate_output_size(1024, 1024, editing=True)
with self.assertRaisesRegex(ValueError, "divisible by 32"):
config.validate_output_size(1008, 1008, editing=True)
with self.assertRaisesRegex(ValueError, "sequence positions"):
config.validate_output_size(2048, 2048, editing=True)
if __name__ == "__main__":
unittest.main()
@@ -2,6 +2,7 @@ import inspect
import os
from dataclasses import fields
import pytest
from fastapi import HTTPException
from PIL import Image
@@ -15,6 +16,7 @@ from sglang.multimodal_gen.configs.sample.longcat_image import (
LongCatImageSamplingParams,
)
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.entrypoints.openai import image_api
from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import (
_build_image_response_kwargs,
_fallback_image_urls,
@@ -290,3 +292,16 @@ def test_select_image_variant_path_falls_back_to_single_file_path():
assert _select_image_variant_path(item, None) == "only.png"
assert _select_image_variant_path(item, "0") == "only.png"
def test_request_validation_errors_are_client_errors(monkeypatch):
"""A request that fails sampling-parameter validation returns 400, not 500."""
def reject(request_id, **kwargs):
raise ValueError("unsupported size")
monkeypatch.setattr(image_api, "build_sampling_params", reject)
with pytest.raises(HTTPException) as error:
image_api._build_request_sampling_params("request", size="7x7")
assert error.value.status_code == 400
assert "unsupported size" in error.value.detail
+1
View File
@@ -80,6 +80,7 @@ ATTENTION_BACKEND_CHOICES = [
"fa3",
"fa4",
"flashinfer",
"llada2_cfg_flashinfer",
"flashmla",
"trtllm_mla",
"cutedsl_mla",
@@ -77,6 +77,20 @@ def create_flashinfer_backend(runner):
return FlashInferMLAAttnBackend(runner)
@register_attention_backend("llada2_cfg_flashinfer")
def create_llada2_cfg_flashinfer_backend(runner):
if runner.use_mla_backend:
raise ValueError("LLaDA2 CFG attention does not use an MLA backend")
from sglang.srt.layers.attention.llada2_cfg_flashinfer_backend import (
LLaDA2CFGFlashInferAttnBackend,
)
return LLaDA2CFGFlashInferAttnBackend(
runner, init_new_workspace=runner.init_new_workspace
)
@register_attention_backend("trtllm_mla")
def create_trtllm_mla_backend(runner):
if not runner.use_mla_backend:
@@ -0,0 +1,135 @@
# SPDX-License-Identifier: Apache-2.0
"""FlashInfer backend for LLaDA-Image conditioning masks."""
from __future__ import annotations
import torch
from flashinfer.prefill import BatchPrefillWithRaggedKVCacheWrapper
from sglang.srt.layers.attention.flashinfer_backend import (
FlashInferAttnBackend,
PrefillMetadata,
)
def _build_llada_image_custom_mask(
text_lens: list[int],
sequence_lens: list[int],
device: torch.device | str,
) -> torch.Tensor:
"""Build the official one-pass text/query block mask in ragged layout."""
if len(text_lens) != len(sequence_lens):
raise RuntimeError("LLaDA-Image conditioning metadata batch mismatch")
flattened_masks: list[torch.Tensor] = []
for text_len, sequence_len in zip(text_lens, sequence_lens):
text_len = int(text_len)
sequence_len = int(sequence_len)
if text_len <= 0 or text_len >= sequence_len:
raise RuntimeError(
"LLaDA-Image conditioning text length must leave query tokens"
)
attention_mask = torch.ones(
(sequence_len, sequence_len), dtype=torch.bool, device=device
)
attention_mask[:text_len, text_len:] = False
flattened_masks.append(attention_mask.flatten())
return torch.cat(flattened_masks)
class LLaDA2CFGFlashInferAttnBackend(FlashInferAttnBackend):
"""Stock FlashInfer plus LLaDA-specific ragged attention masks."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._llada_image_conditioning_mask_active = False
self._cfg_prefill_wrapper_ragged = BatchPrefillWithRaggedKVCacheWrapper(
self.workspace_buffer, "NHD", backend="fa2"
)
@property
def conditioning_mask_active(self) -> bool:
return self._llada_image_conditioning_mask_active
def init_forward_metadata(self, forward_batch):
self._llada_image_conditioning_mask_active = False
text_lens = forward_batch.llada_image_conditioning_text_lens_cpu
if text_lens is None:
return super().init_forward_metadata(forward_batch)
if not forward_batch.forward_mode.is_extend_without_speculative():
raise RuntimeError(
"LLaDA-Image conditioning mask requires a one-shot extend"
)
if self.num_wrappers != 1:
raise RuntimeError(
"LLaDA-Image conditioning requires one FlashInfer wrapper"
)
seq_lens = forward_batch.seq_lens
prefix_lens = forward_batch.extend_prefix_lens
if bool(torch.any(prefix_lens != 0).item()):
raise RuntimeError(
"LLaDA-Image conditioning does not support cached prefixes"
)
custom_mask = _build_llada_image_custom_mask(
text_lens,
seq_lens.tolist(),
seq_lens.device,
)
qo_indptr = torch.zeros(
seq_lens.numel() + 1,
dtype=torch.int32,
device=seq_lens.device,
)
qo_indptr[1:] = torch.cumsum(seq_lens, dim=0)
prefill_indices_updater = self.indices_updater_prefill
self._cfg_prefill_wrapper_ragged.begin_forward(
qo_indptr,
qo_indptr,
prefill_indices_updater.num_qo_heads,
prefill_indices_updater.num_kv_heads,
prefill_indices_updater.head_dim,
custom_mask=custom_mask,
causal=False,
q_data_type=prefill_indices_updater.q_data_type,
kv_data_type=prefill_indices_updater.data_type,
non_blocking=True,
fixed_split_size=self.prefill_split_tile_size,
)
self._llada_image_conditioning_mask_active = True
self.forward_metadata = PrefillMetadata(
self.prefill_wrappers_paged,
use_ragged=True,
extend_no_prefix=True,
)
def forward_extend(
self,
q,
k,
v,
layer,
forward_batch,
save_kv_cache=True,
):
if not self._llada_image_conditioning_mask_active:
return super().forward_extend(
q, k, v, layer, forward_batch, save_kv_cache=save_kv_cache
)
if k is None or v is None:
raise RuntimeError("LLaDA ragged attention requires explicit K/V")
q_view = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim)
k_view = k.view(-1, layer.tp_k_head_num, layer.head_dim)
v_view = v.view(-1, layer.tp_v_head_num, layer.head_dim)
attention_output = self._cfg_prefill_wrapper_ragged.forward(
q_view,
k_view,
v_view,
causal=False,
sm_scale=layer.scaling,
logits_soft_cap=layer.logit_cap,
)
return attention_output.view(-1, layer.tp_q_head_num * layer.head_dim)
@@ -728,6 +728,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Encoder-decoder cross-attention mask
cross_attention_custom_mask: Optional[torch.Tensor] = None
# LLaDA-Image conditioning text lengths, declared so batch rebuilds
# carry them to the llada2_cfg_flashinfer backend
llada_image_conditioning_text_lens_cpu: Optional[List[int]] = None
# For DP attention (padding / local info)
dp_padding_mode: Optional[DpPaddingMode] = None
# for extend, local start pos and num tokens is different in logits processor
+43 -1
View File
@@ -1305,6 +1305,43 @@ class LLaDA2MoeModel(nn.Module):
return hidden_states
_EXPERT_PROJECTION_NAMES = frozenset({"gate_proj", "up_proj", "down_proj"})
def _prepare_llada2_language_weights(
weights: Iterable[Tuple[str, torch.Tensor]],
*,
num_experts: int,
) -> Iterable[Tuple[str, torch.Tensor]]:
"""Normalize language-model prefixes and expand fused expert tensors."""
for name, loaded_weight in weights:
if name.startswith("model.language_model."):
name = "model." + name.removeprefix("model.language_model.")
elif name.startswith("model.lm_head."):
name = "lm_head." + name.removeprefix("model.lm_head.")
expert_prefix, separator, projection_name = name.rpartition(".")
is_fused_expert = (
separator
and expert_prefix.endswith(".mlp.experts")
and projection_name in _EXPERT_PROJECTION_NAMES
)
if not is_fused_expert:
yield name, loaded_weight
continue
if loaded_weight.ndim != 3 or loaded_weight.shape[0] != num_experts:
raise ValueError(
f"Invalid fused expert weight {name!r}: expected first dimension "
f"{num_experts}, got shape={tuple(loaded_weight.shape)}"
)
for expert_id in range(num_experts):
yield (
f"{expert_prefix}.{expert_id}.{projection_name}.weight",
loaded_weight[expert_id],
)
class LLaDA2MoeModelLM(nn.Module):
def __init__(
self,
@@ -1346,6 +1383,9 @@ class LLaDA2MoeModelLM(nn.Module):
def end_layer(self):
return self.model.end_layer
def get_input_embeddings(self):
return self.model.word_embeddings
def get_embed_and_head(self):
"""Used by the eagle_worker."""
return self.model.word_embeddings.weight, self.lm_head.weight
@@ -1399,7 +1439,9 @@ class LLaDA2MoeModelLM(nn.Module):
)
params_dict = dict(self.named_parameters())
for name, loaded_weight in weights:
for name, loaded_weight in _prepare_llada2_language_weights(
weights, num_experts=self.config.num_experts
):
if (
("v_head" in name)
or ("inv_freq" in name)
+39 -8
View File
@@ -1505,6 +1505,37 @@ def publish(
(bags re-projected, provenance reset, role overwritten), which is what
lets one process rebuild an engine after shutting the previous one down.
"""
return _configure_context(_CONTEXT, server_args, role=role, ranks=ranks)
def create_context(
server_args, *, role: str, ranks: SpawnRanks | None = None
) -> RuntimeContext:
"""Build a model-owned context without replacing the active process config."""
context = RuntimeContext(parallel=ParallelContext())
return _configure_context(context, server_args, role=role, ranks=ranks)
@contextmanager
def use_context(context: RuntimeContext):
"""Activate a model context for serialized embedded SRT work.
Like process group scopes, this changes process globals and requires the
caller to serialize model execution. It does not isolate concurrent threads.
The previous context is restored by identity without re-projecting its bags.
"""
global _CONTEXT, _PARALLEL
previous_context, previous_parallel = _CONTEXT, _PARALLEL
_CONTEXT, _PARALLEL = context, context.parallel
try:
yield context
finally:
_CONTEXT, _PARALLEL = previous_context, previous_parallel
def _configure_context(
context, server_args, *, role: str, ranks: SpawnRanks | None = None
) -> RuntimeContext:
if _ROLE_NS_MODE == "enforce" and role not in ROLE_NAMESPACE_SETS:
# Fail closed at publish time, not at the first stray read.
raise ValueError(
@@ -1512,8 +1543,8 @@ def publish(
"its namespace set (None for the full tree)."
)
server_args.resolve_once()
discarded = _CONTEXT.overrides_log()
_CONTEXT.set_server_args(server_args)
discarded = context.overrides_log()
context.set_server_args(server_args)
if discarded:
logger.warning(
"publish(role=%s) re-projected the config bags and dropped %d "
@@ -1524,16 +1555,16 @@ def publish(
f"{source}({', '.join(sorted(fields))})" for source, fields in discarded
),
)
_CONTEXT._publish_role = role
context._publish_role = role
# Disabled DCP has rank zero even in processes without a rank bundle.
if not _CONTEXT.parallel.dcp_enabled:
_CONTEXT.parallel.override_permanently(attn_dcp_rank=0)
if not context.parallel.dcp_enabled:
context.parallel.override_permanently(attn_dcp_rank=0)
# The device is assigned by the launcher; it is not a config field.
_CONTEXT.config_bag("device")._set(
context.config_bag("device")._set(
"gpu_id", ranks.gpu_id if ranks is not None else None
)
if ranks is not None:
parallel = _CONTEXT.parallel
parallel = context.parallel
placement = derive_spawn_ranks(
world_rank=ranks.world_rank,
tp_size=parallel.tp_size,
@@ -1567,7 +1598,7 @@ def publish(
file=sys.stderr,
flush=True,
)
return _CONTEXT
return context
def _attention_ranks(parallel, tp_rank: int) -> dict: