[diffusion] Support LLaDA-Image BF16 generation and editing (#37907)
This commit is contained in:
@@ -14,6 +14,7 @@ from sglang.multimodal_gen.configs.models.dits.lingbot_video_moe import (
|
||||
from sglang.multimodal_gen.configs.models.dits.lingbot_world import (
|
||||
LingBotWorldVideoConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.dits.llada_image import LLaDAImageDitConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.longlive2 import LongLive2VideoConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.minimax_h3 import MiniMaxH3DiTConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.mova_audio import MOVAAudioConfig
|
||||
@@ -33,6 +34,7 @@ __all__ = [
|
||||
"Ideogram4DistilledDiTConfig",
|
||||
"LingBotWorldVideoConfig",
|
||||
"LingBotVideoMoEConfig",
|
||||
"LLaDAImageDitConfig",
|
||||
"LongLive2VideoConfig",
|
||||
"MiniMaxH3DiTConfig",
|
||||
"WanAnimate2Config",
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
|
||||
SEQUENCE_MULTIPLE = 32
|
||||
|
||||
|
||||
def editing_rope_rows(
|
||||
caption_tokens: int, image_tokens: list[int], sigvq_tokens: int
|
||||
) -> int:
|
||||
# Must match LLaDAImageTransformer2DModel._prepare_editing_sequences: two
|
||||
# padded caption copies and the padded image grids precede the SigVQ tokens.
|
||||
def padded(length: int) -> int:
|
||||
return length + (-length) % SEQUENCE_MULTIPLE
|
||||
|
||||
return (
|
||||
1
|
||||
+ 2 * padded(caption_tokens)
|
||||
+ sum(padded(tokens) for tokens in image_tokens)
|
||||
+ sigvq_tokens
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LLaDAImageArchConfig(DiTArchConfig):
|
||||
all_patch_size: tuple[int, ...] = (1,)
|
||||
all_f_patch_size: tuple[int, ...] = (1,)
|
||||
axes_dims: tuple[int, ...] = (32, 48, 48)
|
||||
axes_lens: tuple[int, ...] = (32768, 1024, 1024)
|
||||
cap_feat_dim: int = 2560
|
||||
dim: int = 3840
|
||||
in_channels: int = 128
|
||||
n_heads: int = 30
|
||||
n_layers: int = 30
|
||||
n_refiner_layers: int = 2
|
||||
norm_eps: float = 1e-5
|
||||
qk_norm: bool = True
|
||||
rope_theta: float = 256.0
|
||||
semantic_feat_dim: int = 4096
|
||||
t_scale: float = 1000.0
|
||||
_supported_attention_backends: set[AttentionBackendEnum] = field(
|
||||
default_factory=lambda: {
|
||||
AttentionBackendEnum.FA,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
}
|
||||
)
|
||||
param_names_mapping: dict = field(
|
||||
default_factory=lambda: {
|
||||
r"(.*)\.attention\.to_q\.weight$": (r"\1.attention.to_qkv.weight", 0, 3),
|
||||
r"(.*)\.attention\.to_k\.weight$": (r"\1.attention.to_qkv.weight", 1, 3),
|
||||
r"(.*)\.attention\.to_v\.weight$": (r"\1.attention.to_qkv.weight", 2, 3),
|
||||
r"(.*)\.feed_forward\.w1\.weight$": (r"\1.feed_forward.w13.weight", 0, 2),
|
||||
r"(.*)\.feed_forward\.w3\.weight$": (r"\1.feed_forward.w13.weight", 1, 2),
|
||||
}
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
super().__post_init__()
|
||||
self.hidden_size = self.dim
|
||||
self.num_attention_heads = self.n_heads
|
||||
self.num_channels_latents = self.in_channels
|
||||
|
||||
|
||||
@dataclass
|
||||
class LLaDAImageDitConfig(DiTConfig):
|
||||
arch_config: LLaDAImageArchConfig = field(default_factory=LLaDAImageArchConfig)
|
||||
prefix: str = "llada_image"
|
||||
@@ -42,6 +42,9 @@ from sglang.multimodal_gen.configs.pipeline_configs.lingbot_world import (
|
||||
LingBotWorldCausalDMDConfig,
|
||||
LingBotWorldV2CausalDMDConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.llada_image import (
|
||||
LLaDAImagePipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import (
|
||||
LTX2PipelineConfig,
|
||||
LTX23PipelineConfig,
|
||||
@@ -83,6 +86,7 @@ __all__ = [
|
||||
"HunyuanImage3PipelineConfig",
|
||||
"Ideogram4PipelineConfig",
|
||||
"Ideogram4DistilledPipelineConfig",
|
||||
"LLaDAImagePipelineConfig",
|
||||
"FluxPipelineConfig",
|
||||
"Flux2PipelineConfig",
|
||||
"Flux2KleinPipelineConfig",
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.configs.models.dits.llada_image import (
|
||||
LLaDAImageDitConfig,
|
||||
editing_rope_rows,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ModelTaskType,
|
||||
SpatialImagePipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux import _unpatchify_latents
|
||||
|
||||
|
||||
def flux2_vae_bn_stats(vae, like: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
mean = vae.bn.running_mean.view(1, -1, 1, 1).to(like)
|
||||
variance = vae.bn.running_var.view(1, -1, 1, 1)
|
||||
std = torch.sqrt(variance + vae.config.arch_config.batch_norm_eps).to(like)
|
||||
return mean, std
|
||||
|
||||
|
||||
@dataclass
|
||||
class LLaDAImagePipelineConfig(SpatialImagePipelineConfig):
|
||||
task_type: ModelTaskType = ModelTaskType.TI2I
|
||||
should_use_guidance: bool = False
|
||||
|
||||
dit_config: LLaDAImageDitConfig = field(default_factory=LLaDAImageDitConfig)
|
||||
vae_config: Flux2VAEConfig = field(default_factory=Flux2VAEConfig)
|
||||
vae_precision: str = "bf16"
|
||||
vae_tiling: bool = False
|
||||
vae_sp: bool = False
|
||||
|
||||
text_encoder_configs: tuple = ()
|
||||
text_encoder_precisions: tuple[str, ...] = ()
|
||||
preprocess_text_funcs: tuple = ()
|
||||
postprocess_text_funcs: tuple = ()
|
||||
|
||||
latent_scale_factor: int = 16
|
||||
# Editing feeds the source to SigVQ at half size, and its patch is 16 pixels.
|
||||
editing_size_multiple: int = 32
|
||||
# The embedded text worker admits 8192 prefill tokens shared by the two
|
||||
# CFG sequences and each sequence appends 256 query tokens.
|
||||
max_request_text_tokens: int = 3584
|
||||
|
||||
def prepare_sigmas(self, sigmas, num_inference_steps):
|
||||
if sigmas is not None:
|
||||
return sigmas
|
||||
schedule = np.linspace(0.001, 1.0, num_inference_steps + 1)[:-1]
|
||||
schedule = (1 - (1 - schedule**1.17) ** 0.8) ** 1.1
|
||||
return (1 - schedule).tolist()
|
||||
|
||||
def prepare_latent_shape(self, batch, batch_size, num_frames):
|
||||
del num_frames
|
||||
self.validate_output_size(
|
||||
batch.width, batch.height, editing=batch.condition_image is not None
|
||||
)
|
||||
return (
|
||||
batch_size,
|
||||
self.dit_config.num_channels_latents,
|
||||
batch.height // self.latent_scale_factor,
|
||||
batch.width // self.latent_scale_factor,
|
||||
)
|
||||
|
||||
def prepare_calculated_size(self, image):
|
||||
del image
|
||||
return None
|
||||
|
||||
def calculate_condition_image_size(self, image, width, height):
|
||||
del image, width, height
|
||||
return None
|
||||
|
||||
def validate_server_args(self, server_args) -> None:
|
||||
super().validate_server_args(server_args)
|
||||
if any(
|
||||
value != 1
|
||||
for value in (
|
||||
server_args.num_gpus,
|
||||
server_args.sp_degree,
|
||||
server_args.tp_size,
|
||||
server_args.dp_size,
|
||||
server_args.cfg_parallel_degree,
|
||||
)
|
||||
):
|
||||
raise ValueError("LLaDA-Image currently supports single-GPU serving")
|
||||
if server_args.quantization is not None:
|
||||
raise ValueError("LLaDA-Image currently supports BF16 checkpoints")
|
||||
if envs.SGLANG_CACHE_DIT_ENABLED:
|
||||
raise ValueError("LLaDA-Image does not support cache-dit")
|
||||
if server_args.text_encoder_cpu_offload:
|
||||
if server_args.is_arg_explicitly_set("text_encoder_cpu_offload"):
|
||||
raise ValueError("LLaDA-Image requires a resident text encoder")
|
||||
server_args.text_encoder_cpu_offload = False
|
||||
for attribute in ("layerwise_offload_components", "cpu_offload_components"):
|
||||
components = getattr(server_args, attribute)
|
||||
if components is not None and "text_encoder" in components:
|
||||
if server_args.is_arg_explicitly_set(attribute):
|
||||
raise ValueError("LLaDA-Image requires a resident text encoder")
|
||||
setattr(
|
||||
server_args,
|
||||
attribute,
|
||||
[name for name in components if name != "text_encoder"],
|
||||
)
|
||||
if server_args.explicit_residency_mode("text_encoder") not in (
|
||||
None,
|
||||
"resident",
|
||||
):
|
||||
raise ValueError("LLaDA-Image requires a resident text encoder")
|
||||
|
||||
def supports_disaggregation(self) -> bool:
|
||||
return False
|
||||
|
||||
def validate_output_size(self, width: int, height: int, editing: bool) -> None:
|
||||
multiple = self.editing_size_multiple if editing else self.latent_scale_factor
|
||||
if width % multiple != 0 or height % multiple != 0:
|
||||
task = "editing" if editing else "generation"
|
||||
raise ValueError(
|
||||
f"LLaDA-Image {task} width and height must be divisible by "
|
||||
f"{multiple}, got {width}x{height}"
|
||||
)
|
||||
self._validate_spatial_rope_bounds(width, height)
|
||||
if editing:
|
||||
# Lower bound without the prompt, the DiT checks the exact length.
|
||||
image_tokens = (width // self.latent_scale_factor) * (
|
||||
height // self.latent_scale_factor
|
||||
)
|
||||
sigvq_tokens = (width // self.editing_size_multiple) * (
|
||||
height // self.editing_size_multiple
|
||||
)
|
||||
rows = editing_rope_rows(0, [image_tokens, image_tokens], sigvq_tokens)
|
||||
limit = self.dit_config.arch_config.axes_lens[0]
|
||||
if rows > limit:
|
||||
raise ValueError(
|
||||
f"LLaDA-Image editing at {width}x{height} needs at least {rows} "
|
||||
f"sequence positions, above the model limit of {limit}"
|
||||
)
|
||||
|
||||
def _validate_spatial_rope_bounds(self, width: int, height: int) -> None:
|
||||
# The served DiT uses patch size 1.
|
||||
axes_lens = self.dit_config.arch_config.axes_lens
|
||||
max_height = axes_lens[1] * self.latent_scale_factor
|
||||
max_width = axes_lens[2] * self.latent_scale_factor
|
||||
if height > max_height or width > max_width:
|
||||
raise ValueError(
|
||||
f"LLaDA-Image output size {width}x{height} exceeds spatial RoPE "
|
||||
f"bounds of {max_width}x{max_height} pixels"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _cond_kwargs(batch, device, dtype, negative: bool) -> dict:
|
||||
def to_device(values):
|
||||
if values is None:
|
||||
return None
|
||||
return [value.to(device=device, dtype=dtype) for value in values]
|
||||
|
||||
image_embeds = batch.image_embeds
|
||||
if image_embeds:
|
||||
image_embeds = to_device(image_embeds)
|
||||
if negative:
|
||||
# The unconditional edit keeps the source latents without SigVQ.
|
||||
empty = image_embeds[0].new_zeros((0, image_embeds[0].shape[-1]))
|
||||
image_embeds = [empty] * batch.batch_size
|
||||
return {
|
||||
"encoder_hidden_states_image": image_embeds,
|
||||
"source_latents": to_device(batch.source_latents),
|
||||
}
|
||||
|
||||
def prepare_pos_cond_kwargs(self, batch, device, rotary_emb, dtype):
|
||||
return self._cond_kwargs(batch, device, dtype, negative=False)
|
||||
|
||||
def prepare_neg_cond_kwargs(self, batch, device, rotary_emb, dtype):
|
||||
return self._cond_kwargs(batch, device, dtype, negative=True)
|
||||
|
||||
def get_decode_scale_and_shift(self, device, dtype, vae):
|
||||
del device, dtype, vae
|
||||
return 1.0, None
|
||||
|
||||
def preprocess_decoding(self, latents, server_args=None, vae=None):
|
||||
vae_parameter = next(vae.parameters())
|
||||
latents = latents.to(device=vae_parameter.device, dtype=vae_parameter.dtype)
|
||||
mean, std = flux2_vae_bn_stats(vae, latents)
|
||||
return _unpatchify_latents(latents * std + mean)
|
||||
|
||||
def post_denoising_loop(self, latents, batch):
|
||||
del batch
|
||||
return latents
|
||||
|
||||
|
||||
def register():
|
||||
from sglang.multimodal_gen.configs.sample.llada_image import (
|
||||
LLaDAImageSamplingParams,
|
||||
LLaDAImageTurboSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.registry import register_configs
|
||||
|
||||
# Base and Turbo checkpoints share every class name, so Turbo registers
|
||||
# first and the first matching detector wins.
|
||||
register_configs(
|
||||
sampling_param_cls=LLaDAImageTurboSamplingParams,
|
||||
pipeline_config_cls=LLaDAImagePipelineConfig,
|
||||
hf_model_paths=["inclusionAI/LLaDA-Image-Turbo"],
|
||||
model_detectors=[
|
||||
lambda hf_id: "llada-image" in hf_id.lower() and "turbo" in hf_id.lower()
|
||||
],
|
||||
)
|
||||
register_configs(
|
||||
sampling_param_cls=LLaDAImageSamplingParams,
|
||||
pipeline_config_cls=LLaDAImagePipelineConfig,
|
||||
hf_model_paths=["inclusionAI/LLaDA-Image"],
|
||||
model_detectors=[
|
||||
lambda hf_id: (
|
||||
(
|
||||
"lladaimagepipeline" in hf_id.lower()
|
||||
or "llada-image" in hf_id.lower()
|
||||
)
|
||||
and "turbo" not in hf_id.lower()
|
||||
)
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
DataType,
|
||||
SamplingParams,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LLaDAImageSamplingParams(SamplingParams):
|
||||
data_type: DataType = DataType.IMAGE
|
||||
negative_prompt: str | None = None
|
||||
# Base checkpoint recipe from the model card.
|
||||
num_inference_steps: int = 50
|
||||
num_frames: int = 1
|
||||
height: int = 1024
|
||||
width: int = 1024
|
||||
guidance_scale: float = 5.0
|
||||
max_sequence_length: int = 2048
|
||||
|
||||
def _validate_with_pipeline_config(self, pipeline_config):
|
||||
super()._validate_with_pipeline_config(pipeline_config)
|
||||
if self.enable_cache_dit or self.cache_dit_params is not None:
|
||||
raise ValueError("LLaDA-Image does not support cache-dit")
|
||||
if self.max_sequence_length is not None and not (
|
||||
0 < self.max_sequence_length <= pipeline_config.max_request_text_tokens
|
||||
):
|
||||
raise ValueError(
|
||||
"LLaDA-Image max_sequence_length must be between 1 and 3584"
|
||||
)
|
||||
pipeline_config.validate_output_size(
|
||||
self.width, self.height, editing=bool(self.image_path)
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LLaDAImageTurboSamplingParams(LLaDAImageSamplingParams):
|
||||
# The distilled Turbo checkpoint samples in 4 steps without CFG.
|
||||
num_inference_steps: int = 4
|
||||
guidance_scale: float = 1.0
|
||||
@@ -248,6 +248,14 @@ def _get_response_resize(
|
||||
return f"{width}x{height}"
|
||||
|
||||
|
||||
def _build_request_sampling_params(request_id: str, **kwargs):
|
||||
# Request validation errors are client errors, as in the video API.
|
||||
try:
|
||||
return build_sampling_params(request_id, **kwargs)
|
||||
except (TypeError, ValueError) as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
|
||||
|
||||
@router.post("/generations", response_model=ImageResponse)
|
||||
async def generations(
|
||||
request: ImageGenerationsRequest,
|
||||
@@ -266,7 +274,7 @@ async def generations(
|
||||
)
|
||||
|
||||
with temp_dir_if_disabled(server_args.output_path) as output_dir:
|
||||
sampling = build_sampling_params(
|
||||
sampling = _build_request_sampling_params(
|
||||
request_id,
|
||||
prompt=prompt,
|
||||
task_type=request.task_type,
|
||||
@@ -465,7 +473,7 @@ async def edits(
|
||||
task="image_edit",
|
||||
image_paths=input_paths,
|
||||
)
|
||||
sampling = build_sampling_params(
|
||||
sampling = _build_request_sampling_params(
|
||||
request_id,
|
||||
prompt=prompt,
|
||||
task_type=task_type,
|
||||
|
||||
@@ -26,6 +26,10 @@ from sglang.kernels.ops.layernorm.norm import (
|
||||
can_use_fused_inplace_qknorm,
|
||||
fused_inplace_qknorm,
|
||||
)
|
||||
from sglang.kernels.ops.layernorm.rmsnorm_hf import (
|
||||
is_supported_rmsnorm_hf_hidden_size,
|
||||
rmsnorm_hf,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -102,11 +106,13 @@ class RMSNorm(CustomOp):
|
||||
eps: float = 1e-6,
|
||||
dtype: torch.dtype = torch.float32,
|
||||
var_hidden_size: Optional[int] = None,
|
||||
cast_x_before_out_mul: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.variance_epsilon = eps
|
||||
self.hidden_size = hidden_size
|
||||
self.cast_x_before_out_mul = cast_x_before_out_mul
|
||||
self.variance_size_override = (
|
||||
None if var_hidden_size == hidden_size else var_hidden_size
|
||||
)
|
||||
@@ -126,6 +132,22 @@ class RMSNorm(CustomOp):
|
||||
residual: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
shape = x.shape
|
||||
if self.cast_x_before_out_mul:
|
||||
if residual is not None or self.variance_size_override is not None:
|
||||
return self.forward_native(x, residual)
|
||||
x_2d = x.contiguous().reshape(-1, shape[-1])
|
||||
if (
|
||||
x_2d.dtype in (torch.float16, torch.bfloat16)
|
||||
and self.weight.dtype == x_2d.dtype
|
||||
and is_supported_rmsnorm_hf_hidden_size(x_2d.shape[-1])
|
||||
):
|
||||
return rmsnorm_hf(
|
||||
x_2d,
|
||||
self.weight.data,
|
||||
self.variance_epsilon,
|
||||
).view(shape)
|
||||
return self.forward_native(x)
|
||||
|
||||
x = x.reshape(-1, shape[-1])
|
||||
if residual is not None:
|
||||
residual_shape = residual.shape
|
||||
@@ -200,10 +222,13 @@ class RMSNorm(CustomOp):
|
||||
|
||||
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
|
||||
x = x * torch.rsqrt(variance + self.variance_epsilon)
|
||||
weight = self.weight
|
||||
if x.device.type == "mps" and weight.dtype != x.dtype:
|
||||
weight = weight.to(dtype=x.dtype)
|
||||
x = (x * weight).to(orig_dtype)
|
||||
if self.cast_x_before_out_mul:
|
||||
x = self.weight * x.to(orig_dtype)
|
||||
else:
|
||||
weight = self.weight
|
||||
if x.device.type == "mps" and weight.dtype != x.dtype:
|
||||
weight = weight.to(dtype=x.dtype)
|
||||
x = (x * weight).to(orig_dtype)
|
||||
if residual is None:
|
||||
return x
|
||||
else:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+24
-12
@@ -29,6 +29,7 @@ import torch
|
||||
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
||||
from diffusers.schedulers.scheduling_utils import SchedulerMixin
|
||||
from diffusers.utils import BaseOutput
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import (
|
||||
@@ -92,6 +93,9 @@ class FlowMatchEulerDiscreteScheduler(
|
||||
Whether to use exponential sigmas for step sizes in the noise schedule during sampling.
|
||||
use_beta_sigmas (`bool`, defaults to False):
|
||||
Whether to use beta sigmas for step sizes in the noise schedule during sampling.
|
||||
use_uniform_sigmas (`bool`, defaults to False):
|
||||
Whether to use a uniform pre-shift sigma grid that excludes the terminal zero. The terminal zero is
|
||||
appended after shifting.
|
||||
time_shift_type (`str`, defaults to "exponential"):
|
||||
The type of dynamic resolution-dependent timestep shifting to apply. Either "exponential" or "linear".
|
||||
stochastic_sampling (`bool`, defaults to False):
|
||||
@@ -116,6 +120,7 @@ class FlowMatchEulerDiscreteScheduler(
|
||||
use_karras_sigmas: bool | None = False,
|
||||
use_exponential_sigmas: bool | None = False,
|
||||
use_beta_sigmas: bool | None = False,
|
||||
use_uniform_sigmas: bool | None = False,
|
||||
time_shift_type: str = "exponential",
|
||||
stochastic_sampling: bool = False,
|
||||
):
|
||||
@@ -125,12 +130,14 @@ class FlowMatchEulerDiscreteScheduler(
|
||||
self.config.use_beta_sigmas,
|
||||
self.config.use_exponential_sigmas,
|
||||
self.config.use_karras_sigmas,
|
||||
self.config.use_uniform_sigmas,
|
||||
]
|
||||
)
|
||||
> 1
|
||||
):
|
||||
raise ValueError(
|
||||
"Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, `config.use_karras_sigmas` can be used."
|
||||
"Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, "
|
||||
"`config.use_karras_sigmas`, `config.use_uniform_sigmas` can be used."
|
||||
)
|
||||
if time_shift_type not in {"exponential", "linear"}:
|
||||
raise ValueError(
|
||||
@@ -341,13 +348,18 @@ class FlowMatchEulerDiscreteScheduler(
|
||||
|
||||
sigmas_array: np.ndarray
|
||||
if sigmas is None:
|
||||
if timesteps_array is None:
|
||||
timesteps_array = np.linspace(
|
||||
self._sigma_to_t(self.sigma_max),
|
||||
self._sigma_to_t(self.sigma_min),
|
||||
num_inference_steps,
|
||||
)
|
||||
sigmas_array = timesteps_array / self.config.num_train_timesteps
|
||||
if timesteps_array is None and self.config.use_uniform_sigmas:
|
||||
sigmas_array = np.linspace(
|
||||
1.0, 0.0, num_inference_steps + 1, dtype=np.float32
|
||||
)[:-1]
|
||||
else:
|
||||
if timesteps_array is None:
|
||||
timesteps_array = np.linspace(
|
||||
self._sigma_to_t(self.sigma_max),
|
||||
self._sigma_to_t(self.sigma_min),
|
||||
num_inference_steps,
|
||||
)
|
||||
sigmas_array = timesteps_array / self.config.num_train_timesteps
|
||||
else:
|
||||
sigmas_array = np.array(sigmas).astype(np.float32)
|
||||
num_inference_steps = len(sigmas_array)
|
||||
@@ -451,7 +463,7 @@ class FlowMatchEulerDiscreteScheduler(
|
||||
s_tmin: float = 0.0,
|
||||
s_tmax: float = float("inf"),
|
||||
s_noise: float = 1.0,
|
||||
generator: torch.Generator | None = None,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
per_token_timesteps: torch.Tensor | None = None,
|
||||
batch=None,
|
||||
return_dict: bool = True,
|
||||
@@ -472,8 +484,8 @@ class FlowMatchEulerDiscreteScheduler(
|
||||
s_tmax (`float`):
|
||||
s_noise (`float`, defaults to 1.0):
|
||||
Scaling factor for noise added to the sample.
|
||||
generator (`torch.Generator`, *optional*):
|
||||
A random number generator.
|
||||
generator (`torch.Generator` or `list[torch.Generator]`, *optional*):
|
||||
A random number generator, or one generator per batch item.
|
||||
per_token_timesteps (`torch.Tensor`, *optional*):
|
||||
The timesteps for each token in the sample.
|
||||
return_dict (`bool`):
|
||||
@@ -532,7 +544,7 @@ class FlowMatchEulerDiscreteScheduler(
|
||||
else:
|
||||
if self.config.stochastic_sampling:
|
||||
x0 = sample - current_sigma * model_output
|
||||
noise = torch.randn(
|
||||
noise = randn_tensor(
|
||||
sample.shape,
|
||||
generator=generator,
|
||||
device=sample.device,
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
from diffusers.image_processor import VaeImageProcessor
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
|
||||
PlainStateDictComponentLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages import (
|
||||
DecodingStage,
|
||||
DenoisingStage,
|
||||
InputValidationStage,
|
||||
LatentPreparationStage,
|
||||
TimestepPreparationStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.llada_image.conditioning import (
|
||||
LLaDAImageTextConditioningStage,
|
||||
LLaDAImageTextEncoderRunner,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.llada_image.source import (
|
||||
LLaDAImageSourceImageConditioningStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
class LLaDAImageLatentPreparationStage(LatentPreparationStage):
|
||||
def forward(self, batch, server_args):
|
||||
if batch.latents is None:
|
||||
shape = server_args.pipeline_config.prepare_latent_shape(
|
||||
batch, batch.batch_size, batch.num_frames
|
||||
)
|
||||
model_dtype = next(self.transformer.parameters()).dtype
|
||||
batch.latents = randn_tensor(
|
||||
shape,
|
||||
generator=batch.generator,
|
||||
device=get_local_torch_device(),
|
||||
dtype=model_dtype,
|
||||
).float()
|
||||
batch = super().forward(batch, server_args)
|
||||
return batch
|
||||
|
||||
|
||||
class LLaDAImagePipeline(ComposedPipelineBase):
|
||||
pipeline_name = "LLaDAImagePipeline"
|
||||
component_loaders = {
|
||||
"queryformer": PlainStateDictComponentLoader,
|
||||
"text_projection": PlainStateDictComponentLoader,
|
||||
"sigvq": PlainStateDictComponentLoader,
|
||||
}
|
||||
|
||||
_required_config_modules: ClassVar[list[str]] = [
|
||||
"queryformer",
|
||||
"text_projection",
|
||||
"sigvq",
|
||||
"tokenizer",
|
||||
"transformer",
|
||||
"scheduler",
|
||||
"vae",
|
||||
]
|
||||
|
||||
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
|
||||
text_runner = LLaDAImageTextEncoderRunner(
|
||||
model_root=self.model_path,
|
||||
queryformer=self.get_module("queryformer"),
|
||||
text_projection=self.get_module("text_projection"),
|
||||
tokenizer=self.get_module("tokenizer"),
|
||||
server_args=server_args,
|
||||
)
|
||||
self.add_module("text_encoder", text_runner)
|
||||
|
||||
self.add_stage(InputValidationStage(), "input_validation_stage")
|
||||
self.add_stage(
|
||||
LLaDAImageTextConditioningStage(text_runner),
|
||||
"text_conditioning_stage",
|
||||
)
|
||||
self.add_stage(
|
||||
LLaDAImageSourceImageConditioningStage(
|
||||
sigvq=self.get_module("sigvq"),
|
||||
vae=self.get_module("vae"),
|
||||
image_processor=VaeImageProcessor(
|
||||
vae_scale_factor=server_args.pipeline_config.latent_scale_factor
|
||||
),
|
||||
),
|
||||
"source_image_conditioning_stage",
|
||||
)
|
||||
self.add_stage(
|
||||
TimestepPreparationStage(self.get_module("scheduler")),
|
||||
"timestep_preparation_stage",
|
||||
)
|
||||
self.add_stage(
|
||||
LLaDAImageLatentPreparationStage(
|
||||
scheduler=self.get_module("scheduler"),
|
||||
transformer=self.get_module("transformer"),
|
||||
),
|
||||
"latent_preparation_stage",
|
||||
)
|
||||
self.add_stage(
|
||||
DenoisingStage(
|
||||
transformer=self.get_module("transformer"),
|
||||
scheduler=self.get_module("scheduler"),
|
||||
vae=self.get_module("vae"),
|
||||
pipeline=self,
|
||||
),
|
||||
"denoising_stage",
|
||||
)
|
||||
self.add_stage(
|
||||
DecodingStage(
|
||||
vae=self.get_module("vae"), pipeline=self, component_name="vae"
|
||||
),
|
||||
"decoding_stage",
|
||||
)
|
||||
|
||||
|
||||
EntryClass = [LLaDAImagePipeline]
|
||||
@@ -135,6 +135,7 @@ class Req:
|
||||
|
||||
# Latent tensors
|
||||
latents: torch.Tensor | None = None
|
||||
source_latents: list[torch.Tensor] | None = None
|
||||
y: torch.Tensor | None = None
|
||||
# Flux-2
|
||||
latent_ids: torch.Tensor | None = None
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
+408
@@ -0,0 +1,408 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from array import array
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
|
||||
prepare_diffusers_component_path_for_loading,
|
||||
)
|
||||
|
||||
# srt group globals the diffusion runtime mirrors from its own TP group. srt's
|
||||
# initialize_model_parallel refuses to run while any of them is set.
|
||||
_MIRRORED_SRT_GROUPS = ("_TP", "_ATTN_TP", "_MOE_TP", "_MOE_EP")
|
||||
|
||||
|
||||
def _resolve_llada_image_component_path(
|
||||
model_root: str,
|
||||
server_args: ServerArgs,
|
||||
component_name: str,
|
||||
) -> str:
|
||||
override_path = server_args.component_paths.get(component_name)
|
||||
if override_path is not None:
|
||||
return prepare_diffusers_component_path_for_loading(override_path)
|
||||
return str(Path(model_root) / component_name)
|
||||
|
||||
|
||||
def format_llada_image_prompt(prompt: str | None) -> str:
|
||||
if prompt is None:
|
||||
return "<role>HUMAN</role> Generate an image.\n<role>ASSISTANT</role>\n<IMAGE1>"
|
||||
return (
|
||||
f"<role>HUMAN</role> Generate an image: {prompt.strip()}\n"
|
||||
"<role>ASSISTANT</role>\n<IMAGE1>"
|
||||
)
|
||||
|
||||
|
||||
class LLaDAImageTextEncoderRunner:
|
||||
"""One-shot LLaDA2 prefill runner used to produce diffusion cap features."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_root: str,
|
||||
queryformer,
|
||||
text_projection,
|
||||
tokenizer,
|
||||
server_args: ServerArgs,
|
||||
) -> None:
|
||||
import sglang.srt.distributed.parallel_state as srt_parallel_state
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.distributed.bootstrap import (
|
||||
init_layer_runtime,
|
||||
init_parallel_runtime,
|
||||
)
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.unified_cache.components import ComponentType
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
from sglang.srt.runtime_context import SpawnRanks, create_context, use_context
|
||||
from sglang.srt.server_args import ServerArgs as SRTServerArgs
|
||||
|
||||
self.queryformer = queryformer
|
||||
self.text_projection = text_projection
|
||||
self.tokenizer = tokenizer
|
||||
text_encoder_path = _resolve_llada_image_component_path(
|
||||
model_root, server_args, "text_encoder"
|
||||
)
|
||||
text_encoder_config = json.loads(
|
||||
(Path(text_encoder_path) / "config.json").read_text()
|
||||
)
|
||||
if "quantization_config" in text_encoder_config or (
|
||||
text_encoder_config.get("llada_fp8_experts", {}).get("enabled")
|
||||
):
|
||||
raise ValueError(
|
||||
"LLaDA-Image serves only the BF16 checkpoints, such as "
|
||||
"inclusionAI/LLaDA-Image, and this text encoder is quantized"
|
||||
)
|
||||
tokenizer_path = _resolve_llada_image_component_path(
|
||||
model_root, server_args, "tokenizer"
|
||||
)
|
||||
device = get_local_torch_device()
|
||||
gpu_id = device.index
|
||||
nccl_port = server_args.nccl_port or 29500
|
||||
srt_args = SRTServerArgs(
|
||||
model_path=text_encoder_path,
|
||||
tokenizer_path=tokenizer_path,
|
||||
trust_remote_code=server_args.trust_remote_code,
|
||||
revision=server_args.revision,
|
||||
skip_tokenizer_init=True,
|
||||
dtype="bfloat16",
|
||||
tp_size=1,
|
||||
attention_backend="llada2_cfg_flashinfer",
|
||||
disable_cuda_graph=True,
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
max_prefill_tokens=8192,
|
||||
max_total_tokens=8192,
|
||||
max_running_requests=2,
|
||||
)
|
||||
self.runtime_context = create_context(
|
||||
srt_args,
|
||||
role="scheduler",
|
||||
ranks=SpawnRanks(world_rank=0, gpu_id=gpu_id),
|
||||
)
|
||||
with use_context(self.runtime_context):
|
||||
# The diffusion runtime mirrors its TP group into the srt globals,
|
||||
# the MoE groups included. Detach the groups without clearing the
|
||||
# encoder context's ranks, then restore the mirror and keep the
|
||||
# encoder groups for forwards.
|
||||
saved_groups = {
|
||||
name: getattr(srt_parallel_state, name) for name in _MIRRORED_SRT_GROUPS
|
||||
}
|
||||
try:
|
||||
for name in _MIRRORED_SRT_GROUPS:
|
||||
setattr(srt_parallel_state, name, None)
|
||||
init_parallel_runtime(
|
||||
server_args=srt_args,
|
||||
device=device.type,
|
||||
dist_port=nccl_port,
|
||||
)
|
||||
init_layer_runtime(model_config=ModelConfig.from_server_args(srt_args))
|
||||
self.worker = TpModelWorker(
|
||||
server_args=srt_args,
|
||||
gpu_id=gpu_id,
|
||||
nccl_port=nccl_port,
|
||||
)
|
||||
# Run the post-construction init phases the srt scheduler drives.
|
||||
self.worker.alloc_memory_pool()
|
||||
self.worker.init_attention_backends()
|
||||
self.worker.init_cuda_graphs()
|
||||
self.encoder_groups = {
|
||||
name: getattr(srt_parallel_state, name)
|
||||
for name in _MIRRORED_SRT_GROUPS
|
||||
}
|
||||
finally:
|
||||
for name, group in saved_groups.items():
|
||||
setattr(srt_parallel_state, name, group)
|
||||
self.server_args = srt_args
|
||||
self.model_runner = self.worker.model_runner
|
||||
self.page_size = self.model_runner.page_size
|
||||
self.req_to_token_pool, self.token_to_kv_pool_allocator = (
|
||||
self.worker.get_memory_pool()
|
||||
)
|
||||
# The radix-disabled cache srt builds for a full-attention model.
|
||||
self.tree_cache = UnifiedRadixCache(
|
||||
CacheInitParams(
|
||||
disable=True,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
page_size=self.page_size,
|
||||
tree_components=(ComponentType.FULL,),
|
||||
)
|
||||
)
|
||||
|
||||
def _prepare_input_embeds(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
sequence_lengths: list[int],
|
||||
text_lengths: list[int],
|
||||
component_context,
|
||||
) -> torch.Tensor:
|
||||
embeddings = self.model_runner.model.get_input_embeddings()
|
||||
inputs = []
|
||||
offset = 0
|
||||
with component_context(
|
||||
component_name="queryformer", module=self.queryformer
|
||||
) as queryformer:
|
||||
assert queryformer is not None
|
||||
queryformer_dtype = queryformer.dtype
|
||||
for sequence_length, text_length in zip(
|
||||
sequence_lengths, text_lengths, strict=True
|
||||
):
|
||||
text_ids = input_ids[offset : offset + text_length]
|
||||
text_embeds = embeddings(text_ids)
|
||||
text_mask = torch.ones(
|
||||
(1, text_length), dtype=torch.bool, device=text_embeds.device
|
||||
)
|
||||
query_embeds = queryformer(
|
||||
text_embeds.unsqueeze(0).to(dtype=queryformer_dtype),
|
||||
text_mask,
|
||||
)
|
||||
inputs.append(
|
||||
torch.cat(
|
||||
[text_embeds, query_embeds.squeeze(0).to(text_embeds.dtype)],
|
||||
dim=0,
|
||||
)
|
||||
)
|
||||
offset += sequence_length
|
||||
return torch.cat(inputs, dim=0)
|
||||
|
||||
@torch.no_grad()
|
||||
def encode(self, prompts: list[str], max_sequence_length: int, component_context):
|
||||
import sglang.multimodal_gen.runtime.distributed.parallel_state as mm_parallel_state
|
||||
import sglang.srt.distributed.parallel_state as srt_parallel_state
|
||||
from sglang.srt.runtime_context import use_context
|
||||
|
||||
# Restore the encoder context inside the diffusion TP scope.
|
||||
with (
|
||||
mm_parallel_state.use_tensor_parallel_group(self.encoder_groups["_TP"]),
|
||||
use_context(self.runtime_context),
|
||||
):
|
||||
# The TP scope points _TP and _ATTN_TP at one group. Use every
|
||||
# encoder group the srt layers read during the forward.
|
||||
swapped = [name for name in _MIRRORED_SRT_GROUPS if name != "_TP"]
|
||||
saved_groups = {name: getattr(srt_parallel_state, name) for name in swapped}
|
||||
try:
|
||||
for name in swapped:
|
||||
setattr(srt_parallel_state, name, self.encoder_groups[name])
|
||||
return self._encode_impl(
|
||||
prompts, max_sequence_length, component_context=component_context
|
||||
)
|
||||
finally:
|
||||
for name, group in saved_groups.items():
|
||||
setattr(srt_parallel_state, name, group)
|
||||
|
||||
def _encode_impl(
|
||||
self, prompts: list[str], max_sequence_length: int, component_context
|
||||
):
|
||||
from sglang.srt.managers.overlap_utils import resolve_forward_inputs
|
||||
from sglang.srt.managers.schedule_batch import Req as SRTReq
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder
|
||||
from sglang.srt.mem_cache.common import release_kv_cache
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
ForwardBatch,
|
||||
ForwardMode,
|
||||
)
|
||||
from sglang.srt.sampling.sampling_params import (
|
||||
SamplingParams as SRTSamplingParams,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
tokenized = self.tokenizer(
|
||||
prompts,
|
||||
add_special_tokens=True,
|
||||
padding=False,
|
||||
truncation=True,
|
||||
max_length=max_sequence_length,
|
||||
).input_ids
|
||||
query_count = int(self.queryformer.config.num_queries)
|
||||
reqs = []
|
||||
text_lengths = []
|
||||
sequence_lengths = []
|
||||
for index, text_input_ids in enumerate(tokenized):
|
||||
sampling_params = SRTSamplingParams(max_new_tokens=1, temperature=0.0)
|
||||
sampling_params.normalize(None)
|
||||
sequence_ids = list(text_input_ids) + [0] * query_count
|
||||
req = SRTReq(
|
||||
rid=f"llada-image-cap-{index}",
|
||||
origin_input_text="",
|
||||
origin_input_ids=array("q", sequence_ids),
|
||||
sampling_params=sampling_params,
|
||||
vocab_size=self.worker.model_config.vocab_size,
|
||||
eos_token_ids=set(),
|
||||
return_hidden_states=True,
|
||||
dllm_config=None,
|
||||
)
|
||||
req.skip_radix_cache_insert = True
|
||||
reqs.append(req)
|
||||
text_lengths.append(len(text_input_ids))
|
||||
sequence_lengths.append(len(sequence_ids))
|
||||
|
||||
chunked_prefill_size = self.server_args.chunked_prefill_size
|
||||
if chunked_prefill_size <= 0:
|
||||
chunked_prefill_size = None
|
||||
|
||||
adder = PrefillAdder(
|
||||
self.page_size,
|
||||
self.tree_cache,
|
||||
self.token_to_kv_pool_allocator,
|
||||
None,
|
||||
0.5,
|
||||
self.server_args.max_prefill_tokens,
|
||||
chunked_prefill_size,
|
||||
prefill_max_requests=len(reqs),
|
||||
dllm_config=None,
|
||||
)
|
||||
for req in reqs:
|
||||
req.init_next_round_input(self.tree_cache)
|
||||
result = adder.add_one_req(
|
||||
req, has_chunked_req=False, truncation_align_size=None
|
||||
)
|
||||
if req not in adder.can_run_list:
|
||||
raise RuntimeError(
|
||||
"Insufficient prefill capacity for LLaDA-Image conditioning"
|
||||
)
|
||||
if result != AddReqResult.CONTINUE:
|
||||
break
|
||||
|
||||
if len(adder.can_run_list) != len(reqs):
|
||||
raise RuntimeError("Could not batch all LLaDA-Image conditioning prompts")
|
||||
|
||||
batch = ScheduleBatch.init_new(
|
||||
reqs=adder.can_run_list,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
tree_cache=self.tree_cache,
|
||||
model_config=self.worker.model_config,
|
||||
enable_overlap=False,
|
||||
spec_algorithm=SpeculativeAlgorithm.NONE,
|
||||
dllm_config=None,
|
||||
)
|
||||
try:
|
||||
batch.prepare_for_extend()
|
||||
batch.forward_mode = ForwardMode.EXTEND
|
||||
# The FutureMap argument is only consulted on decode/spec paths.
|
||||
resolve_forward_inputs(batch, None)
|
||||
forward_batch = ForwardBatch.init_new(
|
||||
batch,
|
||||
self.model_runner,
|
||||
return_hidden_states_before_norm=False,
|
||||
)
|
||||
forward_batch.input_embeds = self._prepare_input_embeds(
|
||||
forward_batch.input_ids,
|
||||
sequence_lengths,
|
||||
text_lengths,
|
||||
component_context=component_context,
|
||||
)
|
||||
forward_batch.llada_image_conditioning_text_lens_cpu = text_lengths
|
||||
model_output = self.model_runner.forward(forward_batch=forward_batch)
|
||||
if not self.model_runner.attn_backend.conditioning_mask_active:
|
||||
raise RuntimeError(
|
||||
"LLaDA-Image conditioning ran without its block attention mask "
|
||||
"because the forward batch lost the conditioning text lengths"
|
||||
)
|
||||
hidden_states = model_output.logits_output.hidden_states
|
||||
|
||||
projected = []
|
||||
offset = 0
|
||||
with component_context(
|
||||
component_name="text_projection", module=self.text_projection
|
||||
) as text_projection:
|
||||
assert text_projection is not None
|
||||
projection_dtype = text_projection.dtype
|
||||
for sequence_length in sequence_lengths:
|
||||
output = text_projection(
|
||||
hidden_states[offset : offset + sequence_length]
|
||||
.unsqueeze(0)
|
||||
.to(dtype=projection_dtype)
|
||||
)
|
||||
projected.append(output.squeeze(0))
|
||||
offset += sequence_length
|
||||
return projected
|
||||
finally:
|
||||
for req in reqs:
|
||||
if not req.kv.is_kv_released:
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
elif req.kv.holds_kv:
|
||||
# KV alloc failed after the row grab. Freeing the bare row
|
||||
# avoids the release_kv_cache lifecycle assert.
|
||||
self.req_to_token_pool.free(req)
|
||||
|
||||
|
||||
class LLaDAImageTextConditioningStage(PipelineStage):
|
||||
def __init__(self, runner: LLaDAImageTextEncoderRunner) -> None:
|
||||
super().__init__()
|
||||
self.runner = runner
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
del server_args
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
return [
|
||||
ComponentUse(stage_name, "queryformer"),
|
||||
ComponentUse(stage_name, "text_projection"),
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
if not isinstance(batch.prompt, str):
|
||||
raise TypeError("LLaDA-Image currently supports one prompt per request")
|
||||
do_cfg = batch.guidance_scale > 1.0
|
||||
prompts = [format_llada_image_prompt(batch.prompt)]
|
||||
if do_cfg:
|
||||
prompts.append(format_llada_image_prompt(batch.negative_prompt))
|
||||
|
||||
outputs = self.runner.encode(
|
||||
prompts,
|
||||
max_sequence_length=batch.max_sequence_length or 2048,
|
||||
component_context=self.use_declared_component,
|
||||
)
|
||||
masks = [
|
||||
torch.ones(output.shape[0], dtype=torch.bool, device=output.device)
|
||||
for output in outputs
|
||||
]
|
||||
output_count = batch.num_outputs_per_prompt
|
||||
batch.prompt_embeds = [outputs[0]] * output_count
|
||||
batch.prompt_attention_mask = [masks[0]] * output_count
|
||||
batch.do_classifier_free_guidance = do_cfg
|
||||
if do_cfg:
|
||||
batch.negative_prompt_embeds = [outputs[1]] * output_count
|
||||
batch.negative_attention_mask = [masks[1]] * output_count
|
||||
else:
|
||||
batch.negative_prompt_embeds = []
|
||||
batch.negative_attention_mask = []
|
||||
return batch
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux import _patchify_latents
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.llada_image import (
|
||||
flux2_vae_bn_stats,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
def _module_device_dtype(module: torch.nn.Module) -> tuple[torch.device, torch.dtype]:
|
||||
parameter = next(module.parameters())
|
||||
return parameter.device, parameter.dtype
|
||||
|
||||
|
||||
class LLaDAImageSourceImageConditioningStage(PipelineStage):
|
||||
def __init__(self, sigvq, vae, image_processor) -> None:
|
||||
super().__init__()
|
||||
self.sigvq = sigvq
|
||||
self.vae = vae
|
||||
self.image_processor = image_processor
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
del server_args
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
return [
|
||||
ComponentUse(stage_name, "sigvq"),
|
||||
ComponentUse(stage_name, "vae"),
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
del server_args
|
||||
if batch.condition_image is None:
|
||||
batch.source_latents = None
|
||||
return batch
|
||||
|
||||
source_images = (
|
||||
batch.condition_image
|
||||
if isinstance(batch.condition_image, list)
|
||||
else [batch.condition_image]
|
||||
)
|
||||
if len(source_images) != 1:
|
||||
raise ValueError("LLaDA-Image editing requires exactly one source image")
|
||||
|
||||
image = self.image_processor.preprocess(
|
||||
source_images[0],
|
||||
height=batch.height,
|
||||
width=batch.width,
|
||||
)
|
||||
if batch.batch_size > 1:
|
||||
image = image.repeat(batch.batch_size, 1, 1, 1)
|
||||
|
||||
sigvq_pixels = F.interpolate(
|
||||
image.float(),
|
||||
size=(batch.height // 2, batch.width // 2),
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
with self.use_declared_component(
|
||||
component_name="sigvq", module=self.sigvq
|
||||
) as sigvq:
|
||||
assert sigvq is not None
|
||||
sigvq_device, sigvq_dtype = _module_device_dtype(sigvq)
|
||||
semantic_features = sigvq(
|
||||
sigvq_pixels.to(device=sigvq_device, dtype=sigvq_dtype)
|
||||
)
|
||||
|
||||
with self.use_declared_component(component_name="vae", module=self.vae) as vae:
|
||||
assert vae is not None
|
||||
vae_device, vae_dtype = _module_device_dtype(vae)
|
||||
posterior = vae.encode(image.to(device=vae_device, dtype=vae_dtype))
|
||||
source_latents = _patchify_latents(posterior.mode())
|
||||
mean, std = flux2_vae_bn_stats(vae, source_latents)
|
||||
source_latents = (source_latents - mean) / std
|
||||
|
||||
batch.image_embeds = list(semantic_features.unbind(dim=0))
|
||||
batch.source_latents = [
|
||||
latent.unsqueeze(1) for latent in source_latents.unbind(dim=0)
|
||||
]
|
||||
return batch
|
||||
@@ -98,7 +98,13 @@ class TimestepPreparationStage(PipelineStage):
|
||||
sigmas = batch.sigmas
|
||||
n_tokens = batch.n_tokens
|
||||
|
||||
sigmas = server_args.pipeline_config.prepare_sigmas(sigmas, num_inference_steps)
|
||||
use_uniform_sigmas = getattr(
|
||||
getattr(scheduler, "config", None), "use_uniform_sigmas", False
|
||||
)
|
||||
if sigmas is not None or not use_uniform_sigmas:
|
||||
sigmas = server_args.pipeline_config.prepare_sigmas(
|
||||
sigmas, num_inference_steps
|
||||
)
|
||||
batch.sigmas = sigmas
|
||||
|
||||
# Prepare extra kwargs for set_timesteps
|
||||
|
||||
@@ -562,6 +562,50 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
|
||||
),
|
||||
]
|
||||
|
||||
_LLADA_IMAGE_SERVER_ARGS = DiffusionServerArgs(
|
||||
model_path="inclusionAI/LLaDA-Image",
|
||||
modality="image",
|
||||
num_gpus=1,
|
||||
tp_size=1,
|
||||
ulysses_degree=1,
|
||||
ring_degree=1,
|
||||
cfg_parallel=False,
|
||||
extras=[
|
||||
"--trust-remote-code",
|
||||
"--revision",
|
||||
"e4e2703f410f7ddb6ee8d6b09dac6a8ec5093039",
|
||||
],
|
||||
)
|
||||
_LLADA_IMAGE_SAMPLING = {"num_inference_steps": 50, "guidance_scale": 5.0, "seed": 42}
|
||||
if not current_platform.is_hip() and not current_platform.is_xpu():
|
||||
ONE_GPU_CASES += [
|
||||
DiffusionTestCase(
|
||||
"llada_image_bf16_t2i",
|
||||
_LLADA_IMAGE_SERVER_ARGS,
|
||||
DiffusionSamplingParams(
|
||||
prompt="A red fox sitting on a snowy hill",
|
||||
output_size="512x512",
|
||||
extras=_LLADA_IMAGE_SAMPLING,
|
||||
),
|
||||
run_perf_check=False,
|
||||
run_consistency_check=False,
|
||||
run_component_accuracy_check=False,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
"llada_image_bf16_edit",
|
||||
_LLADA_IMAGE_SERVER_ARGS,
|
||||
DiffusionSamplingParams(
|
||||
prompt="Convert 2D style to 3D style",
|
||||
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
|
||||
output_size="512x512",
|
||||
extras=_LLADA_IMAGE_SAMPLING,
|
||||
),
|
||||
run_perf_check=False,
|
||||
run_consistency_check=False,
|
||||
run_component_accuracy_check=False,
|
||||
),
|
||||
]
|
||||
|
||||
# Skip hunyuan3d on AMD: marching_cubes surface extraction produces invalid SDF on ROCm.
|
||||
if not current_platform.is_hip():
|
||||
ONE_GPU_CASES.append(
|
||||
|
||||
@@ -3578,6 +3578,152 @@
|
||||
"runtime_peak_allocated_mb": 25745.0,
|
||||
"estimated_full_test_time_s": 404.0,
|
||||
"warmup_peak_vram_mb": 14598.0
|
||||
},
|
||||
"llada_image_bf16_t2i": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 0.07,
|
||||
"LLaDAImageTextConditioningStage": 36.79,
|
||||
"LLaDAImageSourceImageConditioningStage": 0.05,
|
||||
"TimestepPreparationStage": 1.94,
|
||||
"LLaDAImageLatentPreparationStage": 0.2,
|
||||
"DenoisingStage": 4052.86,
|
||||
"DecodingStage": 40.77
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 77.02,
|
||||
"1": 78.74,
|
||||
"2": 83.82,
|
||||
"3": 81.04,
|
||||
"4": 80.83,
|
||||
"5": 81.33,
|
||||
"6": 80.96,
|
||||
"7": 81.2,
|
||||
"8": 80.97,
|
||||
"9": 81.26,
|
||||
"10": 80.74,
|
||||
"11": 81.22,
|
||||
"12": 80.93,
|
||||
"13": 81.19,
|
||||
"14": 80.91,
|
||||
"15": 81.32,
|
||||
"16": 80.96,
|
||||
"17": 82.78,
|
||||
"18": 81.03,
|
||||
"19": 81.29,
|
||||
"20": 83.72,
|
||||
"21": 81.44,
|
||||
"22": 80.94,
|
||||
"23": 81.4,
|
||||
"24": 80.98,
|
||||
"25": 81.26,
|
||||
"26": 80.87,
|
||||
"27": 81.42,
|
||||
"28": 81.16,
|
||||
"29": 81.41,
|
||||
"30": 81.13,
|
||||
"31": 81.58,
|
||||
"32": 81.1,
|
||||
"33": 81.47,
|
||||
"34": 81.13,
|
||||
"35": 81.34,
|
||||
"36": 81.12,
|
||||
"37": 81.37,
|
||||
"38": 81.12,
|
||||
"39": 81.42,
|
||||
"40": 81.09,
|
||||
"41": 81.6,
|
||||
"42": 92.45,
|
||||
"43": 83.16,
|
||||
"44": 81.2,
|
||||
"45": 81.45,
|
||||
"46": 81.32,
|
||||
"47": 81.65,
|
||||
"48": 81.22,
|
||||
"49": 81.53
|
||||
},
|
||||
"expected_e2e_ms": 4214.9,
|
||||
"expected_load_ms": 51491.39,
|
||||
"expected_avg_denoise_ms": 80.99,
|
||||
"expected_median_denoise_ms": 81.06,
|
||||
"load_peak_vram_mb": 48046.0,
|
||||
"runtime_peak_vram_mb": 50028.0,
|
||||
"load_peak_allocated_mb": 47799.72,
|
||||
"runtime_peak_allocated_mb": 48224.27,
|
||||
"estimated_full_test_time_s": 76.8,
|
||||
"warmup_peak_vram_mb": 50028.0
|
||||
},
|
||||
"llada_image_bf16_edit": {
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 10.58,
|
||||
"LLaDAImageTextConditioningStage": 37.47,
|
||||
"LLaDAImageSourceImageConditioningStage": 70.63,
|
||||
"TimestepPreparationStage": 1.28,
|
||||
"LLaDAImageLatentPreparationStage": 0.21,
|
||||
"DenoisingStage": 8880.41,
|
||||
"DecodingStage": 55.64
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 156.23,
|
||||
"1": 177.2,
|
||||
"2": 177.21,
|
||||
"3": 177.7,
|
||||
"4": 177.58,
|
||||
"5": 179.87,
|
||||
"6": 178.99,
|
||||
"7": 180.97,
|
||||
"8": 177.51,
|
||||
"9": 177.87,
|
||||
"10": 177.94,
|
||||
"11": 178.05,
|
||||
"12": 184.78,
|
||||
"13": 178.68,
|
||||
"14": 178.51,
|
||||
"15": 178.74,
|
||||
"16": 177.99,
|
||||
"17": 179.74,
|
||||
"18": 179.31,
|
||||
"19": 179.54,
|
||||
"20": 178.13,
|
||||
"21": 178.0,
|
||||
"22": 179.02,
|
||||
"23": 178.76,
|
||||
"24": 180.81,
|
||||
"25": 179.14,
|
||||
"26": 178.3,
|
||||
"27": 178.71,
|
||||
"28": 178.18,
|
||||
"29": 178.26,
|
||||
"30": 179.02,
|
||||
"31": 179.39,
|
||||
"32": 180.16,
|
||||
"33": 179.42,
|
||||
"34": 179.84,
|
||||
"35": 179.17,
|
||||
"36": 179.35,
|
||||
"37": 178.6,
|
||||
"38": 179.37,
|
||||
"39": 179.67,
|
||||
"40": 178.92,
|
||||
"41": 179.36,
|
||||
"42": 179.29,
|
||||
"43": 179.02,
|
||||
"44": 179.62,
|
||||
"45": 178.28,
|
||||
"46": 179.62,
|
||||
"47": 178.49,
|
||||
"48": 178.7,
|
||||
"49": 180.12
|
||||
},
|
||||
"expected_e2e_ms": 9060.3,
|
||||
"expected_load_ms": 48754.68,
|
||||
"expected_avg_denoise_ms": 177.56,
|
||||
"expected_median_denoise_ms": 177.89,
|
||||
"load_peak_vram_mb": 48046.0,
|
||||
"runtime_peak_vram_mb": 50028.0,
|
||||
"load_peak_allocated_mb": 47799.72,
|
||||
"runtime_peak_allocated_mb": 48227.33,
|
||||
"estimated_full_test_time_s": 78.9,
|
||||
"warmup_peak_vram_mb": 50028.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import math
|
||||
import unittest
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.llada_image import (
|
||||
LLaDAImageArchConfig,
|
||||
editing_rope_rows,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.llada_image import (
|
||||
LLaDAImagePipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.llada_image import (
|
||||
LLaDAImageSamplingParams,
|
||||
LLaDAImageTurboSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.registry import _get_config_info
|
||||
|
||||
|
||||
class TestLLaDAImageConfig(unittest.TestCase):
|
||||
def test_base_and_turbo_checkpoints_get_their_own_defaults(self):
|
||||
"""A default Base request must not run the 4-step Turbo recipe."""
|
||||
_get_config_info.cache_clear()
|
||||
cases = {
|
||||
"inclusionAI/LLaDA-Image": (LLaDAImageSamplingParams, 50, 5.0),
|
||||
"inclusionAI/LLaDA-Image-Turbo": (LLaDAImageTurboSamplingParams, 4, 1.0),
|
||||
"inclusionAI/LLaDA-Image-Turbo-FP8": (
|
||||
LLaDAImageTurboSamplingParams,
|
||||
4,
|
||||
1.0,
|
||||
),
|
||||
}
|
||||
for model_id, (sampling_cls, steps, guidance) in cases.items():
|
||||
with self.subTest(model_id=model_id):
|
||||
info = _get_config_info(model_id)
|
||||
self.assertIs(info.sampling_param_cls, sampling_cls)
|
||||
params = info.sampling_param_cls()
|
||||
self.assertEqual(params.num_inference_steps, steps)
|
||||
self.assertEqual(params.guidance_scale, guidance)
|
||||
|
||||
def test_editing_size_limit_follows_sequence_rope_table(self):
|
||||
"""Editing sizes past the sequence RoPE table must be caught on the host."""
|
||||
# A short prompt of 20 tokens plus the 256 conditioning query tokens.
|
||||
caption_tokens = 20 + 256
|
||||
limit = LLaDAImageArchConfig().axes_lens[0]
|
||||
|
||||
def rows(side: int) -> int:
|
||||
image_tokens = (side // 16) ** 2
|
||||
sigvq_tokens = math.ceil(side / 32) ** 2
|
||||
return editing_rope_rows(
|
||||
caption_tokens, [image_tokens, image_tokens], sigvq_tokens
|
||||
)
|
||||
|
||||
self.assertLessEqual(rows(1904), limit)
|
||||
self.assertGreater(rows(1920), limit)
|
||||
|
||||
def test_unservable_editing_sizes_are_rejected_at_validation(self):
|
||||
"""Edits off the SigVQ grid or past the RoPE table fail before any GPU work."""
|
||||
config = LLaDAImagePipelineConfig()
|
||||
config.validate_output_size(1008, 1008, editing=False)
|
||||
config.validate_output_size(1024, 1024, editing=True)
|
||||
with self.assertRaisesRegex(ValueError, "divisible by 32"):
|
||||
config.validate_output_size(1008, 1008, editing=True)
|
||||
with self.assertRaisesRegex(ValueError, "sequence positions"):
|
||||
config.validate_output_size(2048, 2048, editing=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -2,6 +2,7 @@ import inspect
|
||||
import os
|
||||
from dataclasses import fields
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from PIL import Image
|
||||
|
||||
@@ -15,6 +16,7 @@ from sglang.multimodal_gen.configs.sample.longcat_image import (
|
||||
LongCatImageSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai import image_api
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import (
|
||||
_build_image_response_kwargs,
|
||||
_fallback_image_urls,
|
||||
@@ -290,3 +292,16 @@ def test_select_image_variant_path_falls_back_to_single_file_path():
|
||||
|
||||
assert _select_image_variant_path(item, None) == "only.png"
|
||||
assert _select_image_variant_path(item, "0") == "only.png"
|
||||
|
||||
|
||||
def test_request_validation_errors_are_client_errors(monkeypatch):
|
||||
"""A request that fails sampling-parameter validation returns 400, not 500."""
|
||||
|
||||
def reject(request_id, **kwargs):
|
||||
raise ValueError("unsupported size")
|
||||
|
||||
monkeypatch.setattr(image_api, "build_sampling_params", reject)
|
||||
with pytest.raises(HTTPException) as error:
|
||||
image_api._build_request_sampling_params("request", size="7x7")
|
||||
assert error.value.status_code == 400
|
||||
assert "unsupported size" in error.value.detail
|
||||
|
||||
@@ -80,6 +80,7 @@ ATTENTION_BACKEND_CHOICES = [
|
||||
"fa3",
|
||||
"fa4",
|
||||
"flashinfer",
|
||||
"llada2_cfg_flashinfer",
|
||||
"flashmla",
|
||||
"trtllm_mla",
|
||||
"cutedsl_mla",
|
||||
|
||||
@@ -77,6 +77,20 @@ def create_flashinfer_backend(runner):
|
||||
return FlashInferMLAAttnBackend(runner)
|
||||
|
||||
|
||||
@register_attention_backend("llada2_cfg_flashinfer")
|
||||
def create_llada2_cfg_flashinfer_backend(runner):
|
||||
if runner.use_mla_backend:
|
||||
raise ValueError("LLaDA2 CFG attention does not use an MLA backend")
|
||||
|
||||
from sglang.srt.layers.attention.llada2_cfg_flashinfer_backend import (
|
||||
LLaDA2CFGFlashInferAttnBackend,
|
||||
)
|
||||
|
||||
return LLaDA2CFGFlashInferAttnBackend(
|
||||
runner, init_new_workspace=runner.init_new_workspace
|
||||
)
|
||||
|
||||
|
||||
@register_attention_backend("trtllm_mla")
|
||||
def create_trtllm_mla_backend(runner):
|
||||
if not runner.use_mla_backend:
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FlashInfer backend for LLaDA-Image conditioning masks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from flashinfer.prefill import BatchPrefillWithRaggedKVCacheWrapper
|
||||
|
||||
from sglang.srt.layers.attention.flashinfer_backend import (
|
||||
FlashInferAttnBackend,
|
||||
PrefillMetadata,
|
||||
)
|
||||
|
||||
|
||||
def _build_llada_image_custom_mask(
|
||||
text_lens: list[int],
|
||||
sequence_lens: list[int],
|
||||
device: torch.device | str,
|
||||
) -> torch.Tensor:
|
||||
"""Build the official one-pass text/query block mask in ragged layout."""
|
||||
if len(text_lens) != len(sequence_lens):
|
||||
raise RuntimeError("LLaDA-Image conditioning metadata batch mismatch")
|
||||
|
||||
flattened_masks: list[torch.Tensor] = []
|
||||
for text_len, sequence_len in zip(text_lens, sequence_lens):
|
||||
text_len = int(text_len)
|
||||
sequence_len = int(sequence_len)
|
||||
if text_len <= 0 or text_len >= sequence_len:
|
||||
raise RuntimeError(
|
||||
"LLaDA-Image conditioning text length must leave query tokens"
|
||||
)
|
||||
attention_mask = torch.ones(
|
||||
(sequence_len, sequence_len), dtype=torch.bool, device=device
|
||||
)
|
||||
attention_mask[:text_len, text_len:] = False
|
||||
flattened_masks.append(attention_mask.flatten())
|
||||
return torch.cat(flattened_masks)
|
||||
|
||||
|
||||
class LLaDA2CFGFlashInferAttnBackend(FlashInferAttnBackend):
|
||||
"""Stock FlashInfer plus LLaDA-specific ragged attention masks."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._llada_image_conditioning_mask_active = False
|
||||
self._cfg_prefill_wrapper_ragged = BatchPrefillWithRaggedKVCacheWrapper(
|
||||
self.workspace_buffer, "NHD", backend="fa2"
|
||||
)
|
||||
|
||||
@property
|
||||
def conditioning_mask_active(self) -> bool:
|
||||
return self._llada_image_conditioning_mask_active
|
||||
|
||||
def init_forward_metadata(self, forward_batch):
|
||||
self._llada_image_conditioning_mask_active = False
|
||||
text_lens = forward_batch.llada_image_conditioning_text_lens_cpu
|
||||
if text_lens is None:
|
||||
return super().init_forward_metadata(forward_batch)
|
||||
|
||||
if not forward_batch.forward_mode.is_extend_without_speculative():
|
||||
raise RuntimeError(
|
||||
"LLaDA-Image conditioning mask requires a one-shot extend"
|
||||
)
|
||||
if self.num_wrappers != 1:
|
||||
raise RuntimeError(
|
||||
"LLaDA-Image conditioning requires one FlashInfer wrapper"
|
||||
)
|
||||
|
||||
seq_lens = forward_batch.seq_lens
|
||||
prefix_lens = forward_batch.extend_prefix_lens
|
||||
if bool(torch.any(prefix_lens != 0).item()):
|
||||
raise RuntimeError(
|
||||
"LLaDA-Image conditioning does not support cached prefixes"
|
||||
)
|
||||
custom_mask = _build_llada_image_custom_mask(
|
||||
text_lens,
|
||||
seq_lens.tolist(),
|
||||
seq_lens.device,
|
||||
)
|
||||
qo_indptr = torch.zeros(
|
||||
seq_lens.numel() + 1,
|
||||
dtype=torch.int32,
|
||||
device=seq_lens.device,
|
||||
)
|
||||
qo_indptr[1:] = torch.cumsum(seq_lens, dim=0)
|
||||
prefill_indices_updater = self.indices_updater_prefill
|
||||
self._cfg_prefill_wrapper_ragged.begin_forward(
|
||||
qo_indptr,
|
||||
qo_indptr,
|
||||
prefill_indices_updater.num_qo_heads,
|
||||
prefill_indices_updater.num_kv_heads,
|
||||
prefill_indices_updater.head_dim,
|
||||
custom_mask=custom_mask,
|
||||
causal=False,
|
||||
q_data_type=prefill_indices_updater.q_data_type,
|
||||
kv_data_type=prefill_indices_updater.data_type,
|
||||
non_blocking=True,
|
||||
fixed_split_size=self.prefill_split_tile_size,
|
||||
)
|
||||
self._llada_image_conditioning_mask_active = True
|
||||
self.forward_metadata = PrefillMetadata(
|
||||
self.prefill_wrappers_paged,
|
||||
use_ragged=True,
|
||||
extend_no_prefix=True,
|
||||
)
|
||||
|
||||
def forward_extend(
|
||||
self,
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
layer,
|
||||
forward_batch,
|
||||
save_kv_cache=True,
|
||||
):
|
||||
if not self._llada_image_conditioning_mask_active:
|
||||
return super().forward_extend(
|
||||
q, k, v, layer, forward_batch, save_kv_cache=save_kv_cache
|
||||
)
|
||||
|
||||
if k is None or v is None:
|
||||
raise RuntimeError("LLaDA ragged attention requires explicit K/V")
|
||||
|
||||
q_view = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim)
|
||||
k_view = k.view(-1, layer.tp_k_head_num, layer.head_dim)
|
||||
v_view = v.view(-1, layer.tp_v_head_num, layer.head_dim)
|
||||
attention_output = self._cfg_prefill_wrapper_ragged.forward(
|
||||
q_view,
|
||||
k_view,
|
||||
v_view,
|
||||
causal=False,
|
||||
sm_scale=layer.scaling,
|
||||
logits_soft_cap=layer.logit_cap,
|
||||
)
|
||||
return attention_output.view(-1, layer.tp_q_head_num * layer.head_dim)
|
||||
@@ -728,6 +728,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# Encoder-decoder cross-attention mask
|
||||
cross_attention_custom_mask: Optional[torch.Tensor] = None
|
||||
|
||||
# LLaDA-Image conditioning text lengths, declared so batch rebuilds
|
||||
# carry them to the llada2_cfg_flashinfer backend
|
||||
llada_image_conditioning_text_lens_cpu: Optional[List[int]] = None
|
||||
|
||||
# For DP attention (padding / local info)
|
||||
dp_padding_mode: Optional[DpPaddingMode] = None
|
||||
# for extend, local start pos and num tokens is different in logits processor
|
||||
|
||||
@@ -1305,6 +1305,43 @@ class LLaDA2MoeModel(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
_EXPERT_PROJECTION_NAMES = frozenset({"gate_proj", "up_proj", "down_proj"})
|
||||
|
||||
|
||||
def _prepare_llada2_language_weights(
|
||||
weights: Iterable[Tuple[str, torch.Tensor]],
|
||||
*,
|
||||
num_experts: int,
|
||||
) -> Iterable[Tuple[str, torch.Tensor]]:
|
||||
"""Normalize language-model prefixes and expand fused expert tensors."""
|
||||
for name, loaded_weight in weights:
|
||||
if name.startswith("model.language_model."):
|
||||
name = "model." + name.removeprefix("model.language_model.")
|
||||
elif name.startswith("model.lm_head."):
|
||||
name = "lm_head." + name.removeprefix("model.lm_head.")
|
||||
|
||||
expert_prefix, separator, projection_name = name.rpartition(".")
|
||||
is_fused_expert = (
|
||||
separator
|
||||
and expert_prefix.endswith(".mlp.experts")
|
||||
and projection_name in _EXPERT_PROJECTION_NAMES
|
||||
)
|
||||
if not is_fused_expert:
|
||||
yield name, loaded_weight
|
||||
continue
|
||||
|
||||
if loaded_weight.ndim != 3 or loaded_weight.shape[0] != num_experts:
|
||||
raise ValueError(
|
||||
f"Invalid fused expert weight {name!r}: expected first dimension "
|
||||
f"{num_experts}, got shape={tuple(loaded_weight.shape)}"
|
||||
)
|
||||
for expert_id in range(num_experts):
|
||||
yield (
|
||||
f"{expert_prefix}.{expert_id}.{projection_name}.weight",
|
||||
loaded_weight[expert_id],
|
||||
)
|
||||
|
||||
|
||||
class LLaDA2MoeModelLM(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1346,6 +1383,9 @@ class LLaDA2MoeModelLM(nn.Module):
|
||||
def end_layer(self):
|
||||
return self.model.end_layer
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.model.word_embeddings
|
||||
|
||||
def get_embed_and_head(self):
|
||||
"""Used by the eagle_worker."""
|
||||
return self.model.word_embeddings.weight, self.lm_head.weight
|
||||
@@ -1399,7 +1439,9 @@ class LLaDA2MoeModelLM(nn.Module):
|
||||
)
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
for name, loaded_weight in weights:
|
||||
for name, loaded_weight in _prepare_llada2_language_weights(
|
||||
weights, num_experts=self.config.num_experts
|
||||
):
|
||||
if (
|
||||
("v_head" in name)
|
||||
or ("inv_freq" in name)
|
||||
|
||||
@@ -1505,6 +1505,37 @@ def publish(
|
||||
(bags re-projected, provenance reset, role overwritten), which is what
|
||||
lets one process rebuild an engine after shutting the previous one down.
|
||||
"""
|
||||
return _configure_context(_CONTEXT, server_args, role=role, ranks=ranks)
|
||||
|
||||
|
||||
def create_context(
|
||||
server_args, *, role: str, ranks: SpawnRanks | None = None
|
||||
) -> RuntimeContext:
|
||||
"""Build a model-owned context without replacing the active process config."""
|
||||
context = RuntimeContext(parallel=ParallelContext())
|
||||
return _configure_context(context, server_args, role=role, ranks=ranks)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def use_context(context: RuntimeContext):
|
||||
"""Activate a model context for serialized embedded SRT work.
|
||||
|
||||
Like process group scopes, this changes process globals and requires the
|
||||
caller to serialize model execution. It does not isolate concurrent threads.
|
||||
The previous context is restored by identity without re-projecting its bags.
|
||||
"""
|
||||
global _CONTEXT, _PARALLEL
|
||||
previous_context, previous_parallel = _CONTEXT, _PARALLEL
|
||||
_CONTEXT, _PARALLEL = context, context.parallel
|
||||
try:
|
||||
yield context
|
||||
finally:
|
||||
_CONTEXT, _PARALLEL = previous_context, previous_parallel
|
||||
|
||||
|
||||
def _configure_context(
|
||||
context, server_args, *, role: str, ranks: SpawnRanks | None = None
|
||||
) -> RuntimeContext:
|
||||
if _ROLE_NS_MODE == "enforce" and role not in ROLE_NAMESPACE_SETS:
|
||||
# Fail closed at publish time, not at the first stray read.
|
||||
raise ValueError(
|
||||
@@ -1512,8 +1543,8 @@ def publish(
|
||||
"its namespace set (None for the full tree)."
|
||||
)
|
||||
server_args.resolve_once()
|
||||
discarded = _CONTEXT.overrides_log()
|
||||
_CONTEXT.set_server_args(server_args)
|
||||
discarded = context.overrides_log()
|
||||
context.set_server_args(server_args)
|
||||
if discarded:
|
||||
logger.warning(
|
||||
"publish(role=%s) re-projected the config bags and dropped %d "
|
||||
@@ -1524,16 +1555,16 @@ def publish(
|
||||
f"{source}({', '.join(sorted(fields))})" for source, fields in discarded
|
||||
),
|
||||
)
|
||||
_CONTEXT._publish_role = role
|
||||
context._publish_role = role
|
||||
# Disabled DCP has rank zero even in processes without a rank bundle.
|
||||
if not _CONTEXT.parallel.dcp_enabled:
|
||||
_CONTEXT.parallel.override_permanently(attn_dcp_rank=0)
|
||||
if not context.parallel.dcp_enabled:
|
||||
context.parallel.override_permanently(attn_dcp_rank=0)
|
||||
# The device is assigned by the launcher; it is not a config field.
|
||||
_CONTEXT.config_bag("device")._set(
|
||||
context.config_bag("device")._set(
|
||||
"gpu_id", ranks.gpu_id if ranks is not None else None
|
||||
)
|
||||
if ranks is not None:
|
||||
parallel = _CONTEXT.parallel
|
||||
parallel = context.parallel
|
||||
placement = derive_spawn_ranks(
|
||||
world_rank=ranks.world_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
@@ -1567,7 +1598,7 @@ def publish(
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
return _CONTEXT
|
||||
return context
|
||||
|
||||
|
||||
def _attention_ranks(parallel, tp_rank: int) -> dict:
|
||||
|
||||
Reference in New Issue
Block a user