[diffusion] refactor: deduplicate UniPC solvers and Gemma3 shard loading (#42899)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-09 16:26:04 +08:00
committed by GitHub
co-authored by Mick Qian
parent 125f88cb75
commit 45ad463c87
7 changed files with 341 additions and 542 deletions
@@ -34,6 +34,36 @@ from sglang.srt.models.siglip import SiglipVisionModel
logger = logging.getLogger(__name__)
def _load_with_shard_id(
weight_loader, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id
) -> None:
"""Try the checkpoint shard id before its equivalent string or integer form."""
try:
weight_loader(param, loaded_weight, shard_id)
return
except (AssertionError, TypeError):
pass
if isinstance(shard_id, str):
mapping = {"q": 0, "k": 1, "v": 2}
if shard_id in mapping:
weight_loader(param, loaded_weight, mapping[shard_id])
return
if shard_id.isdigit():
weight_loader(param, loaded_weight, int(shard_id))
return
elif isinstance(shard_id, int):
mapping = {0: "q", 1: "k", 2: "v"}
if shard_id in mapping:
weight_loader(param, loaded_weight, mapping[shard_id])
return
raise TypeError(
f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} "
f"(param={param.name})."
)
def get_attention_sliding_window_size(config):
return config.sliding_window - 1
@@ -593,43 +623,6 @@ class Gemma3TextModel(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
def _load_with_shard_id(
weight_loader, param, loaded_weight: torch.Tensor, shard_id
) -> None:
"""Call param.weight_loader with best-effort shard_id normalization.
Different fused-QKV implementations expect different shard_id types:
- Some expect strings: "q"/"k"/"v"
- Some expect integer indices: 0/1/2
We try the provided shard_id first, then fall back between str/int forms.
"""
try:
weight_loader(param, loaded_weight, shard_id)
return
except (AssertionError, TypeError):
pass
# Fall back between common representations.
if isinstance(shard_id, str):
mapping = {"q": 0, "k": 1, "v": 2}
if shard_id in mapping:
weight_loader(param, loaded_weight, mapping[shard_id])
return
if shard_id.isdigit():
weight_loader(param, loaded_weight, int(shard_id))
return
elif isinstance(shard_id, int):
mapping = {0: "q", 1: "k", 2: "v"}
if shard_id in mapping:
weight_loader(param, loaded_weight, mapping[shard_id])
return
# Re-raise with a clearer message.
raise TypeError(
f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} "
f"(param={getattr(param, 'name', '<param>')})."
)
stacked_params_mapping = getattr(
getattr(self.config, "arch_config", object()),
"stacked_params_mapping",
@@ -796,42 +789,6 @@ class Gemma3ForConditionalGeneration(
loaded_params: Set[str] = set()
params_dict = dict(self.named_parameters())
def _load_with_shard_id(
weight_loader, param, loaded_weight: torch.Tensor, shard_id
) -> None:
"""Call param.weight_loader with best-effort shard_id normalization.
Different fused-QKV implementations expect different shard_id types:
- Some expect strings: "q"/"k"/"v"
- Some expect integer indices: 0/1/2
We try the provided shard_id first, then fall back between str/int forms.
"""
try:
weight_loader(param, loaded_weight, shard_id)
return
except (AssertionError, TypeError):
pass
# Fall back between common representations.
if isinstance(shard_id, str):
mapping = {"q": 0, "k": 1, "v": 2}
if shard_id in mapping:
weight_loader(param, loaded_weight, mapping[shard_id])
return
if shard_id.isdigit():
weight_loader(param, loaded_weight, int(shard_id))
return
elif isinstance(shard_id, int):
mapping = {0: "q", 1: "k", 2: "v"}
if shard_id in mapping:
weight_loader(param, loaded_weight, mapping[shard_id])
return
raise TypeError(
f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} "
f"(param={getattr(param, 'name', '<param>')})."
)
# Separate weights
language_model_weights: list[tuple[str, torch.Tensor]] = []
other_weights: list[tuple[str, torch.Tensor]] = []
@@ -19,9 +19,16 @@ from diffusers.schedulers.scheduling_utils import (
from diffusers.utils import deprecate
from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
from sglang.multimodal_gen.runtime.models.schedulers.unipc import (
UniPCSchedulerMixin,
compute_unipc_bh_coefficients,
unipc_predictor_step,
)
class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
class FlowUniPCMultistepScheduler(
UniPCSchedulerMixin, SchedulerMixin, ConfigMixin, BaseScheduler
):
"""
`UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models.
@@ -139,34 +146,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
BaseScheduler.__init__(self)
@property
def step_index(self):
"""
The index counter for current timestep. It will increase 1 after each scheduler step.
"""
return self._step_index
@property
def begin_index(self):
"""
The index for the first timestep. It should be set from pipeline with `set_begin_index` method.
"""
return self._begin_index
def set_shift(self, shift: float) -> None:
self.config.shift = shift
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index
def set_begin_index(self, begin_index: int = 0):
"""
Sets the begin index for the scheduler. This function should be run from pipeline before the inference.
Args:
begin_index (`int`):
The begin index for the scheduler.
"""
self._begin_index = begin_index
# Modified from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler.set_timesteps
def set_timesteps(
self,
@@ -236,44 +218,6 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
self._step_index = None
self._begin_index = None
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample
def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:
"""
"Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the
prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by
s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing
pixels from saturation at each step. We find that dynamic thresholding results in significantly better
photorealism as well as better image-text alignment, especially when using very large guidance weights."
https://arxiv.org/abs/2205.11487
"""
dtype = sample.dtype
batch_size, channels, *remaining_dims = sample.shape
if dtype not in (torch.float32, torch.float64):
sample = (
sample.float()
) # upcast for quantile calculation, and clamp not implemented for cpu half
# Flatten sample for doing quantile calculation along each image
sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
abs_sample = sample.abs() # "a certain percentile absolute pixel value"
s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
s = torch.clamp(
s, min=1, max=self.config.sample_max_value
) # When clamped to min=1, equivalent to standard clipping to [-1, 1]
s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0
sample = (
torch.clamp(sample, -s, s) / s
) # "we threshold xt0 to the range [-s, s] and then divide by s"
sample = sample.reshape(batch_size, channels, *remaining_dims)
sample = sample.to(dtype)
return sample
def _sigma_to_alpha_sigma_t(self, sigma) -> tuple[Any, Any]:
return 1 - sigma, sigma
@@ -433,27 +377,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
else:
rks = torch.ones(1, device=device, dtype=h.dtype)
R = []
b = []
hh = -h if self.predict_x0 else h
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
h_phi_k = h_phi_1 / hh - 1
factorial_i = 1
if self.config.solver_type == "bh1":
B_h = hh
elif self.config.solver_type == "bh2":
B_h = torch.expm1(hh)
else:
raise NotImplementedError()
for i in range(1, order + 1):
R.append(torch.pow(rks, i - 1))
b.append(h_phi_k * factorial_i / B_h)
factorial_i *= i + 1
h_phi_k = h_phi_k / hh - 1 / factorial_i
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
h, rks, order, self.predict_x0, self.config.solver_type
)
R = torch.stack(R)
b = torch.stack(b)
@@ -592,27 +518,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
rks = torch.stack(rks_list + [torch.ones_like(rks_list[0])])
D1s = torch.stack(D1s_list, dim=1) if len(D1s_list) > 0 else None
R = []
b = []
hh = -h if self.predict_x0 else h
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
h_phi_k = h_phi_1 / hh - 1
factorial_i = 1
if self.config.solver_type == "bh1":
B_h = hh
elif self.config.solver_type == "bh2":
B_h = torch.expm1(hh)
else:
raise NotImplementedError()
for i in range(1, order + 1):
R.append(torch.pow(rks, i - 1))
b.append(h_phi_k * factorial_i / B_h)
factorial_i *= i + 1
h_phi_k = h_phi_k / hh - 1 / factorial_i
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
h, rks, order, self.predict_x0, self.config.solver_type
)
R = torch.stack(R)
# Avoid torch.tensor(list_of_gpu_scalars) which syncs to host
@@ -671,19 +579,6 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
return step_index
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index
def _init_step_index(self, timestep) -> None:
"""
Initialize the step_index counter for the scheduler.
"""
if self.begin_index is None:
if isinstance(timestep, torch.Tensor):
timestep = timestep.to(self.timesteps.device)
self._step_index = self.index_for_timestep(timestep)
else:
self._step_index = self._begin_index
def step(
self,
model_output: torch.Tensor,
@@ -737,35 +632,10 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
order=self.this_order,
)
for i in range(self.config.solver_order - 1):
self.model_outputs[i] = self.model_outputs[i + 1]
self.timestep_list[i] = self.timestep_list[i + 1]
self.model_outputs[-1] = model_output_convert
self.timestep_list[-1] = timestep # pyright: ignore
if self.config.lower_order_final:
this_order = min(
self.config.solver_order, len(self.timesteps) - self.step_index
) # pyright: ignore
else:
this_order = self.config.solver_order
self.this_order: int = min(
this_order, self.lower_order_nums + 1
) # warmup for multistep
assert self.this_order > 0
self.last_sample = sample
prev_sample = self.multistep_uni_p_bh_update(
model_output=model_output, # pass the original non-converted model output, in case solver-p is used
sample=sample,
order=self.this_order,
prev_sample = unipc_predictor_step(
self, model_output, model_output_convert, timestep, sample
)
if self.lower_order_nums < self.config.solver_order:
self.lower_order_nums += 1
# upon completion increase step index by one
assert self._step_index is not None
self._step_index += 1 # pyright: ignore
@@ -775,61 +645,5 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
return SchedulerOutput(prev_sample=prev_sample)
def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor:
"""
Ensures interchangeability with schedulers that need to scale the denoising model input depending on the
current timestep.
Args:
sample (`torch.Tensor`):
The input sample.
Returns:
`torch.Tensor`:
A scaled input sample.
"""
return sample
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise
def add_noise(
self,
original_samples: torch.Tensor,
noise: torch.Tensor,
timesteps: torch.IntTensor,
) -> torch.Tensor:
# Make sure sigmas and timesteps have the same device and dtype as original_samples
sigmas = self.sigmas.to(
device=original_samples.device, dtype=original_samples.dtype
)
if original_samples.device.type == "mps" and torch.is_floating_point(timesteps):
# mps does not support float64
schedule_timesteps = self.timesteps.to(
original_samples.device, dtype=torch.float32
)
timesteps = timesteps.to(original_samples.device, dtype=torch.float32)
else:
schedule_timesteps = self.timesteps.to(original_samples.device)
timesteps = timesteps.to(original_samples.device)
# begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index
if self.begin_index is None:
step_indices = [
self.index_for_timestep(t, schedule_timesteps) for t in timesteps
]
elif self.step_index is not None:
# add_noise is called after first denoising step (for inpainting)
step_indices = [self.step_index] * timesteps.shape[0]
else:
# add noise is called before first denoising step to create initial latent(img2img)
step_indices = [self.begin_index] * timesteps.shape[0]
sigma = sigmas[step_indices].flatten()
while len(sigma.shape) < len(original_samples.shape):
sigma = sigma.unsqueeze(-1)
alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)
noisy_samples = alpha_t * original_samples + sigma_t * noise
return noisy_samples
EntryClass = FlowUniPCMultistepScheduler
@@ -14,6 +14,10 @@ from dataclasses import dataclass
import numpy as np
import torch
from sglang.multimodal_gen.runtime.models.schedulers.unipc import (
compute_unipc_bh_coefficients,
unipc_predictor_step,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
@@ -406,26 +410,9 @@ class HeliosScheduler:
rks.append(1.0)
rks = torch.tensor(rks, device=device)
R = []
b = []
hh = -h if self.predict_x0 else h
h_phi_1 = torch.expm1(hh)
h_phi_k = h_phi_1 / hh - 1
factorial_i = 1
if self.config.solver_type == "bh1":
B_h = hh
elif self.config.solver_type == "bh2":
B_h = torch.expm1(hh)
else:
raise NotImplementedError()
for i in range(1, order + 1):
R.append(torch.pow(rks, i - 1))
b.append(h_phi_k * factorial_i / B_h)
factorial_i *= i + 1
h_phi_k = h_phi_k / hh - 1 / factorial_i
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
h, rks, order, self.predict_x0, self.config.solver_type
)
R = torch.stack(R)
b = torch.tensor(b, device=device)
@@ -497,25 +484,9 @@ class HeliosScheduler:
rks.append(1.0)
rks = torch.tensor(rks, device=device)
R = []
b = []
hh = -h if self.predict_x0 else h
h_phi_1 = torch.expm1(hh)
h_phi_k = h_phi_1 / hh - 1
factorial_i = 1
if self.config.solver_type == "bh1":
B_h = hh
elif self.config.solver_type == "bh2":
B_h = torch.expm1(hh)
else:
raise NotImplementedError()
for i in range(1, order + 1):
R.append(torch.pow(rks, i - 1))
b.append(h_phi_k * factorial_i / B_h)
factorial_i *= i + 1
h_phi_k = h_phi_k / hh - 1 / factorial_i
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
h, rks, order, self.predict_x0, self.config.solver_type
)
R = torch.stack(R)
b = torch.tensor(b, device=device)
@@ -583,31 +554,10 @@ class HeliosScheduler:
order=self.this_order,
)
for i in range(self.config.solver_order - 1):
self.model_outputs[i] = self.model_outputs[i + 1]
self.timestep_list[i] = self.timestep_list[i + 1]
self.model_outputs[-1] = model_output_convert
self.timestep_list[-1] = timestep
if self.config.lower_order_final:
this_order = min(
self.config.solver_order, len(self.timesteps) - self.step_index
)
else:
this_order = self.config.solver_order
self.this_order = min(this_order, self.lower_order_nums + 1)
assert self.this_order > 0
self.last_sample = sample
prev_sample = self.multistep_uni_p_bh_update(
model_output=model_output,
sample=sample,
order=self.this_order,
prev_sample = unipc_predictor_step(
self, model_output, model_output_convert, timestep, sample
)
if self.lower_order_nums < self.config.solver_order:
self.lower_order_nums += 1
self._step_index += 1
if not return_dict:
@@ -38,6 +38,11 @@ from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import (
SigmaScheduleMixin,
)
from sglang.multimodal_gen.runtime.models.schedulers.unipc import (
UniPCSchedulerMixin,
compute_unipc_bh_coefficients,
unipc_predictor_step,
)
# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar
@@ -123,7 +128,7 @@ def rescale_zero_terminal_snr(betas):
class UniPCMultistepScheduler(
SchedulerMixin, ConfigMixin, BaseScheduler, SigmaScheduleMixin
UniPCSchedulerMixin, SchedulerMixin, ConfigMixin, BaseScheduler, SigmaScheduleMixin
):
"""
`UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models.
@@ -295,9 +300,6 @@ class UniPCMultistepScheduler(
self.alphas_cumprod[-1] = 2**-24
# Currently we only support VP-type noise schedule
self.alpha_t = torch.sqrt(self.alphas_cumprod)
self.sigma_t = torch.sqrt(1 - self.alphas_cumprod)
self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t)
self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5
# standard deviation of the initial noise distribution
@@ -331,34 +333,9 @@ class UniPCMultistepScheduler(
BaseScheduler.__init__(self)
@property
def step_index(self):
"""
The index counter for current timestep. It will increase 1 after each scheduler step.
"""
return self._step_index
@property
def begin_index(self):
"""
The index for the first timestep. It should be set from pipeline with `set_begin_index` method.
"""
return self._begin_index
def set_shift(self, shift: float) -> None:
self.config.flow_shift = shift
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index
def set_begin_index(self, begin_index: int = 0):
"""
Sets the begin index for the scheduler. This function should be run from pipeline before the inference.
Args:
begin_index (`int`):
The begin index for the scheduler.
"""
self._begin_index = begin_index
def set_timesteps(
self,
num_inference_steps: int,
@@ -539,44 +516,6 @@ class UniPCMultistepScheduler(
if device is None or torch.device(device).type == "cpu":
self.sigmas = self.sigmas.to("cpu")
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample
def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:
"""
"Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the
prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by
s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing
pixels from saturation at each step. We find that dynamic thresholding results in significantly better
photorealism as well as better image-text alignment, especially when using very large guidance weights."
https://huggingface.co/papers/2205.11487
"""
dtype = sample.dtype
batch_size, channels, *remaining_dims = sample.shape
if dtype not in (torch.float32, torch.float64):
sample = (
sample.float()
) # upcast for quantile calculation, and clamp not implemented for cpu half
# Flatten sample for doing quantile calculation along each image
sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
abs_sample = sample.abs() # "a certain percentile absolute pixel value"
s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
s = torch.clamp(
s, min=1, max=self.config.sample_max_value
) # When clamped to min=1, equivalent to standard clipping to [-1, 1]
s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0
sample = (
torch.clamp(sample, -s, s) / s
) # "we threshold xt0 to the range [-s, s] and then divide by s"
sample = sample.reshape(batch_size, channels, *remaining_dims)
sample = sample.to(dtype)
return sample
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t
def _sigma_to_t(self, sigma, log_sigmas):
# get log sigma
@@ -768,27 +707,9 @@ class UniPCMultistepScheduler(
rks.append(torch.ones((), dtype=h.dtype, device=h.device))
rks = torch.stack(rks).to(device=device)
R = []
b = []
hh = -h if self.predict_x0 else h
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
h_phi_k = h_phi_1 / hh - 1
factorial_i = 1
if self.config.solver_type == "bh1":
B_h = hh
elif self.config.solver_type == "bh2":
B_h = torch.expm1(hh)
else:
raise NotImplementedError()
for i in range(1, order + 1):
R.append(torch.pow(rks, i - 1))
b.append(h_phi_k * factorial_i / B_h)
factorial_i *= i + 1
h_phi_k = h_phi_k / hh - 1 / factorial_i
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
h, rks, order, self.predict_x0, self.config.solver_type
)
R = torch.stack(R)
b = torch.stack(b).to(device=device)
@@ -906,27 +827,9 @@ class UniPCMultistepScheduler(
rks.append(torch.ones((), dtype=h.dtype, device=h.device))
rks = torch.stack(rks).to(device=device)
R = []
b = []
hh = -h if self.predict_x0 else h
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
h_phi_k = h_phi_1 / hh - 1
factorial_i = 1
if self.config.solver_type == "bh1":
B_h = hh
elif self.config.solver_type == "bh2":
B_h = torch.expm1(hh)
else:
raise NotImplementedError()
for i in range(1, order + 1):
R.append(torch.pow(rks, i - 1))
b.append(h_phi_k * factorial_i / B_h)
factorial_i *= i + 1
h_phi_k = h_phi_k / hh - 1 / factorial_i
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
h, rks, order, self.predict_x0, self.config.solver_type
)
R = torch.stack(R)
b = torch.stack(b).to(device=device)
@@ -981,19 +884,6 @@ class UniPCMultistepScheduler(
return step_index
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index
def _init_step_index(self, timestep):
"""
Initialize the step_index counter for the scheduler.
"""
if self.begin_index is None:
if isinstance(timestep, torch.Tensor):
timestep = timestep.to(self.timesteps.device)
self._step_index = self.index_for_timestep(timestep)
else:
self._step_index = self._begin_index
def step(
self,
model_output: torch.Tensor,
@@ -1044,35 +934,10 @@ class UniPCMultistepScheduler(
order=self.this_order,
)
for i in range(self.config.solver_order - 1):
self.model_outputs[i] = self.model_outputs[i + 1]
self.timestep_list[i] = self.timestep_list[i + 1]
self.model_outputs[-1] = model_output_convert
self.timestep_list[-1] = timestep
if self.config.lower_order_final:
this_order = min(
self.config.solver_order, len(self.timesteps) - self.step_index
)
else:
this_order = self.config.solver_order
self.this_order = min(
this_order, self.lower_order_nums + 1
) # warmup for multistep
assert self.this_order > 0
self.last_sample = sample
prev_sample = self.multistep_uni_p_bh_update(
model_output=model_output, # pass the original non-converted model output, in case solver-p is used
sample=sample,
order=self.this_order,
prev_sample = unipc_predictor_step(
self, model_output, model_output_convert, timestep, sample
)
if self.lower_order_nums < self.config.solver_order:
self.lower_order_nums += 1
# upon completion increase step index by one
self._step_index += 1
@@ -1081,62 +946,6 @@ class UniPCMultistepScheduler(
return SchedulerOutput(prev_sample=prev_sample)
def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor:
"""
Ensures interchangeability with schedulers that need to scale the denoising model input depending on the
current timestep.
Args:
sample (`torch.Tensor`):
The input sample.
Returns:
`torch.Tensor`:
A scaled input sample.
"""
return sample
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise
def add_noise(
self,
original_samples: torch.Tensor,
noise: torch.Tensor,
timesteps: torch.IntTensor,
) -> torch.Tensor:
# Make sure sigmas and timesteps have the same device and dtype as original_samples
sigmas = self.sigmas.to(
device=original_samples.device, dtype=original_samples.dtype
)
if original_samples.device.type == "mps" and torch.is_floating_point(timesteps):
# mps does not support float64
schedule_timesteps = self.timesteps.to(
original_samples.device, dtype=torch.float32
)
timesteps = timesteps.to(original_samples.device, dtype=torch.float32)
else:
schedule_timesteps = self.timesteps.to(original_samples.device)
timesteps = timesteps.to(original_samples.device)
# begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index
if self.begin_index is None:
step_indices = [
self.index_for_timestep(t, schedule_timesteps) for t in timesteps
]
elif self.step_index is not None:
# add_noise is called after first denoising step (for inpainting)
step_indices = [self.step_index] * timesteps.shape[0]
else:
# add noise is called before first denoising step to create initial latent(img2img)
step_indices = [self.begin_index] * timesteps.shape[0]
sigma = sigmas[step_indices].flatten()
while len(sigma.shape) < len(original_samples.shape):
sigma = sigma.unsqueeze(-1)
alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)
noisy_samples = alpha_t * original_samples + sigma_t * noise
return noisy_samples
def __len__(self):
return self.config.num_train_timesteps
@@ -0,0 +1,147 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 TSAIL Team and The HuggingFace Team. All rights reserved.
import numpy as np
import torch
def compute_unipc_bh_coefficients(
h: torch.Tensor,
rks: torch.Tensor,
order: int,
predict_x0: bool,
solver_type: str,
):
"""Build UniPC coefficients, leaving tensor assembly to each scheduler."""
R = []
b = []
hh = -h if predict_x0 else h
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
h_phi_k = h_phi_1 / hh - 1
factorial_i = 1
if solver_type == "bh1":
B_h = hh
elif solver_type == "bh2":
B_h = torch.expm1(hh)
else:
raise NotImplementedError()
for i in range(1, order + 1):
R.append(torch.pow(rks, i - 1))
b.append(h_phi_k * factorial_i / B_h)
factorial_i *= i + 1
h_phi_k = h_phi_k / hh - 1 / factorial_i
return R, b, h_phi_1, B_h
def unipc_predictor_step(
scheduler, model_output, model_output_convert, timestep, sample
):
"""Update UniPC history and order, then predict; callers own correction and indexing."""
for i in range(scheduler.config.solver_order - 1):
scheduler.model_outputs[i] = scheduler.model_outputs[i + 1]
scheduler.timestep_list[i] = scheduler.timestep_list[i + 1]
scheduler.model_outputs[-1] = model_output_convert
scheduler.timestep_list[-1] = timestep
if scheduler.config.lower_order_final:
this_order = min(
scheduler.config.solver_order,
len(scheduler.timesteps) - scheduler.step_index,
)
else:
this_order = scheduler.config.solver_order
scheduler.this_order = min(this_order, scheduler.lower_order_nums + 1)
assert scheduler.this_order > 0
scheduler.last_sample = sample
# an external solver-p consumes the original, unconverted model output
prev_sample = scheduler.multistep_uni_p_bh_update(
model_output=model_output,
sample=sample,
order=scheduler.this_order,
)
if scheduler.lower_order_nums < scheduler.config.solver_order:
scheduler.lower_order_nums += 1
return prev_sample
class UniPCSchedulerMixin:
"""Shared sample transforms and step state; subclasses own schedules and solvers."""
@property
def step_index(self):
return self._step_index
@property
def begin_index(self):
return self._begin_index
def set_begin_index(self, begin_index: int = 0):
self._begin_index = begin_index
def _init_step_index(self, timestep):
if self.begin_index is None:
if isinstance(timestep, torch.Tensor):
timestep = timestep.to(self.timesteps.device)
self._step_index = self.index_for_timestep(timestep)
else:
self._step_index = self._begin_index
def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor:
return sample
def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:
"""Apply per-sample dynamic thresholding before restoring the input dtype."""
dtype = sample.dtype
batch_size, channels, *remaining_dims = sample.shape
if dtype not in (torch.float32, torch.float64):
sample = sample.float()
sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
abs_sample = sample.abs()
s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
s = torch.clamp(s, min=1, max=self.config.sample_max_value)
s = s.unsqueeze(1)
sample = torch.clamp(sample, -s, s) / s
sample = sample.reshape(batch_size, channels, *remaining_dims)
return sample.to(dtype)
def add_noise(
self,
original_samples: torch.Tensor,
noise: torch.Tensor,
timesteps: torch.IntTensor,
) -> torch.Tensor:
sigmas = self.sigmas.to(
device=original_samples.device, dtype=original_samples.dtype
)
if original_samples.device.type == "mps" and torch.is_floating_point(timesteps):
# mps does not support float64 timesteps
schedule_timesteps = self.timesteps.to(
original_samples.device, dtype=torch.float32
)
timesteps = timesteps.to(original_samples.device, dtype=torch.float32)
else:
schedule_timesteps = self.timesteps.to(original_samples.device)
timesteps = timesteps.to(original_samples.device)
# training looks up each timestep; img2img and inpainting use the step state
if self.begin_index is None:
step_indices = [
self.index_for_timestep(t, schedule_timesteps) for t in timesteps
]
elif self.step_index is not None:
step_indices = [self.step_index] * timesteps.shape[0]
else:
step_indices = [self.begin_index] * timesteps.shape[0]
sigma = sigmas[step_indices].flatten()
while len(sigma.shape) < len(original_samples.shape):
sigma = sigma.unsqueeze(-1)
alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)
return alpha_t * original_samples + sigma_t * noise
@@ -1,6 +1,8 @@
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from torch import nn
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
@@ -75,3 +77,52 @@ def test_gemma3_maps_hf_siglip_projection_name():
mapped, _, _ = map_name("vision_tower.encoder.layers.0.self_attn.out_proj.weight")
assert mapped == "vision_tower.vision_model.encoder.layers.0.self_attn.proj.weight"
@pytest.mark.parametrize(
"shard,normalized",
[("q", 0), ("k", 1), ("v", 2), (0, "q"), (1, "k"), (2, "v"), ("0", 0), ("01", 1)],
)
@pytest.mark.parametrize("error", [AssertionError, TypeError])
def test_gemma3_shard_loader_preserves_first_attempt_and_fallback(
shard, normalized, error
):
parameter = nn.Parameter(torch.zeros(2))
weight = torch.ones(2)
for accepted in (shard, normalized):
calls = []
def loader(param, loaded, shard_id):
calls.append(shard_id)
if shard_id != accepted:
raise error("unsupported shard representation")
param.data.copy_(loaded)
gemma_3._load_with_shard_id(loader, parameter, weight, shard)
assert calls == ([shard] if accepted == shard else [shard, normalized])
torch.testing.assert_close(parameter, weight, rtol=0, atol=0)
@pytest.mark.parametrize("shard", ["unsupported", 3, None])
def test_gemma3_shard_loader_does_not_retry_unknown_ids(shard):
calls = []
def loader(param, loaded, shard_id):
calls.append(shard_id)
raise TypeError("unsupported")
with pytest.raises(TypeError, match="Unsupported shard_id=.*param=None"):
gemma_3._load_with_shard_id(
loader, nn.Parameter(torch.zeros(1)), torch.ones(1), shard
)
assert calls == [shard]
def test_gemma3_shard_loader_propagates_other_errors():
def loader(*args):
raise RuntimeError("load failure")
with pytest.raises(RuntimeError, match="load failure"):
gemma_3._load_with_shard_id(
loader, nn.Parameter(torch.zeros(1)), torch.ones(1), "q"
)
@@ -0,0 +1,71 @@
# SPDX-License-Identifier: Apache-2.0
import pytest
import torch
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_unipc_multistep import (
FlowUniPCMultistepScheduler,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_unipc_multistep import (
UniPCMultistepScheduler,
)
@pytest.mark.parametrize("cls", [FlowUniPCMultistepScheduler, UniPCMultistepScheduler])
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_unipc_sample_transforms_and_state(cls, device, dtype):
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA required")
scheduler = cls(dynamic_thresholding_ratio=0.5, sample_max_value=5.0)
restored = cls.from_config(scheduler.config)
assert dict(restored.config) == dict(scheduler.config)
x = torch.tensor([-8, -2, 0, 2, 4, 20], device=device, dtype=dtype)
x = x.repeat(2).reshape(2, 1, 2, 3).transpose(-1, -2)
original = x.clone()
expected = (x.float().clamp(-3, 3) / 3).to(dtype)
torch.testing.assert_close(scheduler._threshold_sample(x), expected, rtol=0, atol=0)
assert scheduler.scale_model_input(x, 0) is x
scheduler.timesteps = torch.tensor([90, 60, 60, 10], device=device)
scheduler.sigmas = torch.tensor([0.9, 0.6, 0.4, 0.1, 0], device=device)
timesteps = torch.tensor([60, 10])
noise = torch.ones_like(x)
for mode, indices in (
("training", [2, 3]),
("img2img", [1, 1]),
("inpaint", [2, 2]),
):
if mode != "training":
scheduler.set_begin_index(1)
if mode == "inpaint":
scheduler._step_index = 2
state = scheduler.begin_index, scheduler.step_index
sigma = scheduler.sigmas.to(dtype)[indices].reshape(2, 1, 1, 1)
if cls is FlowUniPCMultistepScheduler:
alpha = 1 - sigma
else:
alpha = 1 / ((sigma**2 + 1) ** 0.5)
sigma = sigma * alpha
expected = alpha * x + sigma * noise
torch.testing.assert_close(
scheduler.add_noise(x, noise, timesteps), expected, rtol=0, atol=0
)
assert (scheduler.begin_index, scheduler.step_index) == state
torch.testing.assert_close(x, original, rtol=0, atol=0)
scheduler.set_timesteps(5, device=device)
assert scheduler.begin_index is None and scheduler.step_index is None
scheduler._init_step_index(scheduler.timesteps[0])
assert scheduler.step_index == 0
scheduler.set_begin_index(2)
scheduler._init_step_index(scheduler.timesteps[0])
assert scheduler.step_index == 2
def test_unipc_missing_timestep_behavior_is_preserved():
regular, flow = UniPCMultistepScheduler(), FlowUniPCMultistepScheduler()
regular.set_timesteps(5)
flow.set_timesteps(5)
assert regular.index_for_timestep(-1) == 4
with pytest.raises(IndexError):
flow.index_for_timestep(-1)