From 45ad463c8793cbfd2609d0f5e775784659cd29f7 Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 9 Oct 2026 16:26:04 +0800 Subject: [PATCH] [diffusion] refactor: deduplicate UniPC solvers and Gemma3 shard loading (#42899) Co-authored-by: Mick Qian --- .../runtime/models/encoders/gemma_3.py | 103 +++----- .../scheduling_flow_unipc_multistep.py | 218 ++--------------- .../models/schedulers/scheduling_helios.py | 74 +----- .../schedulers/scheduling_unipc_multistep.py | 219 ++---------------- .../runtime/models/schedulers/unipc.py | 147 ++++++++++++ .../test/unit/test_srt_siglip_reuse.py | 51 ++++ .../test/unit/test_unipc_sample_transforms.py | 71 ++++++ 7 files changed, 341 insertions(+), 542 deletions(-) create mode 100644 python/sglang/multimodal_gen/runtime/models/schedulers/unipc.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_unipc_sample_transforms.py diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py index 63da9eda6c..5dc62b7c69 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py @@ -34,6 +34,36 @@ from sglang.srt.models.siglip import SiglipVisionModel logger = logging.getLogger(__name__) +def _load_with_shard_id( + weight_loader, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id +) -> None: + """Try the checkpoint shard id before its equivalent string or integer form.""" + try: + weight_loader(param, loaded_weight, shard_id) + return + except (AssertionError, TypeError): + pass + + if isinstance(shard_id, str): + mapping = {"q": 0, "k": 1, "v": 2} + if shard_id in mapping: + weight_loader(param, loaded_weight, mapping[shard_id]) + return + if shard_id.isdigit(): + weight_loader(param, loaded_weight, int(shard_id)) + return + elif isinstance(shard_id, int): + mapping = {0: "q", 1: "k", 2: "v"} + if shard_id in mapping: + weight_loader(param, loaded_weight, mapping[shard_id]) + return + + raise TypeError( + f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} " + f"(param={param.name})." + ) + + def get_attention_sliding_window_size(config): return config.sliding_window - 1 @@ -593,43 +623,6 @@ class Gemma3TextModel(nn.Module): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() - def _load_with_shard_id( - weight_loader, param, loaded_weight: torch.Tensor, shard_id - ) -> None: - """Call param.weight_loader with best-effort shard_id normalization. - - Different fused-QKV implementations expect different shard_id types: - - Some expect strings: "q"/"k"/"v" - - Some expect integer indices: 0/1/2 - We try the provided shard_id first, then fall back between str/int forms. - """ - try: - weight_loader(param, loaded_weight, shard_id) - return - except (AssertionError, TypeError): - pass - - # Fall back between common representations. - if isinstance(shard_id, str): - mapping = {"q": 0, "k": 1, "v": 2} - if shard_id in mapping: - weight_loader(param, loaded_weight, mapping[shard_id]) - return - if shard_id.isdigit(): - weight_loader(param, loaded_weight, int(shard_id)) - return - elif isinstance(shard_id, int): - mapping = {0: "q", 1: "k", 2: "v"} - if shard_id in mapping: - weight_loader(param, loaded_weight, mapping[shard_id]) - return - - # Re-raise with a clearer message. - raise TypeError( - f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} " - f"(param={getattr(param, 'name', '')})." - ) - stacked_params_mapping = getattr( getattr(self.config, "arch_config", object()), "stacked_params_mapping", @@ -796,42 +789,6 @@ class Gemma3ForConditionalGeneration( loaded_params: Set[str] = set() params_dict = dict(self.named_parameters()) - def _load_with_shard_id( - weight_loader, param, loaded_weight: torch.Tensor, shard_id - ) -> None: - """Call param.weight_loader with best-effort shard_id normalization. - - Different fused-QKV implementations expect different shard_id types: - - Some expect strings: "q"/"k"/"v" - - Some expect integer indices: 0/1/2 - We try the provided shard_id first, then fall back between str/int forms. - """ - try: - weight_loader(param, loaded_weight, shard_id) - return - except (AssertionError, TypeError): - pass - - # Fall back between common representations. - if isinstance(shard_id, str): - mapping = {"q": 0, "k": 1, "v": 2} - if shard_id in mapping: - weight_loader(param, loaded_weight, mapping[shard_id]) - return - if shard_id.isdigit(): - weight_loader(param, loaded_weight, int(shard_id)) - return - elif isinstance(shard_id, int): - mapping = {0: "q", 1: "k", 2: "v"} - if shard_id in mapping: - weight_loader(param, loaded_weight, mapping[shard_id]) - return - - raise TypeError( - f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} " - f"(param={getattr(param, 'name', '')})." - ) - # Separate weights language_model_weights: list[tuple[str, torch.Tensor]] = [] other_weights: list[tuple[str, torch.Tensor]] = [] diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_unipc_multistep.py b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_unipc_multistep.py index c78d3ddf7d..3b879b9b48 100644 --- a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_unipc_multistep.py +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_flow_unipc_multistep.py @@ -19,9 +19,16 @@ from diffusers.schedulers.scheduling_utils import ( from diffusers.utils import deprecate from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler +from sglang.multimodal_gen.runtime.models.schedulers.unipc import ( + UniPCSchedulerMixin, + compute_unipc_bh_coefficients, + unipc_predictor_step, +) -class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): +class FlowUniPCMultistepScheduler( + UniPCSchedulerMixin, SchedulerMixin, ConfigMixin, BaseScheduler +): """ `UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models. @@ -139,34 +146,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): BaseScheduler.__init__(self) - @property - def step_index(self): - """ - The index counter for current timestep. It will increase 1 after each scheduler step. - """ - return self._step_index - - @property - def begin_index(self): - """ - The index for the first timestep. It should be set from pipeline with `set_begin_index` method. - """ - return self._begin_index - def set_shift(self, shift: float) -> None: self.config.shift = shift - # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index - def set_begin_index(self, begin_index: int = 0): - """ - Sets the begin index for the scheduler. This function should be run from pipeline before the inference. - - Args: - begin_index (`int`): - The begin index for the scheduler. - """ - self._begin_index = begin_index - # Modified from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler.set_timesteps def set_timesteps( self, @@ -236,44 +218,6 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): self._step_index = None self._begin_index = None - # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample - def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: - """ - "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the - prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by - s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing - pixels from saturation at each step. We find that dynamic thresholding results in significantly better - photorealism as well as better image-text alignment, especially when using very large guidance weights." - - https://arxiv.org/abs/2205.11487 - """ - dtype = sample.dtype - batch_size, channels, *remaining_dims = sample.shape - - if dtype not in (torch.float32, torch.float64): - sample = ( - sample.float() - ) # upcast for quantile calculation, and clamp not implemented for cpu half - - # Flatten sample for doing quantile calculation along each image - sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) - - abs_sample = sample.abs() # "a certain percentile absolute pixel value" - - s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) - s = torch.clamp( - s, min=1, max=self.config.sample_max_value - ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] - s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 - sample = ( - torch.clamp(sample, -s, s) / s - ) # "we threshold xt0 to the range [-s, s] and then divide by s" - - sample = sample.reshape(batch_size, channels, *remaining_dims) - sample = sample.to(dtype) - - return sample - def _sigma_to_alpha_sigma_t(self, sigma) -> tuple[Any, Any]: return 1 - sigma, sigma @@ -433,27 +377,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): else: rks = torch.ones(1, device=device, dtype=h.dtype) - R = [] - b = [] - - hh = -h if self.predict_x0 else h - h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 - h_phi_k = h_phi_1 / hh - 1 - - factorial_i = 1 - - if self.config.solver_type == "bh1": - B_h = hh - elif self.config.solver_type == "bh2": - B_h = torch.expm1(hh) - else: - raise NotImplementedError() - - for i in range(1, order + 1): - R.append(torch.pow(rks, i - 1)) - b.append(h_phi_k * factorial_i / B_h) - factorial_i *= i + 1 - h_phi_k = h_phi_k / hh - 1 / factorial_i + R, b, h_phi_1, B_h = compute_unipc_bh_coefficients( + h, rks, order, self.predict_x0, self.config.solver_type + ) R = torch.stack(R) b = torch.stack(b) @@ -592,27 +518,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): rks = torch.stack(rks_list + [torch.ones_like(rks_list[0])]) D1s = torch.stack(D1s_list, dim=1) if len(D1s_list) > 0 else None - R = [] - b = [] - - hh = -h if self.predict_x0 else h - h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 - h_phi_k = h_phi_1 / hh - 1 - - factorial_i = 1 - - if self.config.solver_type == "bh1": - B_h = hh - elif self.config.solver_type == "bh2": - B_h = torch.expm1(hh) - else: - raise NotImplementedError() - - for i in range(1, order + 1): - R.append(torch.pow(rks, i - 1)) - b.append(h_phi_k * factorial_i / B_h) - factorial_i *= i + 1 - h_phi_k = h_phi_k / hh - 1 / factorial_i + R, b, h_phi_1, B_h = compute_unipc_bh_coefficients( + h, rks, order, self.predict_x0, self.config.solver_type + ) R = torch.stack(R) # Avoid torch.tensor(list_of_gpu_scalars) which syncs to host @@ -671,19 +579,6 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): return step_index - # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index - def _init_step_index(self, timestep) -> None: - """ - Initialize the step_index counter for the scheduler. - """ - - if self.begin_index is None: - if isinstance(timestep, torch.Tensor): - timestep = timestep.to(self.timesteps.device) - self._step_index = self.index_for_timestep(timestep) - else: - self._step_index = self._begin_index - def step( self, model_output: torch.Tensor, @@ -737,35 +632,10 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): order=self.this_order, ) - for i in range(self.config.solver_order - 1): - self.model_outputs[i] = self.model_outputs[i + 1] - self.timestep_list[i] = self.timestep_list[i + 1] - - self.model_outputs[-1] = model_output_convert - self.timestep_list[-1] = timestep # pyright: ignore - - if self.config.lower_order_final: - this_order = min( - self.config.solver_order, len(self.timesteps) - self.step_index - ) # pyright: ignore - else: - this_order = self.config.solver_order - - self.this_order: int = min( - this_order, self.lower_order_nums + 1 - ) # warmup for multistep - assert self.this_order > 0 - - self.last_sample = sample - prev_sample = self.multistep_uni_p_bh_update( - model_output=model_output, # pass the original non-converted model output, in case solver-p is used - sample=sample, - order=self.this_order, + prev_sample = unipc_predictor_step( + self, model_output, model_output_convert, timestep, sample ) - if self.lower_order_nums < self.config.solver_order: - self.lower_order_nums += 1 - # upon completion increase step index by one assert self._step_index is not None self._step_index += 1 # pyright: ignore @@ -775,61 +645,5 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): return SchedulerOutput(prev_sample=prev_sample) - def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor: - """ - Ensures interchangeability with schedulers that need to scale the denoising model input depending on the - current timestep. - - Args: - sample (`torch.Tensor`): - The input sample. - - Returns: - `torch.Tensor`: - A scaled input sample. - """ - return sample - - # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise - def add_noise( - self, - original_samples: torch.Tensor, - noise: torch.Tensor, - timesteps: torch.IntTensor, - ) -> torch.Tensor: - # Make sure sigmas and timesteps have the same device and dtype as original_samples - sigmas = self.sigmas.to( - device=original_samples.device, dtype=original_samples.dtype - ) - if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): - # mps does not support float64 - schedule_timesteps = self.timesteps.to( - original_samples.device, dtype=torch.float32 - ) - timesteps = timesteps.to(original_samples.device, dtype=torch.float32) - else: - schedule_timesteps = self.timesteps.to(original_samples.device) - timesteps = timesteps.to(original_samples.device) - - # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index - if self.begin_index is None: - step_indices = [ - self.index_for_timestep(t, schedule_timesteps) for t in timesteps - ] - elif self.step_index is not None: - # add_noise is called after first denoising step (for inpainting) - step_indices = [self.step_index] * timesteps.shape[0] - else: - # add noise is called before first denoising step to create initial latent(img2img) - step_indices = [self.begin_index] * timesteps.shape[0] - - sigma = sigmas[step_indices].flatten() - while len(sigma.shape) < len(original_samples.shape): - sigma = sigma.unsqueeze(-1) - - alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) - noisy_samples = alpha_t * original_samples + sigma_t * noise - return noisy_samples - EntryClass = FlowUniPCMultistepScheduler diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_helios.py b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_helios.py index a881d1a9b4..81c24e8251 100644 --- a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_helios.py +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_helios.py @@ -14,6 +14,10 @@ from dataclasses import dataclass import numpy as np import torch +from sglang.multimodal_gen.runtime.models.schedulers.unipc import ( + compute_unipc_bh_coefficients, + unipc_predictor_step, +) from sglang.multimodal_gen.runtime.platforms import current_platform @@ -406,26 +410,9 @@ class HeliosScheduler: rks.append(1.0) rks = torch.tensor(rks, device=device) - R = [] - b = [] - - hh = -h if self.predict_x0 else h - h_phi_1 = torch.expm1(hh) - h_phi_k = h_phi_1 / hh - 1 - factorial_i = 1 - - if self.config.solver_type == "bh1": - B_h = hh - elif self.config.solver_type == "bh2": - B_h = torch.expm1(hh) - else: - raise NotImplementedError() - - for i in range(1, order + 1): - R.append(torch.pow(rks, i - 1)) - b.append(h_phi_k * factorial_i / B_h) - factorial_i *= i + 1 - h_phi_k = h_phi_k / hh - 1 / factorial_i + R, b, h_phi_1, B_h = compute_unipc_bh_coefficients( + h, rks, order, self.predict_x0, self.config.solver_type + ) R = torch.stack(R) b = torch.tensor(b, device=device) @@ -497,25 +484,9 @@ class HeliosScheduler: rks.append(1.0) rks = torch.tensor(rks, device=device) - R = [] - b = [] - hh = -h if self.predict_x0 else h - h_phi_1 = torch.expm1(hh) - h_phi_k = h_phi_1 / hh - 1 - factorial_i = 1 - - if self.config.solver_type == "bh1": - B_h = hh - elif self.config.solver_type == "bh2": - B_h = torch.expm1(hh) - else: - raise NotImplementedError() - - for i in range(1, order + 1): - R.append(torch.pow(rks, i - 1)) - b.append(h_phi_k * factorial_i / B_h) - factorial_i *= i + 1 - h_phi_k = h_phi_k / hh - 1 / factorial_i + R, b, h_phi_1, B_h = compute_unipc_bh_coefficients( + h, rks, order, self.predict_x0, self.config.solver_type + ) R = torch.stack(R) b = torch.tensor(b, device=device) @@ -583,31 +554,10 @@ class HeliosScheduler: order=self.this_order, ) - for i in range(self.config.solver_order - 1): - self.model_outputs[i] = self.model_outputs[i + 1] - self.timestep_list[i] = self.timestep_list[i + 1] - self.model_outputs[-1] = model_output_convert - self.timestep_list[-1] = timestep - - if self.config.lower_order_final: - this_order = min( - self.config.solver_order, len(self.timesteps) - self.step_index - ) - else: - this_order = self.config.solver_order - self.this_order = min(this_order, self.lower_order_nums + 1) - assert self.this_order > 0 - - self.last_sample = sample - prev_sample = self.multistep_uni_p_bh_update( - model_output=model_output, - sample=sample, - order=self.this_order, + prev_sample = unipc_predictor_step( + self, model_output, model_output_convert, timestep, sample ) - if self.lower_order_nums < self.config.solver_order: - self.lower_order_nums += 1 - self._step_index += 1 if not return_dict: diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py index de84315e4b..7b0c4c4674 100644 --- a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_unipc_multistep.py @@ -38,6 +38,11 @@ from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import ( SigmaScheduleMixin, ) +from sglang.multimodal_gen.runtime.models.schedulers.unipc import ( + UniPCSchedulerMixin, + compute_unipc_bh_coefficients, + unipc_predictor_step, +) # Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar @@ -123,7 +128,7 @@ def rescale_zero_terminal_snr(betas): class UniPCMultistepScheduler( - SchedulerMixin, ConfigMixin, BaseScheduler, SigmaScheduleMixin + UniPCSchedulerMixin, SchedulerMixin, ConfigMixin, BaseScheduler, SigmaScheduleMixin ): """ `UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models. @@ -295,9 +300,6 @@ class UniPCMultistepScheduler( self.alphas_cumprod[-1] = 2**-24 # Currently we only support VP-type noise schedule - self.alpha_t = torch.sqrt(self.alphas_cumprod) - self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) - self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 # standard deviation of the initial noise distribution @@ -331,34 +333,9 @@ class UniPCMultistepScheduler( BaseScheduler.__init__(self) - @property - def step_index(self): - """ - The index counter for current timestep. It will increase 1 after each scheduler step. - """ - return self._step_index - - @property - def begin_index(self): - """ - The index for the first timestep. It should be set from pipeline with `set_begin_index` method. - """ - return self._begin_index - def set_shift(self, shift: float) -> None: self.config.flow_shift = shift - # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index - def set_begin_index(self, begin_index: int = 0): - """ - Sets the begin index for the scheduler. This function should be run from pipeline before the inference. - - Args: - begin_index (`int`): - The begin index for the scheduler. - """ - self._begin_index = begin_index - def set_timesteps( self, num_inference_steps: int, @@ -539,44 +516,6 @@ class UniPCMultistepScheduler( if device is None or torch.device(device).type == "cpu": self.sigmas = self.sigmas.to("cpu") - # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample - def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: - """ - "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the - prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by - s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing - pixels from saturation at each step. We find that dynamic thresholding results in significantly better - photorealism as well as better image-text alignment, especially when using very large guidance weights." - - https://huggingface.co/papers/2205.11487 - """ - dtype = sample.dtype - batch_size, channels, *remaining_dims = sample.shape - - if dtype not in (torch.float32, torch.float64): - sample = ( - sample.float() - ) # upcast for quantile calculation, and clamp not implemented for cpu half - - # Flatten sample for doing quantile calculation along each image - sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) - - abs_sample = sample.abs() # "a certain percentile absolute pixel value" - - s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) - s = torch.clamp( - s, min=1, max=self.config.sample_max_value - ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] - s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 - sample = ( - torch.clamp(sample, -s, s) / s - ) # "we threshold xt0 to the range [-s, s] and then divide by s" - - sample = sample.reshape(batch_size, channels, *remaining_dims) - sample = sample.to(dtype) - - return sample - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma @@ -768,27 +707,9 @@ class UniPCMultistepScheduler( rks.append(torch.ones((), dtype=h.dtype, device=h.device)) rks = torch.stack(rks).to(device=device) - R = [] - b = [] - - hh = -h if self.predict_x0 else h - h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 - h_phi_k = h_phi_1 / hh - 1 - - factorial_i = 1 - - if self.config.solver_type == "bh1": - B_h = hh - elif self.config.solver_type == "bh2": - B_h = torch.expm1(hh) - else: - raise NotImplementedError() - - for i in range(1, order + 1): - R.append(torch.pow(rks, i - 1)) - b.append(h_phi_k * factorial_i / B_h) - factorial_i *= i + 1 - h_phi_k = h_phi_k / hh - 1 / factorial_i + R, b, h_phi_1, B_h = compute_unipc_bh_coefficients( + h, rks, order, self.predict_x0, self.config.solver_type + ) R = torch.stack(R) b = torch.stack(b).to(device=device) @@ -906,27 +827,9 @@ class UniPCMultistepScheduler( rks.append(torch.ones((), dtype=h.dtype, device=h.device)) rks = torch.stack(rks).to(device=device) - R = [] - b = [] - - hh = -h if self.predict_x0 else h - h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 - h_phi_k = h_phi_1 / hh - 1 - - factorial_i = 1 - - if self.config.solver_type == "bh1": - B_h = hh - elif self.config.solver_type == "bh2": - B_h = torch.expm1(hh) - else: - raise NotImplementedError() - - for i in range(1, order + 1): - R.append(torch.pow(rks, i - 1)) - b.append(h_phi_k * factorial_i / B_h) - factorial_i *= i + 1 - h_phi_k = h_phi_k / hh - 1 / factorial_i + R, b, h_phi_1, B_h = compute_unipc_bh_coefficients( + h, rks, order, self.predict_x0, self.config.solver_type + ) R = torch.stack(R) b = torch.stack(b).to(device=device) @@ -981,19 +884,6 @@ class UniPCMultistepScheduler( return step_index - # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index - def _init_step_index(self, timestep): - """ - Initialize the step_index counter for the scheduler. - """ - - if self.begin_index is None: - if isinstance(timestep, torch.Tensor): - timestep = timestep.to(self.timesteps.device) - self._step_index = self.index_for_timestep(timestep) - else: - self._step_index = self._begin_index - def step( self, model_output: torch.Tensor, @@ -1044,35 +934,10 @@ class UniPCMultistepScheduler( order=self.this_order, ) - for i in range(self.config.solver_order - 1): - self.model_outputs[i] = self.model_outputs[i + 1] - self.timestep_list[i] = self.timestep_list[i + 1] - - self.model_outputs[-1] = model_output_convert - self.timestep_list[-1] = timestep - - if self.config.lower_order_final: - this_order = min( - self.config.solver_order, len(self.timesteps) - self.step_index - ) - else: - this_order = self.config.solver_order - - self.this_order = min( - this_order, self.lower_order_nums + 1 - ) # warmup for multistep - assert self.this_order > 0 - - self.last_sample = sample - prev_sample = self.multistep_uni_p_bh_update( - model_output=model_output, # pass the original non-converted model output, in case solver-p is used - sample=sample, - order=self.this_order, + prev_sample = unipc_predictor_step( + self, model_output, model_output_convert, timestep, sample ) - if self.lower_order_nums < self.config.solver_order: - self.lower_order_nums += 1 - # upon completion increase step index by one self._step_index += 1 @@ -1081,62 +946,6 @@ class UniPCMultistepScheduler( return SchedulerOutput(prev_sample=prev_sample) - def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor: - """ - Ensures interchangeability with schedulers that need to scale the denoising model input depending on the - current timestep. - - Args: - sample (`torch.Tensor`): - The input sample. - - Returns: - `torch.Tensor`: - A scaled input sample. - """ - return sample - - # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise - def add_noise( - self, - original_samples: torch.Tensor, - noise: torch.Tensor, - timesteps: torch.IntTensor, - ) -> torch.Tensor: - # Make sure sigmas and timesteps have the same device and dtype as original_samples - sigmas = self.sigmas.to( - device=original_samples.device, dtype=original_samples.dtype - ) - if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): - # mps does not support float64 - schedule_timesteps = self.timesteps.to( - original_samples.device, dtype=torch.float32 - ) - timesteps = timesteps.to(original_samples.device, dtype=torch.float32) - else: - schedule_timesteps = self.timesteps.to(original_samples.device) - timesteps = timesteps.to(original_samples.device) - - # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index - if self.begin_index is None: - step_indices = [ - self.index_for_timestep(t, schedule_timesteps) for t in timesteps - ] - elif self.step_index is not None: - # add_noise is called after first denoising step (for inpainting) - step_indices = [self.step_index] * timesteps.shape[0] - else: - # add noise is called before first denoising step to create initial latent(img2img) - step_indices = [self.begin_index] * timesteps.shape[0] - - sigma = sigmas[step_indices].flatten() - while len(sigma.shape) < len(original_samples.shape): - sigma = sigma.unsqueeze(-1) - - alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) - noisy_samples = alpha_t * original_samples + sigma_t * noise - return noisy_samples - def __len__(self): return self.config.num_train_timesteps diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/unipc.py b/python/sglang/multimodal_gen/runtime/models/schedulers/unipc.py new file mode 100644 index 0000000000..ebd51534f2 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/unipc.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2025 TSAIL Team and The HuggingFace Team. All rights reserved. + +import numpy as np +import torch + + +def compute_unipc_bh_coefficients( + h: torch.Tensor, + rks: torch.Tensor, + order: int, + predict_x0: bool, + solver_type: str, +): + """Build UniPC coefficients, leaving tensor assembly to each scheduler.""" + R = [] + b = [] + + hh = -h if predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + factorial_i = 1 + + if solver_type == "bh1": + B_h = hh + elif solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + return R, b, h_phi_1, B_h + + +def unipc_predictor_step( + scheduler, model_output, model_output_convert, timestep, sample +): + """Update UniPC history and order, then predict; callers own correction and indexing.""" + for i in range(scheduler.config.solver_order - 1): + scheduler.model_outputs[i] = scheduler.model_outputs[i + 1] + scheduler.timestep_list[i] = scheduler.timestep_list[i + 1] + scheduler.model_outputs[-1] = model_output_convert + scheduler.timestep_list[-1] = timestep + + if scheduler.config.lower_order_final: + this_order = min( + scheduler.config.solver_order, + len(scheduler.timesteps) - scheduler.step_index, + ) + else: + this_order = scheduler.config.solver_order + scheduler.this_order = min(this_order, scheduler.lower_order_nums + 1) + assert scheduler.this_order > 0 + + scheduler.last_sample = sample + # an external solver-p consumes the original, unconverted model output + prev_sample = scheduler.multistep_uni_p_bh_update( + model_output=model_output, + sample=sample, + order=scheduler.this_order, + ) + if scheduler.lower_order_nums < scheduler.config.solver_order: + scheduler.lower_order_nums += 1 + return prev_sample + + +class UniPCSchedulerMixin: + """Shared sample transforms and step state; subclasses own schedules and solvers.""" + + @property + def step_index(self): + return self._step_index + + @property + def begin_index(self): + return self._begin_index + + def set_begin_index(self, begin_index: int = 0): + self._begin_index = begin_index + + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor: + return sample + + def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: + """Apply per-sample dynamic thresholding before restoring the input dtype.""" + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + if dtype not in (torch.float32, torch.float64): + sample = sample.float() + + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + abs_sample = sample.abs() + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp(s, min=1, max=self.config.sample_max_value) + s = s.unsqueeze(1) + sample = torch.clamp(sample, -s, s) / s + sample = sample.reshape(batch_size, channels, *remaining_dims) + return sample.to(dtype) + + def add_noise( + self, + original_samples: torch.Tensor, + noise: torch.Tensor, + timesteps: torch.IntTensor, + ) -> torch.Tensor: + sigmas = self.sigmas.to( + device=original_samples.device, dtype=original_samples.dtype + ) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 timesteps + schedule_timesteps = self.timesteps.to( + original_samples.device, dtype=torch.float32 + ) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # training looks up each timestep; img2img and inpainting use the step state + if self.begin_index is None: + step_indices = [ + self.index_for_timestep(t, schedule_timesteps) for t in timesteps + ] + elif self.step_index is not None: + step_indices = [self.step_index] * timesteps.shape[0] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + return alpha_t * original_samples + sigma_t * noise diff --git a/python/sglang/multimodal_gen/test/unit/test_srt_siglip_reuse.py b/python/sglang/multimodal_gen/test/unit/test_srt_siglip_reuse.py index 6d7a6b1622..ba16de6c8c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_srt_siglip_reuse.py +++ b/python/sglang/multimodal_gen/test/unit/test_srt_siglip_reuse.py @@ -1,6 +1,8 @@ from types import SimpleNamespace from unittest.mock import patch +import pytest +import torch from torch import nn from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping @@ -75,3 +77,52 @@ def test_gemma3_maps_hf_siglip_projection_name(): mapped, _, _ = map_name("vision_tower.encoder.layers.0.self_attn.out_proj.weight") assert mapped == "vision_tower.vision_model.encoder.layers.0.self_attn.proj.weight" + + +@pytest.mark.parametrize( + "shard,normalized", + [("q", 0), ("k", 1), ("v", 2), (0, "q"), (1, "k"), (2, "v"), ("0", 0), ("01", 1)], +) +@pytest.mark.parametrize("error", [AssertionError, TypeError]) +def test_gemma3_shard_loader_preserves_first_attempt_and_fallback( + shard, normalized, error +): + parameter = nn.Parameter(torch.zeros(2)) + weight = torch.ones(2) + for accepted in (shard, normalized): + calls = [] + + def loader(param, loaded, shard_id): + calls.append(shard_id) + if shard_id != accepted: + raise error("unsupported shard representation") + param.data.copy_(loaded) + + gemma_3._load_with_shard_id(loader, parameter, weight, shard) + assert calls == ([shard] if accepted == shard else [shard, normalized]) + torch.testing.assert_close(parameter, weight, rtol=0, atol=0) + + +@pytest.mark.parametrize("shard", ["unsupported", 3, None]) +def test_gemma3_shard_loader_does_not_retry_unknown_ids(shard): + calls = [] + + def loader(param, loaded, shard_id): + calls.append(shard_id) + raise TypeError("unsupported") + + with pytest.raises(TypeError, match="Unsupported shard_id=.*param=None"): + gemma_3._load_with_shard_id( + loader, nn.Parameter(torch.zeros(1)), torch.ones(1), shard + ) + assert calls == [shard] + + +def test_gemma3_shard_loader_propagates_other_errors(): + def loader(*args): + raise RuntimeError("load failure") + + with pytest.raises(RuntimeError, match="load failure"): + gemma_3._load_with_shard_id( + loader, nn.Parameter(torch.zeros(1)), torch.ones(1), "q" + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_unipc_sample_transforms.py b/python/sglang/multimodal_gen/test/unit/test_unipc_sample_transforms.py new file mode 100644 index 0000000000..ec07f58d0b --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_unipc_sample_transforms.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import torch + +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_unipc_multistep import ( + UniPCMultistepScheduler, +) + + +@pytest.mark.parametrize("cls", [FlowUniPCMultistepScheduler, UniPCMultistepScheduler]) +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_unipc_sample_transforms_and_state(cls, device, dtype): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA required") + scheduler = cls(dynamic_thresholding_ratio=0.5, sample_max_value=5.0) + restored = cls.from_config(scheduler.config) + assert dict(restored.config) == dict(scheduler.config) + x = torch.tensor([-8, -2, 0, 2, 4, 20], device=device, dtype=dtype) + x = x.repeat(2).reshape(2, 1, 2, 3).transpose(-1, -2) + original = x.clone() + expected = (x.float().clamp(-3, 3) / 3).to(dtype) + torch.testing.assert_close(scheduler._threshold_sample(x), expected, rtol=0, atol=0) + assert scheduler.scale_model_input(x, 0) is x + + scheduler.timesteps = torch.tensor([90, 60, 60, 10], device=device) + scheduler.sigmas = torch.tensor([0.9, 0.6, 0.4, 0.1, 0], device=device) + timesteps = torch.tensor([60, 10]) + noise = torch.ones_like(x) + for mode, indices in ( + ("training", [2, 3]), + ("img2img", [1, 1]), + ("inpaint", [2, 2]), + ): + if mode != "training": + scheduler.set_begin_index(1) + if mode == "inpaint": + scheduler._step_index = 2 + state = scheduler.begin_index, scheduler.step_index + sigma = scheduler.sigmas.to(dtype)[indices].reshape(2, 1, 1, 1) + if cls is FlowUniPCMultistepScheduler: + alpha = 1 - sigma + else: + alpha = 1 / ((sigma**2 + 1) ** 0.5) + sigma = sigma * alpha + expected = alpha * x + sigma * noise + torch.testing.assert_close( + scheduler.add_noise(x, noise, timesteps), expected, rtol=0, atol=0 + ) + assert (scheduler.begin_index, scheduler.step_index) == state + torch.testing.assert_close(x, original, rtol=0, atol=0) + scheduler.set_timesteps(5, device=device) + assert scheduler.begin_index is None and scheduler.step_index is None + scheduler._init_step_index(scheduler.timesteps[0]) + assert scheduler.step_index == 0 + scheduler.set_begin_index(2) + scheduler._init_step_index(scheduler.timesteps[0]) + assert scheduler.step_index == 2 + + +def test_unipc_missing_timestep_behavior_is_preserved(): + regular, flow = UniPCMultistepScheduler(), FlowUniPCMultistepScheduler() + regular.set_timesteps(5) + flow.set_timesteps(5) + assert regular.index_for_timestep(-1) == 4 + with pytest.raises(IndexError): + flow.index_for_timestep(-1)