[diffusion] refactor: deduplicate UniPC solvers and Gemma3 shard loading (#42899)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -34,6 +34,36 @@ from sglang.srt.models.siglip import SiglipVisionModel
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _load_with_shard_id(
|
||||
weight_loader, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id
|
||||
) -> None:
|
||||
"""Try the checkpoint shard id before its equivalent string or integer form."""
|
||||
try:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
return
|
||||
except (AssertionError, TypeError):
|
||||
pass
|
||||
|
||||
if isinstance(shard_id, str):
|
||||
mapping = {"q": 0, "k": 1, "v": 2}
|
||||
if shard_id in mapping:
|
||||
weight_loader(param, loaded_weight, mapping[shard_id])
|
||||
return
|
||||
if shard_id.isdigit():
|
||||
weight_loader(param, loaded_weight, int(shard_id))
|
||||
return
|
||||
elif isinstance(shard_id, int):
|
||||
mapping = {0: "q", 1: "k", 2: "v"}
|
||||
if shard_id in mapping:
|
||||
weight_loader(param, loaded_weight, mapping[shard_id])
|
||||
return
|
||||
|
||||
raise TypeError(
|
||||
f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} "
|
||||
f"(param={param.name})."
|
||||
)
|
||||
|
||||
|
||||
def get_attention_sliding_window_size(config):
|
||||
return config.sliding_window - 1
|
||||
|
||||
@@ -593,43 +623,6 @@ class Gemma3TextModel(nn.Module):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
def _load_with_shard_id(
|
||||
weight_loader, param, loaded_weight: torch.Tensor, shard_id
|
||||
) -> None:
|
||||
"""Call param.weight_loader with best-effort shard_id normalization.
|
||||
|
||||
Different fused-QKV implementations expect different shard_id types:
|
||||
- Some expect strings: "q"/"k"/"v"
|
||||
- Some expect integer indices: 0/1/2
|
||||
We try the provided shard_id first, then fall back between str/int forms.
|
||||
"""
|
||||
try:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
return
|
||||
except (AssertionError, TypeError):
|
||||
pass
|
||||
|
||||
# Fall back between common representations.
|
||||
if isinstance(shard_id, str):
|
||||
mapping = {"q": 0, "k": 1, "v": 2}
|
||||
if shard_id in mapping:
|
||||
weight_loader(param, loaded_weight, mapping[shard_id])
|
||||
return
|
||||
if shard_id.isdigit():
|
||||
weight_loader(param, loaded_weight, int(shard_id))
|
||||
return
|
||||
elif isinstance(shard_id, int):
|
||||
mapping = {0: "q", 1: "k", 2: "v"}
|
||||
if shard_id in mapping:
|
||||
weight_loader(param, loaded_weight, mapping[shard_id])
|
||||
return
|
||||
|
||||
# Re-raise with a clearer message.
|
||||
raise TypeError(
|
||||
f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} "
|
||||
f"(param={getattr(param, 'name', '<param>')})."
|
||||
)
|
||||
|
||||
stacked_params_mapping = getattr(
|
||||
getattr(self.config, "arch_config", object()),
|
||||
"stacked_params_mapping",
|
||||
@@ -796,42 +789,6 @@ class Gemma3ForConditionalGeneration(
|
||||
loaded_params: Set[str] = set()
|
||||
params_dict = dict(self.named_parameters())
|
||||
|
||||
def _load_with_shard_id(
|
||||
weight_loader, param, loaded_weight: torch.Tensor, shard_id
|
||||
) -> None:
|
||||
"""Call param.weight_loader with best-effort shard_id normalization.
|
||||
|
||||
Different fused-QKV implementations expect different shard_id types:
|
||||
- Some expect strings: "q"/"k"/"v"
|
||||
- Some expect integer indices: 0/1/2
|
||||
We try the provided shard_id first, then fall back between str/int forms.
|
||||
"""
|
||||
try:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
return
|
||||
except (AssertionError, TypeError):
|
||||
pass
|
||||
|
||||
# Fall back between common representations.
|
||||
if isinstance(shard_id, str):
|
||||
mapping = {"q": 0, "k": 1, "v": 2}
|
||||
if shard_id in mapping:
|
||||
weight_loader(param, loaded_weight, mapping[shard_id])
|
||||
return
|
||||
if shard_id.isdigit():
|
||||
weight_loader(param, loaded_weight, int(shard_id))
|
||||
return
|
||||
elif isinstance(shard_id, int):
|
||||
mapping = {0: "q", 1: "k", 2: "v"}
|
||||
if shard_id in mapping:
|
||||
weight_loader(param, loaded_weight, mapping[shard_id])
|
||||
return
|
||||
|
||||
raise TypeError(
|
||||
f"Unsupported shard_id={shard_id!r} for weight_loader={weight_loader} "
|
||||
f"(param={getattr(param, 'name', '<param>')})."
|
||||
)
|
||||
|
||||
# Separate weights
|
||||
language_model_weights: list[tuple[str, torch.Tensor]] = []
|
||||
other_weights: list[tuple[str, torch.Tensor]] = []
|
||||
|
||||
+16
-202
@@ -19,9 +19,16 @@ from diffusers.schedulers.scheduling_utils import (
|
||||
from diffusers.utils import deprecate
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.unipc import (
|
||||
UniPCSchedulerMixin,
|
||||
compute_unipc_bh_coefficients,
|
||||
unipc_predictor_step,
|
||||
)
|
||||
|
||||
|
||||
class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
class FlowUniPCMultistepScheduler(
|
||||
UniPCSchedulerMixin, SchedulerMixin, ConfigMixin, BaseScheduler
|
||||
):
|
||||
"""
|
||||
`UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models.
|
||||
|
||||
@@ -139,34 +146,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
|
||||
BaseScheduler.__init__(self)
|
||||
|
||||
@property
|
||||
def step_index(self):
|
||||
"""
|
||||
The index counter for current timestep. It will increase 1 after each scheduler step.
|
||||
"""
|
||||
return self._step_index
|
||||
|
||||
@property
|
||||
def begin_index(self):
|
||||
"""
|
||||
The index for the first timestep. It should be set from pipeline with `set_begin_index` method.
|
||||
"""
|
||||
return self._begin_index
|
||||
|
||||
def set_shift(self, shift: float) -> None:
|
||||
self.config.shift = shift
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index
|
||||
def set_begin_index(self, begin_index: int = 0):
|
||||
"""
|
||||
Sets the begin index for the scheduler. This function should be run from pipeline before the inference.
|
||||
|
||||
Args:
|
||||
begin_index (`int`):
|
||||
The begin index for the scheduler.
|
||||
"""
|
||||
self._begin_index = begin_index
|
||||
|
||||
# Modified from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler.set_timesteps
|
||||
def set_timesteps(
|
||||
self,
|
||||
@@ -236,44 +218,6 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
self._step_index = None
|
||||
self._begin_index = None
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample
|
||||
def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
"Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the
|
||||
prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by
|
||||
s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing
|
||||
pixels from saturation at each step. We find that dynamic thresholding results in significantly better
|
||||
photorealism as well as better image-text alignment, especially when using very large guidance weights."
|
||||
|
||||
https://arxiv.org/abs/2205.11487
|
||||
"""
|
||||
dtype = sample.dtype
|
||||
batch_size, channels, *remaining_dims = sample.shape
|
||||
|
||||
if dtype not in (torch.float32, torch.float64):
|
||||
sample = (
|
||||
sample.float()
|
||||
) # upcast for quantile calculation, and clamp not implemented for cpu half
|
||||
|
||||
# Flatten sample for doing quantile calculation along each image
|
||||
sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
|
||||
|
||||
abs_sample = sample.abs() # "a certain percentile absolute pixel value"
|
||||
|
||||
s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
|
||||
s = torch.clamp(
|
||||
s, min=1, max=self.config.sample_max_value
|
||||
) # When clamped to min=1, equivalent to standard clipping to [-1, 1]
|
||||
s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0
|
||||
sample = (
|
||||
torch.clamp(sample, -s, s) / s
|
||||
) # "we threshold xt0 to the range [-s, s] and then divide by s"
|
||||
|
||||
sample = sample.reshape(batch_size, channels, *remaining_dims)
|
||||
sample = sample.to(dtype)
|
||||
|
||||
return sample
|
||||
|
||||
def _sigma_to_alpha_sigma_t(self, sigma) -> tuple[Any, Any]:
|
||||
return 1 - sigma, sigma
|
||||
|
||||
@@ -433,27 +377,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
else:
|
||||
rks = torch.ones(1, device=device, dtype=h.dtype)
|
||||
|
||||
R = []
|
||||
b = []
|
||||
|
||||
hh = -h if self.predict_x0 else h
|
||||
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
|
||||
h_phi_k = h_phi_1 / hh - 1
|
||||
|
||||
factorial_i = 1
|
||||
|
||||
if self.config.solver_type == "bh1":
|
||||
B_h = hh
|
||||
elif self.config.solver_type == "bh2":
|
||||
B_h = torch.expm1(hh)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
for i in range(1, order + 1):
|
||||
R.append(torch.pow(rks, i - 1))
|
||||
b.append(h_phi_k * factorial_i / B_h)
|
||||
factorial_i *= i + 1
|
||||
h_phi_k = h_phi_k / hh - 1 / factorial_i
|
||||
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
|
||||
h, rks, order, self.predict_x0, self.config.solver_type
|
||||
)
|
||||
|
||||
R = torch.stack(R)
|
||||
b = torch.stack(b)
|
||||
@@ -592,27 +518,9 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
rks = torch.stack(rks_list + [torch.ones_like(rks_list[0])])
|
||||
D1s = torch.stack(D1s_list, dim=1) if len(D1s_list) > 0 else None
|
||||
|
||||
R = []
|
||||
b = []
|
||||
|
||||
hh = -h if self.predict_x0 else h
|
||||
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
|
||||
h_phi_k = h_phi_1 / hh - 1
|
||||
|
||||
factorial_i = 1
|
||||
|
||||
if self.config.solver_type == "bh1":
|
||||
B_h = hh
|
||||
elif self.config.solver_type == "bh2":
|
||||
B_h = torch.expm1(hh)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
for i in range(1, order + 1):
|
||||
R.append(torch.pow(rks, i - 1))
|
||||
b.append(h_phi_k * factorial_i / B_h)
|
||||
factorial_i *= i + 1
|
||||
h_phi_k = h_phi_k / hh - 1 / factorial_i
|
||||
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
|
||||
h, rks, order, self.predict_x0, self.config.solver_type
|
||||
)
|
||||
|
||||
R = torch.stack(R)
|
||||
# Avoid torch.tensor(list_of_gpu_scalars) which syncs to host
|
||||
@@ -671,19 +579,6 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
|
||||
return step_index
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index
|
||||
def _init_step_index(self, timestep) -> None:
|
||||
"""
|
||||
Initialize the step_index counter for the scheduler.
|
||||
"""
|
||||
|
||||
if self.begin_index is None:
|
||||
if isinstance(timestep, torch.Tensor):
|
||||
timestep = timestep.to(self.timesteps.device)
|
||||
self._step_index = self.index_for_timestep(timestep)
|
||||
else:
|
||||
self._step_index = self._begin_index
|
||||
|
||||
def step(
|
||||
self,
|
||||
model_output: torch.Tensor,
|
||||
@@ -737,35 +632,10 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
order=self.this_order,
|
||||
)
|
||||
|
||||
for i in range(self.config.solver_order - 1):
|
||||
self.model_outputs[i] = self.model_outputs[i + 1]
|
||||
self.timestep_list[i] = self.timestep_list[i + 1]
|
||||
|
||||
self.model_outputs[-1] = model_output_convert
|
||||
self.timestep_list[-1] = timestep # pyright: ignore
|
||||
|
||||
if self.config.lower_order_final:
|
||||
this_order = min(
|
||||
self.config.solver_order, len(self.timesteps) - self.step_index
|
||||
) # pyright: ignore
|
||||
else:
|
||||
this_order = self.config.solver_order
|
||||
|
||||
self.this_order: int = min(
|
||||
this_order, self.lower_order_nums + 1
|
||||
) # warmup for multistep
|
||||
assert self.this_order > 0
|
||||
|
||||
self.last_sample = sample
|
||||
prev_sample = self.multistep_uni_p_bh_update(
|
||||
model_output=model_output, # pass the original non-converted model output, in case solver-p is used
|
||||
sample=sample,
|
||||
order=self.this_order,
|
||||
prev_sample = unipc_predictor_step(
|
||||
self, model_output, model_output_convert, timestep, sample
|
||||
)
|
||||
|
||||
if self.lower_order_nums < self.config.solver_order:
|
||||
self.lower_order_nums += 1
|
||||
|
||||
# upon completion increase step index by one
|
||||
assert self._step_index is not None
|
||||
self._step_index += 1 # pyright: ignore
|
||||
@@ -775,61 +645,5 @@ class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
|
||||
return SchedulerOutput(prev_sample=prev_sample)
|
||||
|
||||
def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor:
|
||||
"""
|
||||
Ensures interchangeability with schedulers that need to scale the denoising model input depending on the
|
||||
current timestep.
|
||||
|
||||
Args:
|
||||
sample (`torch.Tensor`):
|
||||
The input sample.
|
||||
|
||||
Returns:
|
||||
`torch.Tensor`:
|
||||
A scaled input sample.
|
||||
"""
|
||||
return sample
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise
|
||||
def add_noise(
|
||||
self,
|
||||
original_samples: torch.Tensor,
|
||||
noise: torch.Tensor,
|
||||
timesteps: torch.IntTensor,
|
||||
) -> torch.Tensor:
|
||||
# Make sure sigmas and timesteps have the same device and dtype as original_samples
|
||||
sigmas = self.sigmas.to(
|
||||
device=original_samples.device, dtype=original_samples.dtype
|
||||
)
|
||||
if original_samples.device.type == "mps" and torch.is_floating_point(timesteps):
|
||||
# mps does not support float64
|
||||
schedule_timesteps = self.timesteps.to(
|
||||
original_samples.device, dtype=torch.float32
|
||||
)
|
||||
timesteps = timesteps.to(original_samples.device, dtype=torch.float32)
|
||||
else:
|
||||
schedule_timesteps = self.timesteps.to(original_samples.device)
|
||||
timesteps = timesteps.to(original_samples.device)
|
||||
|
||||
# begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index
|
||||
if self.begin_index is None:
|
||||
step_indices = [
|
||||
self.index_for_timestep(t, schedule_timesteps) for t in timesteps
|
||||
]
|
||||
elif self.step_index is not None:
|
||||
# add_noise is called after first denoising step (for inpainting)
|
||||
step_indices = [self.step_index] * timesteps.shape[0]
|
||||
else:
|
||||
# add noise is called before first denoising step to create initial latent(img2img)
|
||||
step_indices = [self.begin_index] * timesteps.shape[0]
|
||||
|
||||
sigma = sigmas[step_indices].flatten()
|
||||
while len(sigma.shape) < len(original_samples.shape):
|
||||
sigma = sigma.unsqueeze(-1)
|
||||
|
||||
alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)
|
||||
noisy_samples = alpha_t * original_samples + sigma_t * noise
|
||||
return noisy_samples
|
||||
|
||||
|
||||
EntryClass = FlowUniPCMultistepScheduler
|
||||
|
||||
@@ -14,6 +14,10 @@ from dataclasses import dataclass
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.unipc import (
|
||||
compute_unipc_bh_coefficients,
|
||||
unipc_predictor_step,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
|
||||
|
||||
@@ -406,26 +410,9 @@ class HeliosScheduler:
|
||||
rks.append(1.0)
|
||||
rks = torch.tensor(rks, device=device)
|
||||
|
||||
R = []
|
||||
b = []
|
||||
|
||||
hh = -h if self.predict_x0 else h
|
||||
h_phi_1 = torch.expm1(hh)
|
||||
h_phi_k = h_phi_1 / hh - 1
|
||||
factorial_i = 1
|
||||
|
||||
if self.config.solver_type == "bh1":
|
||||
B_h = hh
|
||||
elif self.config.solver_type == "bh2":
|
||||
B_h = torch.expm1(hh)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
for i in range(1, order + 1):
|
||||
R.append(torch.pow(rks, i - 1))
|
||||
b.append(h_phi_k * factorial_i / B_h)
|
||||
factorial_i *= i + 1
|
||||
h_phi_k = h_phi_k / hh - 1 / factorial_i
|
||||
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
|
||||
h, rks, order, self.predict_x0, self.config.solver_type
|
||||
)
|
||||
|
||||
R = torch.stack(R)
|
||||
b = torch.tensor(b, device=device)
|
||||
@@ -497,25 +484,9 @@ class HeliosScheduler:
|
||||
rks.append(1.0)
|
||||
rks = torch.tensor(rks, device=device)
|
||||
|
||||
R = []
|
||||
b = []
|
||||
hh = -h if self.predict_x0 else h
|
||||
h_phi_1 = torch.expm1(hh)
|
||||
h_phi_k = h_phi_1 / hh - 1
|
||||
factorial_i = 1
|
||||
|
||||
if self.config.solver_type == "bh1":
|
||||
B_h = hh
|
||||
elif self.config.solver_type == "bh2":
|
||||
B_h = torch.expm1(hh)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
for i in range(1, order + 1):
|
||||
R.append(torch.pow(rks, i - 1))
|
||||
b.append(h_phi_k * factorial_i / B_h)
|
||||
factorial_i *= i + 1
|
||||
h_phi_k = h_phi_k / hh - 1 / factorial_i
|
||||
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
|
||||
h, rks, order, self.predict_x0, self.config.solver_type
|
||||
)
|
||||
|
||||
R = torch.stack(R)
|
||||
b = torch.tensor(b, device=device)
|
||||
@@ -583,31 +554,10 @@ class HeliosScheduler:
|
||||
order=self.this_order,
|
||||
)
|
||||
|
||||
for i in range(self.config.solver_order - 1):
|
||||
self.model_outputs[i] = self.model_outputs[i + 1]
|
||||
self.timestep_list[i] = self.timestep_list[i + 1]
|
||||
self.model_outputs[-1] = model_output_convert
|
||||
self.timestep_list[-1] = timestep
|
||||
|
||||
if self.config.lower_order_final:
|
||||
this_order = min(
|
||||
self.config.solver_order, len(self.timesteps) - self.step_index
|
||||
)
|
||||
else:
|
||||
this_order = self.config.solver_order
|
||||
self.this_order = min(this_order, self.lower_order_nums + 1)
|
||||
assert self.this_order > 0
|
||||
|
||||
self.last_sample = sample
|
||||
prev_sample = self.multistep_uni_p_bh_update(
|
||||
model_output=model_output,
|
||||
sample=sample,
|
||||
order=self.this_order,
|
||||
prev_sample = unipc_predictor_step(
|
||||
self, model_output, model_output_convert, timestep, sample
|
||||
)
|
||||
|
||||
if self.lower_order_nums < self.config.solver_order:
|
||||
self.lower_order_nums += 1
|
||||
|
||||
self._step_index += 1
|
||||
|
||||
if not return_dict:
|
||||
|
||||
+14
-205
@@ -38,6 +38,11 @@ from sglang.multimodal_gen.runtime.models.schedulers.base import BaseScheduler
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.sigma_schedule import (
|
||||
SigmaScheduleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.unipc import (
|
||||
UniPCSchedulerMixin,
|
||||
compute_unipc_bh_coefficients,
|
||||
unipc_predictor_step,
|
||||
)
|
||||
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar
|
||||
@@ -123,7 +128,7 @@ def rescale_zero_terminal_snr(betas):
|
||||
|
||||
|
||||
class UniPCMultistepScheduler(
|
||||
SchedulerMixin, ConfigMixin, BaseScheduler, SigmaScheduleMixin
|
||||
UniPCSchedulerMixin, SchedulerMixin, ConfigMixin, BaseScheduler, SigmaScheduleMixin
|
||||
):
|
||||
"""
|
||||
`UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models.
|
||||
@@ -295,9 +300,6 @@ class UniPCMultistepScheduler(
|
||||
self.alphas_cumprod[-1] = 2**-24
|
||||
|
||||
# Currently we only support VP-type noise schedule
|
||||
self.alpha_t = torch.sqrt(self.alphas_cumprod)
|
||||
self.sigma_t = torch.sqrt(1 - self.alphas_cumprod)
|
||||
self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t)
|
||||
self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5
|
||||
|
||||
# standard deviation of the initial noise distribution
|
||||
@@ -331,34 +333,9 @@ class UniPCMultistepScheduler(
|
||||
|
||||
BaseScheduler.__init__(self)
|
||||
|
||||
@property
|
||||
def step_index(self):
|
||||
"""
|
||||
The index counter for current timestep. It will increase 1 after each scheduler step.
|
||||
"""
|
||||
return self._step_index
|
||||
|
||||
@property
|
||||
def begin_index(self):
|
||||
"""
|
||||
The index for the first timestep. It should be set from pipeline with `set_begin_index` method.
|
||||
"""
|
||||
return self._begin_index
|
||||
|
||||
def set_shift(self, shift: float) -> None:
|
||||
self.config.flow_shift = shift
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index
|
||||
def set_begin_index(self, begin_index: int = 0):
|
||||
"""
|
||||
Sets the begin index for the scheduler. This function should be run from pipeline before the inference.
|
||||
|
||||
Args:
|
||||
begin_index (`int`):
|
||||
The begin index for the scheduler.
|
||||
"""
|
||||
self._begin_index = begin_index
|
||||
|
||||
def set_timesteps(
|
||||
self,
|
||||
num_inference_steps: int,
|
||||
@@ -539,44 +516,6 @@ class UniPCMultistepScheduler(
|
||||
if device is None or torch.device(device).type == "cpu":
|
||||
self.sigmas = self.sigmas.to("cpu")
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample
|
||||
def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
"Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the
|
||||
prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by
|
||||
s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing
|
||||
pixels from saturation at each step. We find that dynamic thresholding results in significantly better
|
||||
photorealism as well as better image-text alignment, especially when using very large guidance weights."
|
||||
|
||||
https://huggingface.co/papers/2205.11487
|
||||
"""
|
||||
dtype = sample.dtype
|
||||
batch_size, channels, *remaining_dims = sample.shape
|
||||
|
||||
if dtype not in (torch.float32, torch.float64):
|
||||
sample = (
|
||||
sample.float()
|
||||
) # upcast for quantile calculation, and clamp not implemented for cpu half
|
||||
|
||||
# Flatten sample for doing quantile calculation along each image
|
||||
sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
|
||||
|
||||
abs_sample = sample.abs() # "a certain percentile absolute pixel value"
|
||||
|
||||
s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
|
||||
s = torch.clamp(
|
||||
s, min=1, max=self.config.sample_max_value
|
||||
) # When clamped to min=1, equivalent to standard clipping to [-1, 1]
|
||||
s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0
|
||||
sample = (
|
||||
torch.clamp(sample, -s, s) / s
|
||||
) # "we threshold xt0 to the range [-s, s] and then divide by s"
|
||||
|
||||
sample = sample.reshape(batch_size, channels, *remaining_dims)
|
||||
sample = sample.to(dtype)
|
||||
|
||||
return sample
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t
|
||||
def _sigma_to_t(self, sigma, log_sigmas):
|
||||
# get log sigma
|
||||
@@ -768,27 +707,9 @@ class UniPCMultistepScheduler(
|
||||
rks.append(torch.ones((), dtype=h.dtype, device=h.device))
|
||||
rks = torch.stack(rks).to(device=device)
|
||||
|
||||
R = []
|
||||
b = []
|
||||
|
||||
hh = -h if self.predict_x0 else h
|
||||
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
|
||||
h_phi_k = h_phi_1 / hh - 1
|
||||
|
||||
factorial_i = 1
|
||||
|
||||
if self.config.solver_type == "bh1":
|
||||
B_h = hh
|
||||
elif self.config.solver_type == "bh2":
|
||||
B_h = torch.expm1(hh)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
for i in range(1, order + 1):
|
||||
R.append(torch.pow(rks, i - 1))
|
||||
b.append(h_phi_k * factorial_i / B_h)
|
||||
factorial_i *= i + 1
|
||||
h_phi_k = h_phi_k / hh - 1 / factorial_i
|
||||
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
|
||||
h, rks, order, self.predict_x0, self.config.solver_type
|
||||
)
|
||||
|
||||
R = torch.stack(R)
|
||||
b = torch.stack(b).to(device=device)
|
||||
@@ -906,27 +827,9 @@ class UniPCMultistepScheduler(
|
||||
rks.append(torch.ones((), dtype=h.dtype, device=h.device))
|
||||
rks = torch.stack(rks).to(device=device)
|
||||
|
||||
R = []
|
||||
b = []
|
||||
|
||||
hh = -h if self.predict_x0 else h
|
||||
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
|
||||
h_phi_k = h_phi_1 / hh - 1
|
||||
|
||||
factorial_i = 1
|
||||
|
||||
if self.config.solver_type == "bh1":
|
||||
B_h = hh
|
||||
elif self.config.solver_type == "bh2":
|
||||
B_h = torch.expm1(hh)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
for i in range(1, order + 1):
|
||||
R.append(torch.pow(rks, i - 1))
|
||||
b.append(h_phi_k * factorial_i / B_h)
|
||||
factorial_i *= i + 1
|
||||
h_phi_k = h_phi_k / hh - 1 / factorial_i
|
||||
R, b, h_phi_1, B_h = compute_unipc_bh_coefficients(
|
||||
h, rks, order, self.predict_x0, self.config.solver_type
|
||||
)
|
||||
|
||||
R = torch.stack(R)
|
||||
b = torch.stack(b).to(device=device)
|
||||
@@ -981,19 +884,6 @@ class UniPCMultistepScheduler(
|
||||
|
||||
return step_index
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index
|
||||
def _init_step_index(self, timestep):
|
||||
"""
|
||||
Initialize the step_index counter for the scheduler.
|
||||
"""
|
||||
|
||||
if self.begin_index is None:
|
||||
if isinstance(timestep, torch.Tensor):
|
||||
timestep = timestep.to(self.timesteps.device)
|
||||
self._step_index = self.index_for_timestep(timestep)
|
||||
else:
|
||||
self._step_index = self._begin_index
|
||||
|
||||
def step(
|
||||
self,
|
||||
model_output: torch.Tensor,
|
||||
@@ -1044,35 +934,10 @@ class UniPCMultistepScheduler(
|
||||
order=self.this_order,
|
||||
)
|
||||
|
||||
for i in range(self.config.solver_order - 1):
|
||||
self.model_outputs[i] = self.model_outputs[i + 1]
|
||||
self.timestep_list[i] = self.timestep_list[i + 1]
|
||||
|
||||
self.model_outputs[-1] = model_output_convert
|
||||
self.timestep_list[-1] = timestep
|
||||
|
||||
if self.config.lower_order_final:
|
||||
this_order = min(
|
||||
self.config.solver_order, len(self.timesteps) - self.step_index
|
||||
)
|
||||
else:
|
||||
this_order = self.config.solver_order
|
||||
|
||||
self.this_order = min(
|
||||
this_order, self.lower_order_nums + 1
|
||||
) # warmup for multistep
|
||||
assert self.this_order > 0
|
||||
|
||||
self.last_sample = sample
|
||||
prev_sample = self.multistep_uni_p_bh_update(
|
||||
model_output=model_output, # pass the original non-converted model output, in case solver-p is used
|
||||
sample=sample,
|
||||
order=self.this_order,
|
||||
prev_sample = unipc_predictor_step(
|
||||
self, model_output, model_output_convert, timestep, sample
|
||||
)
|
||||
|
||||
if self.lower_order_nums < self.config.solver_order:
|
||||
self.lower_order_nums += 1
|
||||
|
||||
# upon completion increase step index by one
|
||||
self._step_index += 1
|
||||
|
||||
@@ -1081,62 +946,6 @@ class UniPCMultistepScheduler(
|
||||
|
||||
return SchedulerOutput(prev_sample=prev_sample)
|
||||
|
||||
def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor:
|
||||
"""
|
||||
Ensures interchangeability with schedulers that need to scale the denoising model input depending on the
|
||||
current timestep.
|
||||
|
||||
Args:
|
||||
sample (`torch.Tensor`):
|
||||
The input sample.
|
||||
|
||||
Returns:
|
||||
`torch.Tensor`:
|
||||
A scaled input sample.
|
||||
"""
|
||||
return sample
|
||||
|
||||
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise
|
||||
def add_noise(
|
||||
self,
|
||||
original_samples: torch.Tensor,
|
||||
noise: torch.Tensor,
|
||||
timesteps: torch.IntTensor,
|
||||
) -> torch.Tensor:
|
||||
# Make sure sigmas and timesteps have the same device and dtype as original_samples
|
||||
sigmas = self.sigmas.to(
|
||||
device=original_samples.device, dtype=original_samples.dtype
|
||||
)
|
||||
if original_samples.device.type == "mps" and torch.is_floating_point(timesteps):
|
||||
# mps does not support float64
|
||||
schedule_timesteps = self.timesteps.to(
|
||||
original_samples.device, dtype=torch.float32
|
||||
)
|
||||
timesteps = timesteps.to(original_samples.device, dtype=torch.float32)
|
||||
else:
|
||||
schedule_timesteps = self.timesteps.to(original_samples.device)
|
||||
timesteps = timesteps.to(original_samples.device)
|
||||
|
||||
# begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index
|
||||
if self.begin_index is None:
|
||||
step_indices = [
|
||||
self.index_for_timestep(t, schedule_timesteps) for t in timesteps
|
||||
]
|
||||
elif self.step_index is not None:
|
||||
# add_noise is called after first denoising step (for inpainting)
|
||||
step_indices = [self.step_index] * timesteps.shape[0]
|
||||
else:
|
||||
# add noise is called before first denoising step to create initial latent(img2img)
|
||||
step_indices = [self.begin_index] * timesteps.shape[0]
|
||||
|
||||
sigma = sigmas[step_indices].flatten()
|
||||
while len(sigma.shape) < len(original_samples.shape):
|
||||
sigma = sigma.unsqueeze(-1)
|
||||
|
||||
alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)
|
||||
noisy_samples = alpha_t * original_samples + sigma_t * noise
|
||||
return noisy_samples
|
||||
|
||||
def __len__(self):
|
||||
return self.config.num_train_timesteps
|
||||
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 TSAIL Team and The HuggingFace Team. All rights reserved.
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
|
||||
def compute_unipc_bh_coefficients(
|
||||
h: torch.Tensor,
|
||||
rks: torch.Tensor,
|
||||
order: int,
|
||||
predict_x0: bool,
|
||||
solver_type: str,
|
||||
):
|
||||
"""Build UniPC coefficients, leaving tensor assembly to each scheduler."""
|
||||
R = []
|
||||
b = []
|
||||
|
||||
hh = -h if predict_x0 else h
|
||||
h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
|
||||
h_phi_k = h_phi_1 / hh - 1
|
||||
factorial_i = 1
|
||||
|
||||
if solver_type == "bh1":
|
||||
B_h = hh
|
||||
elif solver_type == "bh2":
|
||||
B_h = torch.expm1(hh)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
for i in range(1, order + 1):
|
||||
R.append(torch.pow(rks, i - 1))
|
||||
b.append(h_phi_k * factorial_i / B_h)
|
||||
factorial_i *= i + 1
|
||||
h_phi_k = h_phi_k / hh - 1 / factorial_i
|
||||
|
||||
return R, b, h_phi_1, B_h
|
||||
|
||||
|
||||
def unipc_predictor_step(
|
||||
scheduler, model_output, model_output_convert, timestep, sample
|
||||
):
|
||||
"""Update UniPC history and order, then predict; callers own correction and indexing."""
|
||||
for i in range(scheduler.config.solver_order - 1):
|
||||
scheduler.model_outputs[i] = scheduler.model_outputs[i + 1]
|
||||
scheduler.timestep_list[i] = scheduler.timestep_list[i + 1]
|
||||
scheduler.model_outputs[-1] = model_output_convert
|
||||
scheduler.timestep_list[-1] = timestep
|
||||
|
||||
if scheduler.config.lower_order_final:
|
||||
this_order = min(
|
||||
scheduler.config.solver_order,
|
||||
len(scheduler.timesteps) - scheduler.step_index,
|
||||
)
|
||||
else:
|
||||
this_order = scheduler.config.solver_order
|
||||
scheduler.this_order = min(this_order, scheduler.lower_order_nums + 1)
|
||||
assert scheduler.this_order > 0
|
||||
|
||||
scheduler.last_sample = sample
|
||||
# an external solver-p consumes the original, unconverted model output
|
||||
prev_sample = scheduler.multistep_uni_p_bh_update(
|
||||
model_output=model_output,
|
||||
sample=sample,
|
||||
order=scheduler.this_order,
|
||||
)
|
||||
if scheduler.lower_order_nums < scheduler.config.solver_order:
|
||||
scheduler.lower_order_nums += 1
|
||||
return prev_sample
|
||||
|
||||
|
||||
class UniPCSchedulerMixin:
|
||||
"""Shared sample transforms and step state; subclasses own schedules and solvers."""
|
||||
|
||||
@property
|
||||
def step_index(self):
|
||||
return self._step_index
|
||||
|
||||
@property
|
||||
def begin_index(self):
|
||||
return self._begin_index
|
||||
|
||||
def set_begin_index(self, begin_index: int = 0):
|
||||
self._begin_index = begin_index
|
||||
|
||||
def _init_step_index(self, timestep):
|
||||
if self.begin_index is None:
|
||||
if isinstance(timestep, torch.Tensor):
|
||||
timestep = timestep.to(self.timesteps.device)
|
||||
self._step_index = self.index_for_timestep(timestep)
|
||||
else:
|
||||
self._step_index = self._begin_index
|
||||
|
||||
def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor:
|
||||
return sample
|
||||
|
||||
def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply per-sample dynamic thresholding before restoring the input dtype."""
|
||||
dtype = sample.dtype
|
||||
batch_size, channels, *remaining_dims = sample.shape
|
||||
if dtype not in (torch.float32, torch.float64):
|
||||
sample = sample.float()
|
||||
|
||||
sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
|
||||
abs_sample = sample.abs()
|
||||
s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
|
||||
s = torch.clamp(s, min=1, max=self.config.sample_max_value)
|
||||
s = s.unsqueeze(1)
|
||||
sample = torch.clamp(sample, -s, s) / s
|
||||
sample = sample.reshape(batch_size, channels, *remaining_dims)
|
||||
return sample.to(dtype)
|
||||
|
||||
def add_noise(
|
||||
self,
|
||||
original_samples: torch.Tensor,
|
||||
noise: torch.Tensor,
|
||||
timesteps: torch.IntTensor,
|
||||
) -> torch.Tensor:
|
||||
sigmas = self.sigmas.to(
|
||||
device=original_samples.device, dtype=original_samples.dtype
|
||||
)
|
||||
if original_samples.device.type == "mps" and torch.is_floating_point(timesteps):
|
||||
# mps does not support float64 timesteps
|
||||
schedule_timesteps = self.timesteps.to(
|
||||
original_samples.device, dtype=torch.float32
|
||||
)
|
||||
timesteps = timesteps.to(original_samples.device, dtype=torch.float32)
|
||||
else:
|
||||
schedule_timesteps = self.timesteps.to(original_samples.device)
|
||||
timesteps = timesteps.to(original_samples.device)
|
||||
|
||||
# training looks up each timestep; img2img and inpainting use the step state
|
||||
if self.begin_index is None:
|
||||
step_indices = [
|
||||
self.index_for_timestep(t, schedule_timesteps) for t in timesteps
|
||||
]
|
||||
elif self.step_index is not None:
|
||||
step_indices = [self.step_index] * timesteps.shape[0]
|
||||
else:
|
||||
step_indices = [self.begin_index] * timesteps.shape[0]
|
||||
|
||||
sigma = sigmas[step_indices].flatten()
|
||||
while len(sigma.shape) < len(original_samples.shape):
|
||||
sigma = sigma.unsqueeze(-1)
|
||||
|
||||
alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)
|
||||
return alpha_t * original_samples + sigma_t * noise
|
||||
@@ -1,6 +1,8 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
@@ -75,3 +77,52 @@ def test_gemma3_maps_hf_siglip_projection_name():
|
||||
mapped, _, _ = map_name("vision_tower.encoder.layers.0.self_attn.out_proj.weight")
|
||||
|
||||
assert mapped == "vision_tower.vision_model.encoder.layers.0.self_attn.proj.weight"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shard,normalized",
|
||||
[("q", 0), ("k", 1), ("v", 2), (0, "q"), (1, "k"), (2, "v"), ("0", 0), ("01", 1)],
|
||||
)
|
||||
@pytest.mark.parametrize("error", [AssertionError, TypeError])
|
||||
def test_gemma3_shard_loader_preserves_first_attempt_and_fallback(
|
||||
shard, normalized, error
|
||||
):
|
||||
parameter = nn.Parameter(torch.zeros(2))
|
||||
weight = torch.ones(2)
|
||||
for accepted in (shard, normalized):
|
||||
calls = []
|
||||
|
||||
def loader(param, loaded, shard_id):
|
||||
calls.append(shard_id)
|
||||
if shard_id != accepted:
|
||||
raise error("unsupported shard representation")
|
||||
param.data.copy_(loaded)
|
||||
|
||||
gemma_3._load_with_shard_id(loader, parameter, weight, shard)
|
||||
assert calls == ([shard] if accepted == shard else [shard, normalized])
|
||||
torch.testing.assert_close(parameter, weight, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shard", ["unsupported", 3, None])
|
||||
def test_gemma3_shard_loader_does_not_retry_unknown_ids(shard):
|
||||
calls = []
|
||||
|
||||
def loader(param, loaded, shard_id):
|
||||
calls.append(shard_id)
|
||||
raise TypeError("unsupported")
|
||||
|
||||
with pytest.raises(TypeError, match="Unsupported shard_id=.*param=None"):
|
||||
gemma_3._load_with_shard_id(
|
||||
loader, nn.Parameter(torch.zeros(1)), torch.ones(1), shard
|
||||
)
|
||||
assert calls == [shard]
|
||||
|
||||
|
||||
def test_gemma3_shard_loader_propagates_other_errors():
|
||||
def loader(*args):
|
||||
raise RuntimeError("load failure")
|
||||
|
||||
with pytest.raises(RuntimeError, match="load failure"):
|
||||
gemma_3._load_with_shard_id(
|
||||
loader, nn.Parameter(torch.zeros(1)), torch.ones(1), "q"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_unipc_multistep import (
|
||||
FlowUniPCMultistepScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_unipc_multistep import (
|
||||
UniPCMultistepScheduler,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [FlowUniPCMultistepScheduler, UniPCMultistepScheduler])
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_unipc_sample_transforms_and_state(cls, device, dtype):
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required")
|
||||
scheduler = cls(dynamic_thresholding_ratio=0.5, sample_max_value=5.0)
|
||||
restored = cls.from_config(scheduler.config)
|
||||
assert dict(restored.config) == dict(scheduler.config)
|
||||
x = torch.tensor([-8, -2, 0, 2, 4, 20], device=device, dtype=dtype)
|
||||
x = x.repeat(2).reshape(2, 1, 2, 3).transpose(-1, -2)
|
||||
original = x.clone()
|
||||
expected = (x.float().clamp(-3, 3) / 3).to(dtype)
|
||||
torch.testing.assert_close(scheduler._threshold_sample(x), expected, rtol=0, atol=0)
|
||||
assert scheduler.scale_model_input(x, 0) is x
|
||||
|
||||
scheduler.timesteps = torch.tensor([90, 60, 60, 10], device=device)
|
||||
scheduler.sigmas = torch.tensor([0.9, 0.6, 0.4, 0.1, 0], device=device)
|
||||
timesteps = torch.tensor([60, 10])
|
||||
noise = torch.ones_like(x)
|
||||
for mode, indices in (
|
||||
("training", [2, 3]),
|
||||
("img2img", [1, 1]),
|
||||
("inpaint", [2, 2]),
|
||||
):
|
||||
if mode != "training":
|
||||
scheduler.set_begin_index(1)
|
||||
if mode == "inpaint":
|
||||
scheduler._step_index = 2
|
||||
state = scheduler.begin_index, scheduler.step_index
|
||||
sigma = scheduler.sigmas.to(dtype)[indices].reshape(2, 1, 1, 1)
|
||||
if cls is FlowUniPCMultistepScheduler:
|
||||
alpha = 1 - sigma
|
||||
else:
|
||||
alpha = 1 / ((sigma**2 + 1) ** 0.5)
|
||||
sigma = sigma * alpha
|
||||
expected = alpha * x + sigma * noise
|
||||
torch.testing.assert_close(
|
||||
scheduler.add_noise(x, noise, timesteps), expected, rtol=0, atol=0
|
||||
)
|
||||
assert (scheduler.begin_index, scheduler.step_index) == state
|
||||
torch.testing.assert_close(x, original, rtol=0, atol=0)
|
||||
scheduler.set_timesteps(5, device=device)
|
||||
assert scheduler.begin_index is None and scheduler.step_index is None
|
||||
scheduler._init_step_index(scheduler.timesteps[0])
|
||||
assert scheduler.step_index == 0
|
||||
scheduler.set_begin_index(2)
|
||||
scheduler._init_step_index(scheduler.timesteps[0])
|
||||
assert scheduler.step_index == 2
|
||||
|
||||
|
||||
def test_unipc_missing_timestep_behavior_is_preserved():
|
||||
regular, flow = UniPCMultistepScheduler(), FlowUniPCMultistepScheduler()
|
||||
regular.set_timesteps(5)
|
||||
flow.set_timesteps(5)
|
||||
assert regular.index_for_timestep(-1) == 4
|
||||
with pytest.raises(IndexError):
|
||||
flow.index_for_timestep(-1)
|
||||
Reference in New Issue
Block a user