[Perf] diffusion/cosmos3: optimize v2v preprocess performance (#45398)

Signed-off-by: pinjie <pinjiex@nvidia.com>
This commit is contained in:
Pinjie Xu
2026-10-23 13:57:38 +08:00
committed by GitHub
parent 7baacd1d1b
commit 7c680a0d45
7 changed files with 605 additions and 23 deletions
+9
View File
@@ -16,6 +16,15 @@ register_kernel(
)
)
register_kernel(
KernelSpec(
op="mm.lanczos_resize",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.mm.process.lanczos:lanczos_resize",
capabilities=CUDA_ONLY,
)
)
register_kernel(
KernelSpec(
op="mm.normalize_and_patchify",
@@ -1,5 +1,6 @@
"""Multimodal input-processing kernels."""
from sglang.kernels.ops.mm.process.image import normalize_and_patchify
from sglang.kernels.ops.mm.process.lanczos import lanczos_resize
__all__ = ["normalize_and_patchify"]
__all__ = ["normalize_and_patchify", "lanczos_resize"]
@@ -0,0 +1,260 @@
# SPDX-License-Identifier: Apache-2.0
"""GPU RGB Lanczos resize with optional crop and normalization."""
import math
from functools import lru_cache
import numpy as np
import torch
import triton
import triton.language as tl
def _sinc(value: float) -> float:
if value == 0.0:
return 1.0
angle = value * math.pi
return math.sin(angle) / angle
@lru_cache(maxsize=128)
def _coefficients(source: int, target: int, device: torch.device):
"""Pillow's RGB resampling uses normalized signed 22-bit coefficients."""
scale = source / target
filter_scale = max(scale, 1.0)
support = 3.0 * filter_scale
inv_filter_scale = 1.0 / filter_scale
taps = math.ceil(support) * 2 + 1
coefficients = np.zeros((target, taps), dtype=np.int32)
bounds = np.zeros((target, 2), dtype=np.int32)
for out in range(target):
center = (out + 0.5) * scale
begin = max(int(center - support + 0.5), 0)
end = min(int(center + support + 0.5), source)
weights = []
for pos in range(begin, end):
distance = (pos - center + 0.5) * inv_filter_scale
weights.append(
_sinc(distance) * _sinc(distance / 3.0)
if -3.0 <= distance < 3.0
else 0.0
)
# Keep sequential double accumulation, including on Python versions
# whose sum() uses compensated summation.
total = 0.0
for weight in weights:
total += weight
coefficients[out, : len(weights)] = [
int(w / total * (1 << 22) + (0.5 if w >= 0.0 else -0.5)) for w in weights
]
bounds[out] = (begin, len(weights))
return (
torch.from_numpy(coefficients).to(device),
torch.from_numpy(bounds).to(device),
taps,
)
@lru_cache(maxsize=8)
def _normalization(
mean: tuple[float, float, float],
std: tuple[float, float, float],
device: torch.device,
):
# Precompute float32 values to preserve rounding without fused multiply-add
# or approximate division in the kernel. Each RGB channel has its own table.
values = np.arange(256, dtype=np.float32)[None, :] / 255.0
values = (values - np.asarray(mean, dtype=np.float32)[:, None]) / np.asarray(
std, dtype=np.float32
)[:, None]
return torch.from_numpy(values).to(device)
@triton.jit(
do_not_specialize=[
"source_height",
"crop_left",
"stride_batch",
"stride_y",
"stride_x",
"stride_channel",
"taps",
]
)
def _horizontal_lanczos(
src,
dst,
coefficients,
bounds,
source_height,
crop_width: tl.constexpr,
crop_left,
stride_batch,
stride_y,
stride_x,
stride_channel,
taps,
block: tl.constexpr,
):
batch = tl.program_id(1).to(tl.int64)
index = tl.program_id(0) * block + tl.arange(0, block)
valid = index < source_height * crop_width * 3
channel = index % 3
x = index // 3 % crop_width + crop_left
y = index // (3 * crop_width)
first = tl.load(bounds + x * 2, valid, 0)
count = tl.load(bounds + x * 2 + 1, valid, 0)
total = tl.full((block,), 1 << 21, tl.int32)
for tap in range(taps):
weight = tl.load(coefficients + x * taps + tap, valid, 0)
value = tl.load(
src
+ batch * stride_batch
+ y * stride_y
+ (first + tap) * stride_x
+ channel * stride_channel,
valid & (tap < count),
0,
).to(tl.int32)
total += value * weight
pixel = tl.minimum(tl.maximum(total >> 22, 0), 255).to(tl.uint8)
tl.store(dst + batch * source_height * crop_width * 3 + index, pixel, valid)
@triton.jit(do_not_specialize=["crop_top", "stride_batch", "taps"])
def _vertical_lanczos(
src,
dst,
coefficients,
bounds,
normalization,
crop_height: tl.constexpr,
crop_width: tl.constexpr,
crop_top,
stride_batch,
taps,
normalize: tl.constexpr,
block: tl.constexpr,
):
batch = tl.program_id(1).to(tl.int64)
index = tl.program_id(0) * block + tl.arange(0, block)
valid = index < crop_height * crop_width * 3
channel = index % 3
y = index // (3 * crop_width) + crop_top
x = index // 3 % crop_width
first = tl.load(bounds + y * 2, valid, 0)
count = tl.load(bounds + y * 2 + 1, valid, 0)
total = tl.full((block,), 1 << 21, tl.int32)
for tap in range(taps):
weight = tl.load(coefficients + y * taps + tap, valid, 0)
value = tl.load(
src + batch * stride_batch + ((first + tap) * crop_width + x) * 3 + channel,
valid & (tap < count),
0,
).to(tl.int32)
total += value * weight
pixel = tl.minimum(tl.maximum(total >> 22, 0), 255)
if normalize:
value = tl.load(normalization + channel * 256 + pixel)
else:
value = pixel.to(tl.uint8)
tl.store(dst + batch * crop_height * crop_width * 3 + index, value, valid)
def lanczos_resize(
image: torch.Tensor,
height: int,
width: int,
*,
crop: tuple[int, int, int, int] | None = None,
mean: tuple[float, float, float] | None = None,
std: tuple[float, float, float] | None = None,
) -> torch.Tensor:
"""Resize CUDA uint8 HWC/NHWC RGB to ``height x width``, preserving layout.
By default, return uint8 pixels matching Pillow's two-pass Lanczos rounding.
Optional ``crop=(top, left, height, width)`` selects an in-bounds rectangle
in the resized image. Providing both RGB ``mean`` and ``std`` fuses
``(pixel / 255 - mean) / std`` and returns float32 instead.
Batched inputs share resize, crop, and normalization parameters. The input
may have padded batch/row strides. The contiguous output owns its
storage; the input is borrowed only by the horizontal pass. Operations run
on the current PyTorch CUDA stream.
"""
if not image.is_cuda or torch.version.hip is not None:
raise ValueError("Lanczos RGB preprocessing requires a CUDA tensor")
if image.dtype != torch.uint8 or image.ndim not in (3, 4) or image.shape[-1] != 3:
raise ValueError(
"Lanczos RGB input must be uint8 with shape [H, W, 3] or [N, H, W, 3]"
)
if height <= 0 or width <= 0 or any(size <= 0 for size in image.shape):
raise ValueError("Lanczos RGB input and output dimensions must be positive")
if crop is None:
top, left, output_h, output_w = 0, 0, height, width
else:
if len(crop) != 4:
raise ValueError("crop must be (top, left, height, width)")
top, left, output_h, output_w = crop
if (
top < 0
or left < 0
or output_h <= 0
or output_w <= 0
or top + output_h > height
or left + output_w > width
):
raise ValueError(
"crop must be a positive rectangle within the resized image"
)
if (mean is None) != (std is None):
raise ValueError("mean and std must be provided together")
normalize = mean is not None
if normalize:
mean, std = tuple(mean), tuple(std)
if len(mean) != 3 or len(std) != 3:
raise ValueError("mean and std must each contain three RGB values")
if any(not math.isfinite(v) for v in mean + std) or any(v <= 0 for v in std):
raise ValueError("mean/std must be finite and std must be positive")
source_h, source_w = image.shape[-3:-1]
batch_size = image.shape[0] if image.ndim == 4 else 1
stride_batch = image.stride(0) if image.ndim == 4 else 0
coeff_x, bounds_x, taps_x = _coefficients(source_w, width, image.device)
coeff_y, bounds_y, taps_y = _coefficients(source_h, height, image.device)
intermediate = torch.empty(
(batch_size, source_h, output_w, 3), dtype=torch.uint8, device=image.device
)
output = torch.empty(
(*image.shape[:-3], output_h, output_w, 3),
dtype=torch.float32 if normalize else torch.uint8,
device=image.device,
)
_horizontal_lanczos[(triton.cdiv(source_h * output_w * 3, 256), batch_size)](
image,
intermediate,
coeff_x,
bounds_x,
source_h,
output_w,
left,
stride_batch,
*image.stride()[-3:],
taps_x,
256,
)
_vertical_lanczos[(triton.cdiv(output_h * output_w * 3, 256), batch_size)](
intermediate,
output,
coeff_y,
bounds_y,
_normalization(mean, std, image.device) if normalize else None,
output_h,
output_w,
top,
intermediate.stride(0),
taps_y,
normalize,
256,
)
return output
@@ -12,6 +12,11 @@ per-request from ``batch.data_type`` and the presence of
import copy
import json
import math
from collections import deque
from collections.abc import Iterator
from contextlib import closing
from fractions import Fraction
from itertools import islice
from typing import Any
import numpy as np
@@ -72,6 +77,7 @@ from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.utils.profiler import SGLDiffusionProfiler
from sglang.multimodal_gen.runtime.utils.vision import load_image, load_video
from sglang.srt.environ import envs
from sglang.srt.utils.common import get_compiler_backend
logger = init_logger(__name__)
@@ -195,6 +201,84 @@ def _pad_transfer_frames(video: torch.Tensor, target_frames: int) -> torch.Tenso
return video.contiguous()
def _iter_video_frames_pyav(
video_path: str, num_frames: int, keep: str = "first"
) -> Iterator[np.ndarray]:
"""Yield the first/last RGB frames without materializing the whole video.
Tail access seeks backwards to a keyframe and decodes through EOF. If the
estimate leaves fewer than the requested frames, back off further. Unknown
duration/rate falls back to a bounded tail buffer, preserving VFR semantics.
"""
import av
if num_frames <= 0:
raise ValueError("num_frames must be positive")
if keep not in ("first", "last"):
raise ValueError("keep must be 'first' or 'last'")
with av.open(video_path) as container:
if not container.streams.video:
raise ValueError(f"No video stream in {video_path!r}")
stream = container.streams.video[0]
stream.codec_context.thread_type = "SLICE"
stream.codec_context.thread_count = 1
if keep == "first":
frames = islice(container.decode(stream), num_frames)
else:
start = stream.start_time or 0
duration, rate = stream.duration, stream.average_rate
if duration is None and container.duration is not None:
duration = int(
Fraction(container.duration, av.time_base) / stream.time_base
)
if duration is not None and duration > 0 and rate:
span = max(1, int(num_frames / rate / stream.time_base))
while True:
offset = max(start, start + duration - span)
container.seek(offset, stream=stream, backward=True)
frames = deque(container.decode(stream), maxlen=num_frames)
if len(frames) == num_frames or offset == start:
break
span *= 2
else:
frames = deque(container.decode(stream), maxlen=num_frames)
for frame in frames:
yield frame.to_ndarray(format="rgb24")
def _preprocess_video_frames_gpu(
frames: Iterator[np.ndarray], height: int, width: int
) -> list[torch.Tensor]:
"""Resize/crop/normalize batches of up to eight RGB frames on CUDA."""
from sglang.kernels.ops.mm.process.lanczos import lanczos_resize
processed = []
while batch := list(islice(frames, 8)):
pixels = torch.from_numpy(np.stack(batch)).pin_memory().cuda(non_blocking=True)
source_h, source_w = pixels.shape[1:3]
scale = max(width / source_w, height / source_h)
resized_h = math.ceil(scale * source_h)
resized_w = math.ceil(scale * source_w)
output = lanczos_resize(
pixels,
resized_h,
resized_w,
crop=(
(resized_h - height) // 2,
(resized_w - width) // 2,
height,
width,
),
mean=(0.5, 0.5, 0.5),
std=(0.5, 0.5, 0.5),
)
processed.extend(output.permute(0, 3, 1, 2).unbind(0))
return processed
class Cosmos3ImagePreprocessStage(PipelineStage):
"""Load, aspect-resize, and center-crop the conditioning input.
@@ -391,10 +475,6 @@ class Cosmos3ImagePreprocessStage(PipelineStage):
return batch
if isinstance(video_path, str) and video_path:
frames = load_video(video_path)
if not frames:
raise ValueError(f"No frames decoded from video: {video_path!r}")
keep = (
getattr(batch.sampling_params, "condition_video_keep", "first")
or "first"
@@ -411,27 +491,34 @@ class Cosmos3ImagePreprocessStage(PipelineStage):
# to noise.
num_source_frames = max(cond_indexes) * 4 + 1
num_target_frames = batch.num_frames
if keep == "last":
frames = frames[-num_source_frames:]
else:
frames = frames[:num_source_frames]
if len(frames) < num_source_frames:
frames = frames + [frames[-1]] * (num_source_frames - len(frames))
if len(frames) < num_target_frames:
frames = frames + [frames[-1]] * (num_target_frames - len(frames))
force_cpu = envs.SGLANG_FORCE_CPU_IMAGE_PREPROCESSING.get()
with closing(
_iter_video_frames_pyav(video_path, num_source_frames, keep)
) as frames:
if force_cpu:
processed = [
_pil_to_normalized_tensor(
_resize_crop_pil(PIL.Image.fromarray(f), target_w, target_h)
)
for f in frames
]
else:
processed = _preprocess_video_frames_gpu(frames, target_h, target_w)
if not processed:
raise ValueError(f"No frames decoded from video: {video_path!r}")
# Pad CUDA frames after conversion to the VAE dtype instead.
if force_cpu:
if len(processed) < num_source_frames:
processed += [processed[-1]] * (num_source_frames - len(processed))
if len(processed) < num_target_frames:
processed += [processed[-1]] * (num_target_frames - len(processed))
processed = [
_pil_to_normalized_tensor(
_resize_crop_pil(f.convert("RGB"), target_w, target_h)
)
for f in frames
]
video_tensor = torch.stack(processed, dim=1).unsqueeze(0).contiguous()
batch.preprocessed_video = video_tensor
self.log_info(
f"Preprocessed conditioning video to "
f"{video_tensor.shape[2]}x{target_h}x{target_w} "
f"(keep={keep}, source frames={num_source_frames}, padded to {num_target_frames})"
f"(keep={keep}, source frames={num_source_frames}, target frames={num_target_frames})"
)
return batch
@@ -757,6 +844,13 @@ class Cosmos3LatentPreparationStage(PipelineStage):
pixel_input = batch.preprocessed_video.to(
device=device, dtype=vae_dtype
)
batch.preprocessed_video = None
if pixel_input.shape[2] < batch.num_frames:
padding = pixel_input[:, :, -1:].expand(
-1, -1, batch.num_frames - pixel_input.shape[2], -1, -1
)
pixel_input = torch.cat([pixel_input, padding], dim=2)
del padding
cond_indexes = Cosmos3ImagePreprocessStage._resolve_condition_indexes(
batch
)
@@ -770,6 +864,11 @@ class Cosmos3LatentPreparationStage(PipelineStage):
with torch.no_grad():
cond_latent = self._vae_encode(pixel_input).to(dtype)
# Only the encoded latents are needed by the remaining stages.
batch.preprocessed_video = None
batch.preprocessed_image = None
del pixel_input
max_idx = max(cond_indexes)
if max_idx >= num_latent_frames:
raise ValueError(
@@ -1005,9 +1104,13 @@ class Cosmos3TimestepPreparationStage(PipelineStage):
return COSMOS3_T2I_FLOW_SHIFT
if is_edge:
return COSMOS3_EDGE_VIDEO_FLOW_SHIFT
if batch.preprocessed_image is not None:
if batch.preprocessed_image is not None or getattr(
batch.sampling_params, "image_path", None
):
return COSMOS3_I2V_FLOW_SHIFT
if batch.preprocessed_video is not None:
if batch.preprocessed_video is not None or getattr(
batch.sampling_params, "video_path", None
):
return COSMOS3_V2V_FLOW_SHIFT
return COSMOS3_T2V_FLOW_SHIFT
@@ -8,6 +8,7 @@ import types
import unittest
from unittest import mock
import numpy as np
import torch
from PIL import Image
@@ -76,6 +77,8 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.c
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.cosmos3_guardrails import (
is_cosmos_guardrail_available,
)
from sglang.srt.environ import envs
from sglang.test.test_utils import CustomTestCase
def _apply(mapping_fn, key):
@@ -2057,5 +2060,48 @@ class TestCosmos3DurationTemplateSuppression(unittest.TestCase):
self.assertIn("seconds long", final)
@unittest.skipUnless(
torch.cuda.is_available() and torch.version.hip is None, "requires CUDA"
)
class TestCosmos3VideoPreprocess(CustomTestCase):
def test_gpu_preprocess_switch(self):
"""CPU padding and deferred GPU padding preserve the same source pixels."""
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages import (
cosmos3,
)
stage = Cosmos3ImagePreprocessStage.__new__(Cosmos3ImagePreprocessStage)
stage.log_info = lambda *_args, **_kwargs: None
frames = np.random.default_rng(816).integers(
0, 256, (5, 32, 48, 3), dtype=np.uint8
)
outputs = []
for force_cpu in (True, False):
with (
self.subTest(force_cpu=force_cpu),
envs.SGLANG_FORCE_CPU_IMAGE_PREPROCESSING.override(force_cpu),
mock.patch.object(
cosmos3,
"_iter_video_frames_pyav",
side_effect=lambda *_: (frame for frame in frames),
),
):
batch = types.SimpleNamespace(
image_path=None,
video_path="input.mp4",
data_type=DataType.VIDEO,
height=24,
width=40,
num_frames=45,
sampling_params=types.SimpleNamespace(),
)
stage.forward(batch, types.SimpleNamespace())
actual = batch.preprocessed_video
self.assertEqual(actual.device.type, "cpu" if force_cpu else "cuda")
self.assertEqual(actual.shape, (1, 3, 45 if force_cpu else 5, 24, 40))
outputs.append(actual[:, :, :5].cpu())
self.assertTrue(torch.equal(outputs[0], outputs[1]))
if __name__ == "__main__":
unittest.main()
+1 -1
View File
@@ -1566,7 +1566,7 @@ class Envs:
# set False to fall back to the per-image loop.
SGLANG_VIT_ENABLE_VECTORIZED_POS_EMBED = EnvBool(True)
SGLANG_MM_SKIP_COMPUTE_HASH = EnvBool(False)
# Currently supported by the Kimi-K2.5 image processor only.
# Supported by Kimi-K2.5 image and Cosmos3 V2V frame preprocessing.
SGLANG_FORCE_CPU_IMAGE_PREPROCESSING = EnvBool(False)
# For pre-tokenized (list[int]) multimodal prompts,
# preserve the user's original tokens to avoid retokenization drift.