[Diffusion] Fuse LongCat GELU+cat and support Edit-Turbo BCG (#40384)
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
#include <sgl_kernel/vec.cuh>
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
|
||||
namespace sglang {
|
||||
|
||||
template <typename T, int kVecN>
|
||||
__global__ void gelu_tanh_cat_kernel(
|
||||
const T* __restrict__ attn,
|
||||
const T* __restrict__ mlp,
|
||||
T* __restrict__ output,
|
||||
uint32_t num_vecs,
|
||||
uint32_t attn_vecs,
|
||||
uint32_t mlp_vecs) {
|
||||
using vec_t = device::AlignedVector<T, kVecN>;
|
||||
const uint32_t row_vecs = attn_vecs + mlp_vecs;
|
||||
const uint32_t stride = blockDim.x * gridDim.x;
|
||||
for (uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_vecs; i += stride) {
|
||||
const uint32_t row = i / row_vecs;
|
||||
const uint32_t col = i % row_vecs;
|
||||
vec_t value;
|
||||
if (col < attn_vecs) {
|
||||
value.load(attn, row * attn_vecs + col);
|
||||
} else {
|
||||
value.load(mlp, row * mlp_vecs + col - attn_vecs);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kVecN; ++j) {
|
||||
const float x = device::cast<fp32_t>(value[j]);
|
||||
// Match aten's tanh GELU operation order, including the BF16 store
|
||||
// that precedes the eager concatenation.
|
||||
constexpr float kBeta = 0.7978845608028654f;
|
||||
constexpr float kKappa = 0.044715f;
|
||||
const float cube = x * x * x;
|
||||
const float inner = kBeta * (x + kKappa * cube);
|
||||
value[j] = device::cast<T>(0.5f * x * (1.0f + tanhf(inner)));
|
||||
}
|
||||
}
|
||||
value.store(output, i);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void gelu_tanh_cat(tvm::ffi::TensorView attn, tvm::ffi::TensorView mlp, tvm::ffi::TensorView output) {
|
||||
using namespace host;
|
||||
auto rows = SymbolicSize{"rows"};
|
||||
auto attn_width = SymbolicSize{"attn_width"};
|
||||
auto mlp_width = SymbolicSize{"mlp_width"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLCUDA>();
|
||||
TensorMatcher({rows, attn_width}).with_dtype<T>().with_device(device_).verify(attn);
|
||||
TensorMatcher({rows, mlp_width}).with_dtype<T>().with_device(device_).verify(mlp);
|
||||
const int64_t a = attn_width.unwrap();
|
||||
const int64_t m = mlp_width.unwrap();
|
||||
TensorMatcher({rows, a + m}).with_dtype<T>().with_device(device_).verify(output);
|
||||
constexpr int kVecN = 16 / sizeof(T);
|
||||
CHECK_HOST(a > 0 && m > 0 && a % kVecN == 0 && m % kVecN == 0)
|
||||
<< "gelu_tanh_cat: both widths must be positive multiples of " << kVecN;
|
||||
CHECK_HOST(rows.unwrap() > 0) << "gelu_tanh_cat: rows must be positive";
|
||||
CHECK_HOST(
|
||||
reinterpret_cast<uintptr_t>(attn.data_ptr()) % 16 == 0 && reinterpret_cast<uintptr_t>(mlp.data_ptr()) % 16 == 0 &&
|
||||
reinterpret_cast<uintptr_t>(output.data_ptr()) % 16 == 0)
|
||||
<< "gelu_tanh_cat: tensors must be 16-byte aligned";
|
||||
const int64_t n = rows.unwrap() * ((a + m) / kVecN);
|
||||
CHECK_HOST(n <= std::numeric_limits<int32_t>::max()) << "gelu_tanh_cat: tensor is too large";
|
||||
constexpr int kBlockSize = 256;
|
||||
const auto kernel = gelu_tanh_cat_kernel<T, kVecN>;
|
||||
const int64_t occupancy = runtime::get_blocks_per_sm(kernel, kBlockSize);
|
||||
const int64_t sms = runtime::get_sm_count(device_.unwrap().device_id);
|
||||
const int64_t grid = std::min(sms * occupancy, div_ceil(n, int64_t{kBlockSize}));
|
||||
LaunchKernel(grid, kBlockSize, device_.unwrap())(
|
||||
kernel,
|
||||
static_cast<const T*>(attn.data_ptr()),
|
||||
static_cast<const T*>(mlp.data_ptr()),
|
||||
static_cast<T*>(output.data_ptr()),
|
||||
static_cast<uint32_t>(n),
|
||||
static_cast<uint32_t>(a / kVecN),
|
||||
static_cast<uint32_t>(m / kVecN));
|
||||
}
|
||||
|
||||
} // namespace sglang
|
||||
@@ -49,6 +49,13 @@ _HIP = frozenset({CapabilityRequirement.HIP})
|
||||
# inventory: callers name the one they want via ``select_kernel``.
|
||||
# ---------------------------------------------------------------------------
|
||||
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
(
|
||||
"diffusion.gelu_tanh_cat",
|
||||
KernelBackend.JIT,
|
||||
"activation.gelu_tanh_cat_jit:fused_gelu_tanh_cat",
|
||||
_CUDA,
|
||||
"BF16 tanh GELU plus channel concatenation.",
|
||||
),
|
||||
(
|
||||
"diffusion.apply_group_norm_silu",
|
||||
KernelBackend.TRITON,
|
||||
@@ -527,6 +534,8 @@ for _op, _backend, _target, _caps, _description in _SPECS:
|
||||
# then symbol; a new public kernel belongs here and nowhere else.
|
||||
# ---------------------------------------------------------------------------
|
||||
_EXPORTS: dict[str, str] = {
|
||||
"can_use_fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
|
||||
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
|
||||
"load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader",
|
||||
# Normalization: RMSNorm / LayerNorm / GroupNorm and their fused epilogues
|
||||
"can_defer_flux2_gated_residual": "norm.flux2_gated_resnorm_jit",
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Concatenate attention with tanh GELU while retaining the BF16 rounding boundary.
|
||||
|
||||
The inputs are contiguous BF16 tensors with equal leading dimensions and
|
||||
16-byte aligned rows. The result matches ``cat((attn, gelu(mlp)), dim=-1)``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
@cache_once
|
||||
def _module():
|
||||
args = make_cpp_args(torch.bfloat16)
|
||||
return load_jit(
|
||||
"gelu_tanh_cat",
|
||||
*args,
|
||||
cuda_files=["diffusion/gelu_tanh_cat.cuh"],
|
||||
cuda_wrappers=[("gelu_tanh_cat", f"gelu_tanh_cat<{args}>")],
|
||||
extra_cuda_cflags=["-lineinfo"],
|
||||
)
|
||||
|
||||
|
||||
def can_use_fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> bool:
|
||||
return (
|
||||
not torch.is_grad_enabled()
|
||||
and not torch.compiler.is_compiling()
|
||||
and torch.version.hip is None
|
||||
and attn.is_cuda
|
||||
and attn.dtype == mlp.dtype == torch.bfloat16
|
||||
and attn.device == mlp.device
|
||||
and attn.ndim >= 2
|
||||
and attn.shape[:-1] == mlp.shape[:-1]
|
||||
and attn.shape[-1] > 0
|
||||
and mlp.shape[-1] > 0
|
||||
and attn.shape[-1] % 8 == mlp.shape[-1] % 8 == 0
|
||||
and attn.is_contiguous()
|
||||
and mlp.is_contiguous()
|
||||
and attn.numel() > 0
|
||||
and (attn.numel() + mlp.numel()) // 8 <= 2**31 - 1
|
||||
and attn.data_ptr() % 16 == mlp.data_ptr() % 16 == 0
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["output"])
|
||||
def _gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor, output: torch.Tensor) -> None:
|
||||
_module().gelu_tanh_cat(
|
||||
attn.view(-1, attn.shape[-1]),
|
||||
mlp.view(-1, mlp.shape[-1]),
|
||||
output.view(-1, output.shape[-1]),
|
||||
)
|
||||
|
||||
|
||||
def fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> torch.Tensor:
|
||||
output = torch.empty(
|
||||
(*attn.shape[:-1], attn.shape[-1] + mlp.shape[-1]),
|
||||
device=attn.device,
|
||||
dtype=attn.dtype,
|
||||
)
|
||||
_gelu_tanh_cat(attn, mlp, output)
|
||||
return output
|
||||
@@ -40,6 +40,23 @@ class LongCatImageEditSamplingParams(SamplingParams):
|
||||
enable_cfg_renorm: bool = False
|
||||
enable_prompt_rewrite: bool = False
|
||||
|
||||
def prepare_synthetic_warmup_request_for_queue(self, req, server_args) -> None:
|
||||
if server_args.warmup_resolutions is None:
|
||||
return
|
||||
# Editing derives both the output grid and VL prefix from the input
|
||||
# image. A square placeholder discards an explicit non-square warmup
|
||||
# resolution and captures a graph that the real request cannot replay.
|
||||
import base64
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (req.width, req.height)).save(buffer, format="PNG")
|
||||
req.image_path = [
|
||||
"data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class LongCatImageEditTurboSamplingParams(LongCatImageEditSamplingParams):
|
||||
|
||||
+4
-2
@@ -23,8 +23,10 @@ def is_longcat_image_transformer(current_model: Any, call_kwargs: dict) -> bool:
|
||||
def keep_longcat_prompt_shape(
|
||||
call_kwargs: dict, _current_model: Any, _buckets: tuple[int, ...]
|
||||
) -> dict:
|
||||
# LongCat always supplies the complete 512-token prompt body to its DiT.
|
||||
# Generic bucket padding would only create larger, unused graph signatures.
|
||||
# LongCat supplies the complete 512-token prompt body to its DiT. Editing
|
||||
# also prepends VL image tokens, whose count depends on the input shape.
|
||||
# Preserve this exact sequence and the corresponding txt_ids; padding
|
||||
# would change the attention/RoPE semantics for the unmasked joint sequence.
|
||||
return call_kwargs
|
||||
|
||||
|
||||
|
||||
@@ -64,6 +64,30 @@ logger = init_logger(__name__)
|
||||
|
||||
_LONGCAT_QKNORM_ROPE = BitExactFusionGate("LongCat fused QKNorm+RoPE")
|
||||
_LONGCAT_LN_MOD = BitExactFusionGate("LongCat fused LN+modulate", per_signature=True)
|
||||
_LONGCAT_GELU_CAT = BitExactFusionGate("LongCat fused GELU+cat")
|
||||
|
||||
|
||||
def _longcat_gelu_cat(
|
||||
attn: torch.Tensor, mlp: torch.Tensor, activation: nn.Module
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
type(activation) is nn.GELU
|
||||
and activation.approximate == "tanh"
|
||||
and _LONGCAT_GELU_CAT.can_attempt_once()
|
||||
and diffusion_ops.can_use_fused_gelu_tanh_cat(attn, mlp)
|
||||
):
|
||||
try:
|
||||
output = diffusion_ops.fused_gelu_tanh_cat(attn, mlp)
|
||||
except Exception as exc:
|
||||
_LONGCAT_GELU_CAT.on_exception(exc, logger=logger)
|
||||
else:
|
||||
if _LONGCAT_GELU_CAT.verified:
|
||||
return output
|
||||
reference = torch.cat((attn, activation(mlp)), dim=-1)
|
||||
return _LONGCAT_GELU_CAT.accept_or_fallback(
|
||||
output, reference, logger=logger
|
||||
)
|
||||
return torch.cat((attn, activation(mlp)), dim=-1)
|
||||
|
||||
|
||||
def _longcat_norm_modulate(
|
||||
@@ -649,15 +673,15 @@ class _SingleTransformerBlock(nn.Module):
|
||||
|
||||
residual = hidden_states
|
||||
norm_hidden_states, gate = self.norm(hidden_states, emb=temb)
|
||||
if fused_gelu_active(self) and can_use_linear_gelu(
|
||||
gelu_in_projection = fused_gelu_active(self) and can_use_linear_gelu(
|
||||
self.proj_mlp, norm_hidden_states
|
||||
):
|
||||
)
|
||||
if gelu_in_projection:
|
||||
mlp_hidden_states = fused_linear_gelu_tanh(
|
||||
norm_hidden_states, self.proj_mlp.weight, self.proj_mlp.bias
|
||||
)
|
||||
else:
|
||||
mlp_hidden_states, _ = self.proj_mlp(norm_hidden_states)
|
||||
mlp_hidden_states = self.act_mlp(mlp_hidden_states)
|
||||
attn_output = self.attn(
|
||||
hidden_states=norm_hidden_states,
|
||||
image_rotary_emb=image_rotary_emb,
|
||||
@@ -666,7 +690,12 @@ class _SingleTransformerBlock(nn.Module):
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
positions=positions,
|
||||
)
|
||||
hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2)
|
||||
if gelu_in_projection:
|
||||
hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2)
|
||||
else:
|
||||
hidden_states = _longcat_gelu_cat(
|
||||
attn_output, mlp_hidden_states, self.act_mlp
|
||||
)
|
||||
gate = gate.unsqueeze(1)
|
||||
hidden_states, _ = self.proj_out(hidden_states)
|
||||
hidden_states = residual_gate_add(residual, hidden_states, gate)
|
||||
|
||||
@@ -212,6 +212,7 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS = frozenset(
|
||||
"lightricks/ltx-2",
|
||||
"lightricks/ltx-2.3",
|
||||
"meituan-longcat/longcat-image",
|
||||
"meituan-longcat/longcat-image-edit-turbo",
|
||||
"ltx-2",
|
||||
"ltx-2.3",
|
||||
"minimax-h3",
|
||||
@@ -239,6 +240,7 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS = frozenset(
|
||||
"LTX2PipelineConfig",
|
||||
"LTX23PipelineConfig",
|
||||
"LongCatImagePipelineConfig",
|
||||
"LongCatImageEditPipelineConfig",
|
||||
"MiniMaxH3PipelineConfig",
|
||||
"QwenImagePipelineConfig",
|
||||
"QwenImage21PipelineConfig",
|
||||
|
||||
@@ -225,6 +225,23 @@ class TestDiffusionBCGPadding(unittest.TestCase):
|
||||
self.assertEqual(out["encoder_hidden_states"].shape, (1, 512, 3584))
|
||||
self.assertEqual(out["txt_ids"].shape, (512, 3))
|
||||
|
||||
def test_longcat_edit_keeps_vl_prefix_and_image_grid(self):
|
||||
kwargs = {
|
||||
"hidden_states": torch.zeros(1, 8374, 64),
|
||||
"encoder_hidden_states": torch.zeros(1, 859, 64),
|
||||
"encoder_hidden_states_mask": [torch.ones(1, 859, dtype=torch.bool)],
|
||||
"txt_ids": torch.zeros(859, 3),
|
||||
"img_ids": torch.zeros(8374, 3),
|
||||
}
|
||||
with self._patch_buckets(64, 128, 256, 512, 1024):
|
||||
out = self.stage._bcg_pad_prompt_kwargs(
|
||||
kwargs, current_model=self.longcat_model
|
||||
)
|
||||
self.assertIs(out, kwargs)
|
||||
self.assertEqual(out["encoder_hidden_states"].shape[1], 859)
|
||||
self.assertEqual(out["txt_ids"].shape, (859, 3))
|
||||
self.assertEqual(out["img_ids"].shape, (8374, 3))
|
||||
|
||||
def test_qwen_default_bucket_preserves_mask(self):
|
||||
def kwargs(valid_len: int):
|
||||
mask = torch.zeros(1, 64, dtype=torch.bool)
|
||||
@@ -473,6 +490,7 @@ class TestDiffusionBCGPadding(unittest.TestCase):
|
||||
def test_image_generation_models_are_registered_as_bcg_supported(self):
|
||||
for model_id in (
|
||||
"meituan-longcat/longcat-image",
|
||||
"meituan-longcat/longcat-image-edit-turbo",
|
||||
"qwen/qwen-image",
|
||||
"qwen/qwen-image-2512",
|
||||
"tongyi-mai/z-image",
|
||||
@@ -484,6 +502,7 @@ class TestDiffusionBCGPadding(unittest.TestCase):
|
||||
for config_name in (
|
||||
"GlmImagePipelineConfig",
|
||||
"LongCatImagePipelineConfig",
|
||||
"LongCatImageEditPipelineConfig",
|
||||
"QwenImagePipelineConfig",
|
||||
"ZImagePipelineConfig",
|
||||
):
|
||||
|
||||
@@ -11,6 +11,10 @@ from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import (
|
||||
LongCatImagePipelineConfig,
|
||||
_calculate_edit_dimensions,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.longcat_image import (
|
||||
LongCatImageEditSamplingParams,
|
||||
LongCatImageEditTurboSamplingParams,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -37,6 +41,38 @@ def test_edit_dimensions_match_diffusers_formula():
|
||||
assert got_w % 16 == 0 and got_h % 16 == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params_cls", [LongCatImageEditSamplingParams, LongCatImageEditTurboSamplingParams]
|
||||
)
|
||||
@pytest.mark.parametrize("size", [(1264, 848), (848, 1264), (1024, 1024)])
|
||||
def test_explicit_warmup_image_preserves_edit_grids(params_cls, size):
|
||||
import base64
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
params = params_cls()
|
||||
req = _make_batch(width=size[0], height=size[1], image_path=["placeholder.png"])
|
||||
server_args = _make_batch(warmup_resolutions=[f"{size[0]}x{size[1]}"])
|
||||
params.prepare_synthetic_warmup_request_for_queue(req, server_args)
|
||||
image = Image.open(io.BytesIO(base64.b64decode(req.image_path[0].split(",", 1)[1])))
|
||||
assert image.size == size
|
||||
config = LongCatImageEditPipelineConfig()
|
||||
assert config.calculate_condition_image_size(image, *image.size) == size
|
||||
ids = config._edit_img_ids(
|
||||
_make_batch(width=size[0], height=size[1]), 859, torch.device("cpu")
|
||||
)
|
||||
assert ids.shape == (2 * (size[0] // 16) * (size[1] // 16), 3)
|
||||
|
||||
|
||||
def test_default_warmup_keeps_existing_placeholder():
|
||||
req = _make_batch(width=512, height=512, image_path=["placeholder.png"])
|
||||
LongCatImageEditSamplingParams().prepare_synthetic_warmup_request_for_queue(
|
||||
req, _make_batch(warmup_resolutions=None)
|
||||
)
|
||||
assert req.image_path == ["placeholder.png"]
|
||||
|
||||
|
||||
def test_edit_config_task_type_and_generator(edit_config):
|
||||
assert edit_config.task_type == ModelTaskType.I2I
|
||||
assert edit_config.generator_device == "cpu"
|
||||
|
||||
Reference in New Issue
Block a user