diff --git a/python/sglang/kernels/jit/csrc/diffusion/gelu_tanh_cat.cuh b/python/sglang/kernels/jit/csrc/diffusion/gelu_tanh_cat.cuh new file mode 100644 index 0000000000..9172bf0340 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/diffusion/gelu_tanh_cat.cuh @@ -0,0 +1,90 @@ +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +namespace sglang { + +template +__global__ void gelu_tanh_cat_kernel( + const T* __restrict__ attn, + const T* __restrict__ mlp, + T* __restrict__ output, + uint32_t num_vecs, + uint32_t attn_vecs, + uint32_t mlp_vecs) { + using vec_t = device::AlignedVector; + const uint32_t row_vecs = attn_vecs + mlp_vecs; + const uint32_t stride = blockDim.x * gridDim.x; + for (uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_vecs; i += stride) { + const uint32_t row = i / row_vecs; + const uint32_t col = i % row_vecs; + vec_t value; + if (col < attn_vecs) { + value.load(attn, row * attn_vecs + col); + } else { + value.load(mlp, row * mlp_vecs + col - attn_vecs); +#pragma unroll + for (int j = 0; j < kVecN; ++j) { + const float x = device::cast(value[j]); + // Match aten's tanh GELU operation order, including the BF16 store + // that precedes the eager concatenation. + constexpr float kBeta = 0.7978845608028654f; + constexpr float kKappa = 0.044715f; + const float cube = x * x * x; + const float inner = kBeta * (x + kKappa * cube); + value[j] = device::cast(0.5f * x * (1.0f + tanhf(inner))); + } + } + value.store(output, i); + } +} + +template +void gelu_tanh_cat(tvm::ffi::TensorView attn, tvm::ffi::TensorView mlp, tvm::ffi::TensorView output) { + using namespace host; + auto rows = SymbolicSize{"rows"}; + auto attn_width = SymbolicSize{"attn_width"}; + auto mlp_width = SymbolicSize{"mlp_width"}; + auto device_ = SymbolicDevice{}; + device_.set_options(); + TensorMatcher({rows, attn_width}).with_dtype().with_device(device_).verify(attn); + TensorMatcher({rows, mlp_width}).with_dtype().with_device(device_).verify(mlp); + const int64_t a = attn_width.unwrap(); + const int64_t m = mlp_width.unwrap(); + TensorMatcher({rows, a + m}).with_dtype().with_device(device_).verify(output); + constexpr int kVecN = 16 / sizeof(T); + CHECK_HOST(a > 0 && m > 0 && a % kVecN == 0 && m % kVecN == 0) + << "gelu_tanh_cat: both widths must be positive multiples of " << kVecN; + CHECK_HOST(rows.unwrap() > 0) << "gelu_tanh_cat: rows must be positive"; + CHECK_HOST( + reinterpret_cast(attn.data_ptr()) % 16 == 0 && reinterpret_cast(mlp.data_ptr()) % 16 == 0 && + reinterpret_cast(output.data_ptr()) % 16 == 0) + << "gelu_tanh_cat: tensors must be 16-byte aligned"; + const int64_t n = rows.unwrap() * ((a + m) / kVecN); + CHECK_HOST(n <= std::numeric_limits::max()) << "gelu_tanh_cat: tensor is too large"; + constexpr int kBlockSize = 256; + const auto kernel = gelu_tanh_cat_kernel; + const int64_t occupancy = runtime::get_blocks_per_sm(kernel, kBlockSize); + const int64_t sms = runtime::get_sm_count(device_.unwrap().device_id); + const int64_t grid = std::min(sms * occupancy, div_ceil(n, int64_t{kBlockSize})); + LaunchKernel(grid, kBlockSize, device_.unwrap())( + kernel, + static_cast(attn.data_ptr()), + static_cast(mlp.data_ptr()), + static_cast(output.data_ptr()), + static_cast(n), + static_cast(a / kVecN), + static_cast(m / kVecN)); +} + +} // namespace sglang diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 3360619a3c..9e440ed888 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -49,6 +49,13 @@ _HIP = frozenset({CapabilityRequirement.HIP}) # inventory: callers name the one they want via ``select_kernel``. # --------------------------------------------------------------------------- _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = ( + ( + "diffusion.gelu_tanh_cat", + KernelBackend.JIT, + "activation.gelu_tanh_cat_jit:fused_gelu_tanh_cat", + _CUDA, + "BF16 tanh GELU plus channel concatenation.", + ), ( "diffusion.apply_group_norm_silu", KernelBackend.TRITON, @@ -527,6 +534,8 @@ for _op, _backend, _target, _caps, _description in _SPECS: # then symbol; a new public kernel belongs here and nowhere else. # --------------------------------------------------------------------------- _EXPORTS: dict[str, str] = { + "can_use_fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit", + "fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit", "load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader", # Normalization: RMSNorm / LayerNorm / GroupNorm and their fused epilogues "can_defer_flux2_gated_residual": "norm.flux2_gated_resnorm_jit", diff --git a/python/sglang/kernels/ops/diffusion/activation/gelu_tanh_cat_jit.py b/python/sglang/kernels/ops/diffusion/activation/gelu_tanh_cat_jit.py new file mode 100644 index 0000000000..5195686cb9 --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/activation/gelu_tanh_cat_jit.py @@ -0,0 +1,64 @@ +"""Concatenate attention with tanh GELU while retaining the BF16 rounding boundary. + +The inputs are contiguous BF16 tensors with equal leading dimensions and +16-byte aligned rows. The result matches ``cat((attn, gelu(mlp)), dim=-1)``. +""" + +from __future__ import annotations + +import torch + +from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args +from sglang.srt.utils.custom_op import register_custom_op + + +@cache_once +def _module(): + args = make_cpp_args(torch.bfloat16) + return load_jit( + "gelu_tanh_cat", + *args, + cuda_files=["diffusion/gelu_tanh_cat.cuh"], + cuda_wrappers=[("gelu_tanh_cat", f"gelu_tanh_cat<{args}>")], + extra_cuda_cflags=["-lineinfo"], + ) + + +def can_use_fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> bool: + return ( + not torch.is_grad_enabled() + and not torch.compiler.is_compiling() + and torch.version.hip is None + and attn.is_cuda + and attn.dtype == mlp.dtype == torch.bfloat16 + and attn.device == mlp.device + and attn.ndim >= 2 + and attn.shape[:-1] == mlp.shape[:-1] + and attn.shape[-1] > 0 + and mlp.shape[-1] > 0 + and attn.shape[-1] % 8 == mlp.shape[-1] % 8 == 0 + and attn.is_contiguous() + and mlp.is_contiguous() + and attn.numel() > 0 + and (attn.numel() + mlp.numel()) // 8 <= 2**31 - 1 + and attn.data_ptr() % 16 == mlp.data_ptr() % 16 == 0 + ) + + +@register_custom_op(mutates_args=["output"]) +def _gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor, output: torch.Tensor) -> None: + _module().gelu_tanh_cat( + attn.view(-1, attn.shape[-1]), + mlp.view(-1, mlp.shape[-1]), + output.view(-1, output.shape[-1]), + ) + + +def fused_gelu_tanh_cat(attn: torch.Tensor, mlp: torch.Tensor) -> torch.Tensor: + output = torch.empty( + (*attn.shape[:-1], attn.shape[-1] + mlp.shape[-1]), + device=attn.device, + dtype=attn.dtype, + ) + _gelu_tanh_cat(attn, mlp, output) + return output diff --git a/python/sglang/multimodal_gen/configs/sample/longcat_image.py b/python/sglang/multimodal_gen/configs/sample/longcat_image.py index 49890eaf5d..3860e919b3 100644 --- a/python/sglang/multimodal_gen/configs/sample/longcat_image.py +++ b/python/sglang/multimodal_gen/configs/sample/longcat_image.py @@ -40,6 +40,23 @@ class LongCatImageEditSamplingParams(SamplingParams): enable_cfg_renorm: bool = False enable_prompt_rewrite: bool = False + def prepare_synthetic_warmup_request_for_queue(self, req, server_args) -> None: + if server_args.warmup_resolutions is None: + return + # Editing derives both the output grid and VL prefix from the input + # image. A square placeholder discards an explicit non-square warmup + # resolution and captures a graph that the real request cannot replay. + import base64 + import io + + from PIL import Image + + buffer = io.BytesIO() + Image.new("RGB", (req.width, req.height)).save(buffer, format="PNG") + req.image_path = [ + "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + ] + @dataclass class LongCatImageEditTurboSamplingParams(LongCatImageEditSamplingParams): diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/longcat_image.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/longcat_image.py index e972559908..b80859706f 100644 --- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/longcat_image.py +++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/model_padders/longcat_image.py @@ -23,8 +23,10 @@ def is_longcat_image_transformer(current_model: Any, call_kwargs: dict) -> bool: def keep_longcat_prompt_shape( call_kwargs: dict, _current_model: Any, _buckets: tuple[int, ...] ) -> dict: - # LongCat always supplies the complete 512-token prompt body to its DiT. - # Generic bucket padding would only create larger, unused graph signatures. + # LongCat supplies the complete 512-token prompt body to its DiT. Editing + # also prepends VL image tokens, whose count depends on the input shape. + # Preserve this exact sequence and the corresponding txt_ids; padding + # would change the attention/RoPE semantics for the unmasked joint sequence. return call_kwargs diff --git a/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py b/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py index 9f8ed95c33..559d7f2e92 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py @@ -64,6 +64,30 @@ logger = init_logger(__name__) _LONGCAT_QKNORM_ROPE = BitExactFusionGate("LongCat fused QKNorm+RoPE") _LONGCAT_LN_MOD = BitExactFusionGate("LongCat fused LN+modulate", per_signature=True) +_LONGCAT_GELU_CAT = BitExactFusionGate("LongCat fused GELU+cat") + + +def _longcat_gelu_cat( + attn: torch.Tensor, mlp: torch.Tensor, activation: nn.Module +) -> torch.Tensor: + if ( + type(activation) is nn.GELU + and activation.approximate == "tanh" + and _LONGCAT_GELU_CAT.can_attempt_once() + and diffusion_ops.can_use_fused_gelu_tanh_cat(attn, mlp) + ): + try: + output = diffusion_ops.fused_gelu_tanh_cat(attn, mlp) + except Exception as exc: + _LONGCAT_GELU_CAT.on_exception(exc, logger=logger) + else: + if _LONGCAT_GELU_CAT.verified: + return output + reference = torch.cat((attn, activation(mlp)), dim=-1) + return _LONGCAT_GELU_CAT.accept_or_fallback( + output, reference, logger=logger + ) + return torch.cat((attn, activation(mlp)), dim=-1) def _longcat_norm_modulate( @@ -649,15 +673,15 @@ class _SingleTransformerBlock(nn.Module): residual = hidden_states norm_hidden_states, gate = self.norm(hidden_states, emb=temb) - if fused_gelu_active(self) and can_use_linear_gelu( + gelu_in_projection = fused_gelu_active(self) and can_use_linear_gelu( self.proj_mlp, norm_hidden_states - ): + ) + if gelu_in_projection: mlp_hidden_states = fused_linear_gelu_tanh( norm_hidden_states, self.proj_mlp.weight, self.proj_mlp.bias ) else: mlp_hidden_states, _ = self.proj_mlp(norm_hidden_states) - mlp_hidden_states = self.act_mlp(mlp_hidden_states) attn_output = self.attn( hidden_states=norm_hidden_states, image_rotary_emb=image_rotary_emb, @@ -666,7 +690,12 @@ class _SingleTransformerBlock(nn.Module): cos_sin_cache=cos_sin_cache, positions=positions, ) - hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) + if gelu_in_projection: + hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) + else: + hidden_states = _longcat_gelu_cat( + attn_output, mlp_hidden_states, self.act_mlp + ) gate = gate.unsqueeze(1) hidden_states, _ = self.proj_out(hidden_states) hidden_states = residual_gate_add(residual, hidden_states, gate) diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 811f7c2280..9424ee44bb 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -212,6 +212,7 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS = frozenset( "lightricks/ltx-2", "lightricks/ltx-2.3", "meituan-longcat/longcat-image", + "meituan-longcat/longcat-image-edit-turbo", "ltx-2", "ltx-2.3", "minimax-h3", @@ -239,6 +240,7 @@ BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS = frozenset( "LTX2PipelineConfig", "LTX23PipelineConfig", "LongCatImagePipelineConfig", + "LongCatImageEditPipelineConfig", "MiniMaxH3PipelineConfig", "QwenImagePipelineConfig", "QwenImage21PipelineConfig", diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py index c14a1ca1e5..784beb4b10 100644 --- a/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py @@ -225,6 +225,23 @@ class TestDiffusionBCGPadding(unittest.TestCase): self.assertEqual(out["encoder_hidden_states"].shape, (1, 512, 3584)) self.assertEqual(out["txt_ids"].shape, (512, 3)) + def test_longcat_edit_keeps_vl_prefix_and_image_grid(self): + kwargs = { + "hidden_states": torch.zeros(1, 8374, 64), + "encoder_hidden_states": torch.zeros(1, 859, 64), + "encoder_hidden_states_mask": [torch.ones(1, 859, dtype=torch.bool)], + "txt_ids": torch.zeros(859, 3), + "img_ids": torch.zeros(8374, 3), + } + with self._patch_buckets(64, 128, 256, 512, 1024): + out = self.stage._bcg_pad_prompt_kwargs( + kwargs, current_model=self.longcat_model + ) + self.assertIs(out, kwargs) + self.assertEqual(out["encoder_hidden_states"].shape[1], 859) + self.assertEqual(out["txt_ids"].shape, (859, 3)) + self.assertEqual(out["img_ids"].shape, (8374, 3)) + def test_qwen_default_bucket_preserves_mask(self): def kwargs(valid_len: int): mask = torch.zeros(1, 64, dtype=torch.bool) @@ -473,6 +490,7 @@ class TestDiffusionBCGPadding(unittest.TestCase): def test_image_generation_models_are_registered_as_bcg_supported(self): for model_id in ( "meituan-longcat/longcat-image", + "meituan-longcat/longcat-image-edit-turbo", "qwen/qwen-image", "qwen/qwen-image-2512", "tongyi-mai/z-image", @@ -484,6 +502,7 @@ class TestDiffusionBCGPadding(unittest.TestCase): for config_name in ( "GlmImagePipelineConfig", "LongCatImagePipelineConfig", + "LongCatImageEditPipelineConfig", "QwenImagePipelineConfig", "ZImagePipelineConfig", ): diff --git a/python/sglang/multimodal_gen/test/unit/test_longcat_image_edit_config.py b/python/sglang/multimodal_gen/test/unit/test_longcat_image_edit_config.py index 3069a86306..0430a77cd5 100644 --- a/python/sglang/multimodal_gen/test/unit/test_longcat_image_edit_config.py +++ b/python/sglang/multimodal_gen/test/unit/test_longcat_image_edit_config.py @@ -11,6 +11,10 @@ from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import ( LongCatImagePipelineConfig, _calculate_edit_dimensions, ) +from sglang.multimodal_gen.configs.sample.longcat_image import ( + LongCatImageEditSamplingParams, + LongCatImageEditTurboSamplingParams, +) @pytest.fixture @@ -37,6 +41,38 @@ def test_edit_dimensions_match_diffusers_formula(): assert got_w % 16 == 0 and got_h % 16 == 0 +@pytest.mark.parametrize( + "params_cls", [LongCatImageEditSamplingParams, LongCatImageEditTurboSamplingParams] +) +@pytest.mark.parametrize("size", [(1264, 848), (848, 1264), (1024, 1024)]) +def test_explicit_warmup_image_preserves_edit_grids(params_cls, size): + import base64 + import io + + from PIL import Image + + params = params_cls() + req = _make_batch(width=size[0], height=size[1], image_path=["placeholder.png"]) + server_args = _make_batch(warmup_resolutions=[f"{size[0]}x{size[1]}"]) + params.prepare_synthetic_warmup_request_for_queue(req, server_args) + image = Image.open(io.BytesIO(base64.b64decode(req.image_path[0].split(",", 1)[1]))) + assert image.size == size + config = LongCatImageEditPipelineConfig() + assert config.calculate_condition_image_size(image, *image.size) == size + ids = config._edit_img_ids( + _make_batch(width=size[0], height=size[1]), 859, torch.device("cpu") + ) + assert ids.shape == (2 * (size[0] // 16) * (size[1] // 16), 3) + + +def test_default_warmup_keeps_existing_placeholder(): + req = _make_batch(width=512, height=512, image_path=["placeholder.png"]) + LongCatImageEditSamplingParams().prepare_synthetic_warmup_request_for_queue( + req, _make_batch(warmup_resolutions=None) + ) + assert req.image_path == ["placeholder.png"] + + def test_edit_config_task_type_and_generator(edit_config): assert edit_config.task_type == ModelTaskType.I2I assert edit_config.generator_device == "cpu" diff --git a/test/registered/kernels/ops/diffusion/test_gelu_tanh_cat.py b/test/registered/kernels/ops/diffusion/test_gelu_tanh_cat.py new file mode 100644 index 0000000000..85dbacc658 --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_gelu_tanh_cat.py @@ -0,0 +1,47 @@ +"""Exact GELU rounding, layout, validation and replay for the concat fusion.""" + +import sys + +import pytest +import torch +import torch.nn.functional as F + +from sglang.kernels.ops.diffusion import ( + fused_gelu_tanh_cat, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") + + +@torch.inference_mode() +def test_all_finite_bf16_values(): + bits = torch.arange(65536, device="cuda", dtype=torch.int32).to(torch.int16) + values = bits.view(torch.bfloat16) + mlp = values[torch.isfinite(values)].view(1, -1, 8) + attn = torch.randn_like(mlp) + actual = fused_gelu_tanh_cat(attn, mlp) + expected = torch.cat((attn, F.gelu(mlp, approximate="tanh")), dim=-1) + # Compare bit patterns so signed zero is covered too. + assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)) + + +@torch.inference_mode() +def test_changed_inputs_in_graph_replay(): + attn = torch.randn(2, 17, 64, device="cuda", dtype=torch.bfloat16) + mlp = torch.randn(2, 17, 256, device="cuda", dtype=torch.bfloat16) + fused_gelu_tanh_cat(attn, mlp) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = fused_gelu_tanh_cat(attn, mlp) + attn.neg_() + mlp.add_(0.75) + graph.replay() + assert torch.equal( + actual, torch.cat((attn, F.gelu(mlp, approximate="tanh")), dim=-1) + ) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))