From b3a65efe7ec32f1527ed7e95df37b5e5239e1c08 Mon Sep 17 00:00:00 2001 From: Mick Date: Sun, 11 Oct 2026 10:53:43 +0800 Subject: [PATCH] [diffusion] perf: keep Cosmos3's fused kernels in the graph (#43577) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../sglang/multimodal_gen/runtime/layers/layernorm.py | 6 +++++- .../multimodal_gen/runtime/models/dits/cosmos3video.py | 3 --- .../test/unit/test_regional_torch_compile.py | 10 ++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/layers/layernorm.py b/python/sglang/multimodal_gen/runtime/layers/layernorm.py index 0995cc6b4f..362fdf2784 100755 --- a/python/sglang/multimodal_gen/runtime/layers/layernorm.py +++ b/python/sglang/multimodal_gen/runtime/layers/layernorm.py @@ -42,6 +42,11 @@ from sglang.multimodal_gen.runtime.layers.rotary_embedding import ( from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.platforms.aiter import USE_AITER from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var +from sglang.srt.utils.custom_op import register_custom_op + +# An opaque op to torch.compile, as srt.models.utils registers it: a bare JIT +# launch graph-breaks every layer that normalizes q/k. +fused_inplace_qknorm = register_custom_op(fused_inplace_qknorm, mutates_args=["q", "k"]) _is_cuda = current_platform.is_cuda() _is_rocm = current_platform.is_rocm() @@ -1186,7 +1191,6 @@ def apply_qk_norm_rope( if ( fused_enabled and (_is_cuda or _is_rocm) - and not torch.compiler.is_compiling() and allow_inplace and (q_eps == k_eps) and q.dtype in (torch.float16, torch.bfloat16) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py index 183edf0637..18dcc9a7aa 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py @@ -551,7 +551,6 @@ class Cosmos3DenseMLP(nn.Module): and up.dtype == torch.bfloat16 and up.is_contiguous() and not torch.is_grad_enabled() - and not torch.compiler.is_compiling() ): up = relu2(up, fast_math=False) else: @@ -818,7 +817,6 @@ class Cosmos3CrossAttention(nn.Module): use_fused_kv_pack = ( use_fused_qk_norm_rope and q.device.type == "cuda" - and not torch.compiler.is_compiling() and get_sp_world_size() == 1 and q.dtype == k.dtype == v.dtype == k_und.dtype == v_und.dtype and self.norm_q.weight.dtype == q.dtype @@ -1789,7 +1787,6 @@ class Cosmos3OmniTransformer(CachableDiT, LayerwiseOffloadableModuleMixin): use_fused_qk_norm_rope = T > 1 or ( enable_t1_fused_qk_norm_rope and hidden_gen.device.type == "cuda" - and not torch.compiler.is_compiling() and get_sp_world_size() == 1 and can_use_fused_inplace_qknorm_rope( self.head_dim, diff --git a/python/sglang/multimodal_gen/test/unit/test_regional_torch_compile.py b/python/sglang/multimodal_gen/test/unit/test_regional_torch_compile.py index 74937a5dc7..e0420159f5 100644 --- a/python/sglang/multimodal_gen/test/unit/test_regional_torch_compile.py +++ b/python/sglang/multimodal_gen/test/unit/test_regional_torch_compile.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest +import torch import torch.nn as nn from sglang.multimodal_gen.runtime.models.dits.flux import ( @@ -212,6 +213,15 @@ def test_flux_compile_conditions_match_both_block_types(): assert not matches(nn.Linear(2, 2)) +def test_diffusion_qknorm_is_an_opaque_op_for_torch_compile(): + # Importing the layer registers it; a bare JIT launch would graph-break. + from sglang.multimodal_gen.runtime.layers import layernorm # noqa: F401 + + schema = torch.ops.sglang.fused_inplace_qknorm.default._schema + args = {arg.name: arg for arg in schema.arguments} + assert args["q"].alias_info.is_write and args["k"].alias_info.is_write + + def test_compile_matching_submodules_matches_only_declared_regions(): model = _RegionalModel()