[diffusion] perf: keep Cosmos3's fused kernels in the graph (#43577)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-11 10:53:43 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent d45ef0f2d2
commit b3a65efe7e
3 changed files with 15 additions and 4 deletions
@@ -42,6 +42,11 @@ from sglang.multimodal_gen.runtime.layers.rotary_embedding import (
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.platforms.aiter import USE_AITER
from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var
from sglang.srt.utils.custom_op import register_custom_op
# An opaque op to torch.compile, as srt.models.utils registers it: a bare JIT
# launch graph-breaks every layer that normalizes q/k.
fused_inplace_qknorm = register_custom_op(fused_inplace_qknorm, mutates_args=["q", "k"])
_is_cuda = current_platform.is_cuda()
_is_rocm = current_platform.is_rocm()
@@ -1186,7 +1191,6 @@ def apply_qk_norm_rope(
if (
fused_enabled
and (_is_cuda or _is_rocm)
and not torch.compiler.is_compiling()
and allow_inplace
and (q_eps == k_eps)
and q.dtype in (torch.float16, torch.bfloat16)
@@ -551,7 +551,6 @@ class Cosmos3DenseMLP(nn.Module):
and up.dtype == torch.bfloat16
and up.is_contiguous()
and not torch.is_grad_enabled()
and not torch.compiler.is_compiling()
):
up = relu2(up, fast_math=False)
else:
@@ -818,7 +817,6 @@ class Cosmos3CrossAttention(nn.Module):
use_fused_kv_pack = (
use_fused_qk_norm_rope
and q.device.type == "cuda"
and not torch.compiler.is_compiling()
and get_sp_world_size() == 1
and q.dtype == k.dtype == v.dtype == k_und.dtype == v_und.dtype
and self.norm_q.weight.dtype == q.dtype
@@ -1789,7 +1787,6 @@ class Cosmos3OmniTransformer(CachableDiT, LayerwiseOffloadableModuleMixin):
use_fused_qk_norm_rope = T > 1 or (
enable_t1_fused_qk_norm_rope
and hidden_gen.device.type == "cuda"
and not torch.compiler.is_compiling()
and get_sp_world_size() == 1
and can_use_fused_inplace_qknorm_rope(
self.head_dim,
@@ -3,6 +3,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
import torch.nn as nn
from sglang.multimodal_gen.runtime.models.dits.flux import (
@@ -212,6 +213,15 @@ def test_flux_compile_conditions_match_both_block_types():
assert not matches(nn.Linear(2, 2))
def test_diffusion_qknorm_is_an_opaque_op_for_torch_compile():
# Importing the layer registers it; a bare JIT launch would graph-break.
from sglang.multimodal_gen.runtime.layers import layernorm # noqa: F401
schema = torch.ops.sglang.fused_inplace_qknorm.default._schema
args = {arg.name: arg for arg in schema.arguments}
assert args["q"].alias_info.is_write and args["k"].alias_info.is_write
def test_compile_matching_submodules_matches_only_declared_regions():
model = _RegionalModel()