[diffusion] perf: keep Cosmos3's fused kernels in the graph (#43577)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
d45ef0f2d2
commit
b3a65efe7e
@@ -42,6 +42,11 @@ from sglang.multimodal_gen.runtime.layers.rotary_embedding import (
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.platforms.aiter import USE_AITER
|
||||
from sglang.multimodal_gen.runtime.utils.common import get_bool_env_var
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
# An opaque op to torch.compile, as srt.models.utils registers it: a bare JIT
|
||||
# launch graph-breaks every layer that normalizes q/k.
|
||||
fused_inplace_qknorm = register_custom_op(fused_inplace_qknorm, mutates_args=["q", "k"])
|
||||
|
||||
_is_cuda = current_platform.is_cuda()
|
||||
_is_rocm = current_platform.is_rocm()
|
||||
@@ -1186,7 +1191,6 @@ def apply_qk_norm_rope(
|
||||
if (
|
||||
fused_enabled
|
||||
and (_is_cuda or _is_rocm)
|
||||
and not torch.compiler.is_compiling()
|
||||
and allow_inplace
|
||||
and (q_eps == k_eps)
|
||||
and q.dtype in (torch.float16, torch.bfloat16)
|
||||
|
||||
@@ -551,7 +551,6 @@ class Cosmos3DenseMLP(nn.Module):
|
||||
and up.dtype == torch.bfloat16
|
||||
and up.is_contiguous()
|
||||
and not torch.is_grad_enabled()
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
up = relu2(up, fast_math=False)
|
||||
else:
|
||||
@@ -818,7 +817,6 @@ class Cosmos3CrossAttention(nn.Module):
|
||||
use_fused_kv_pack = (
|
||||
use_fused_qk_norm_rope
|
||||
and q.device.type == "cuda"
|
||||
and not torch.compiler.is_compiling()
|
||||
and get_sp_world_size() == 1
|
||||
and q.dtype == k.dtype == v.dtype == k_und.dtype == v_und.dtype
|
||||
and self.norm_q.weight.dtype == q.dtype
|
||||
@@ -1789,7 +1787,6 @@ class Cosmos3OmniTransformer(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
use_fused_qk_norm_rope = T > 1 or (
|
||||
enable_t1_fused_qk_norm_rope
|
||||
and hidden_gen.device.type == "cuda"
|
||||
and not torch.compiler.is_compiling()
|
||||
and get_sp_world_size() == 1
|
||||
and can_use_fused_inplace_qknorm_rope(
|
||||
self.head_dim,
|
||||
|
||||
@@ -3,6 +3,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.dits.flux import (
|
||||
@@ -212,6 +213,15 @@ def test_flux_compile_conditions_match_both_block_types():
|
||||
assert not matches(nn.Linear(2, 2))
|
||||
|
||||
|
||||
def test_diffusion_qknorm_is_an_opaque_op_for_torch_compile():
|
||||
# Importing the layer registers it; a bare JIT launch would graph-break.
|
||||
from sglang.multimodal_gen.runtime.layers import layernorm # noqa: F401
|
||||
|
||||
schema = torch.ops.sglang.fused_inplace_qknorm.default._schema
|
||||
args = {arg.name: arg for arg in schema.arguments}
|
||||
assert args["q"].alias_info.is_write and args["k"].alias_info.is_write
|
||||
|
||||
|
||||
def test_compile_matching_submodules_matches_only_declared_regions():
|
||||
model = _RegionalModel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user