[Diffusion] Fuse lossless SenseNova RoPE for 5% faster H200 inference (#40374)
This commit is contained in:
@@ -33,6 +33,10 @@ _PUBLIC_OVERRIDE_FIELDS = {
|
||||
"output_quality",
|
||||
"output_compression",
|
||||
"quality",
|
||||
"profile",
|
||||
"profile_all_stages",
|
||||
"num_profiled_timesteps",
|
||||
"perf_dump_path",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -429,6 +429,42 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
|
||||
Returns:
|
||||
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
|
||||
"""
|
||||
if (
|
||||
q.is_cuda
|
||||
and q.dtype is torch.bfloat16
|
||||
and k.dtype is torch.bfloat16
|
||||
and cos.dtype is torch.bfloat16
|
||||
and sin.dtype is torch.bfloat16
|
||||
and q.device == k.device == cos.device == sin.device
|
||||
and not torch.is_grad_enabled()
|
||||
and not torch.compiler.is_compiling()
|
||||
and unsqueeze_dim == 1
|
||||
and q.ndim == k.ndim == 4
|
||||
and q.numel() > 0
|
||||
and k.numel() > 0
|
||||
and q.shape[2] >= 128
|
||||
and q.shape[0] == k.shape[0]
|
||||
and q.shape[2:] == k.shape[2:]
|
||||
and q.shape[-1] in (32, 64)
|
||||
and cos.shape == sin.shape == (q.shape[0], q.shape[2], q.shape[3])
|
||||
):
|
||||
from sglang.kernels.ops.diffusion.rope.rope_rotate_half_bitexact import (
|
||||
fused_rope_rotate_half_bitexact,
|
||||
)
|
||||
|
||||
# The kernel rounds both products to bf16 before adding, matching
|
||||
# the eager chain. Spatial axes are slices of the normalized HW
|
||||
# heads; materialize those slices in the kernel's BSHD layout.
|
||||
cos_rows = cos.reshape(-1, cos.shape[-1]).contiguous()
|
||||
sin_rows = sin.reshape(-1, sin.shape[-1]).contiguous()
|
||||
q_embed = fused_rope_rotate_half_bitexact(
|
||||
q.transpose(1, 2).contiguous(), cos_rows, sin_rows
|
||||
)
|
||||
k_embed = fused_rope_rotate_half_bitexact(
|
||||
k.transpose(1, 2).contiguous(), cos_rows, sin_rows
|
||||
)
|
||||
return q_embed.transpose(1, 2), k_embed.transpose(1, 2)
|
||||
|
||||
cos = cos.unsqueeze(unsqueeze_dim)
|
||||
sin = sin.unsqueeze(unsqueeze_dim)
|
||||
q_embed = (q * cos) + (rotate_half(q) * sin)
|
||||
|
||||
@@ -1042,6 +1042,10 @@ def test_sensenova_u1_cli_args_expose_only_sglang_compatible_fields():
|
||||
guidance_scale=4.5,
|
||||
num_inference_steps=30,
|
||||
num_outputs_per_prompt=2,
|
||||
profile=True,
|
||||
profile_all_stages=True,
|
||||
num_profiled_timesteps=3,
|
||||
perf_dump_path="/tmp/sensenova-perf.json",
|
||||
cfg_norm="global",
|
||||
timestep_shift=9.0,
|
||||
think_mode=True,
|
||||
@@ -1058,6 +1062,11 @@ def test_sensenova_u1_cli_args_expose_only_sglang_compatible_fields():
|
||||
assert "cfg_norm" not in cli_args
|
||||
assert "timestep_shift" not in cli_args
|
||||
assert "think_mode" not in cli_args
|
||||
request = Req(sampling_params=SenseNovaU1SamplingParams(**cli_args))
|
||||
assert request.profile
|
||||
assert request.profile_all_stages
|
||||
assert request.num_profiled_timesteps == 3
|
||||
assert request.perf_dump_path == "/tmp/sensenova-perf.json"
|
||||
|
||||
|
||||
def test_sensenova_u1_generation_stage_uses_sglang_params_and_single_model_batch():
|
||||
|
||||
Reference in New Issue
Block a user