Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
(cherry picked from commit 379ec90fb2)
334 lines
10 KiB
Python
334 lines
10 KiB
Python
from typing import Optional, Tuple, Union
|
|
|
|
import torch
|
|
|
|
from sglang.kernels.jit.utils import (
|
|
cache_once,
|
|
is_arch_support_pdl,
|
|
load_jit,
|
|
make_cpp_args,
|
|
)
|
|
from sglang.srt.utils import is_hip, is_xpu
|
|
|
|
from .kv_layout import KVLayout
|
|
from .utils import make_name
|
|
|
|
_is_hip = is_hip()
|
|
_is_xpu = is_xpu()
|
|
|
|
if _is_xpu:
|
|
from sgl_kernel import fused_k_norm_rope_flashmla as fused_k_norm_rope_flashmla_xpu
|
|
from sgl_kernel import fused_q_norm_rope as fused_q_norm_rope_xpu
|
|
|
|
|
|
@cache_once
|
|
def _jit_fused_rope_module():
|
|
args = make_cpp_args(is_arch_support_pdl())
|
|
return load_jit(
|
|
make_name("fused_rope"),
|
|
*args,
|
|
cuda_files=["deepseek_v4/rope.cuh"],
|
|
cuda_wrappers=[("forward", f"FusedQKRopeKernel<{args}>::forward")],
|
|
)
|
|
|
|
|
|
@cache_once
|
|
def _jit_main_q_norm_rope_module(
|
|
dtype: torch.dtype,
|
|
head_dim: int,
|
|
rope_dim: int,
|
|
apply_norm: bool,
|
|
):
|
|
"""Main MLA path Q kernel: rmsnorm-self + RoPE, warp per (token, head)."""
|
|
args = make_cpp_args(dtype, head_dim, rope_dim, apply_norm, is_arch_support_pdl())
|
|
return load_jit(
|
|
make_name("main_q_norm_rope"),
|
|
*args,
|
|
cuda_files=["deepseek_v4/main_norm_rope.cuh"],
|
|
cuda_wrappers=[
|
|
("forward", f"FusedQNormRopeKernel<{args}>::forward"),
|
|
],
|
|
)
|
|
|
|
|
|
@cache_once
|
|
def _jit_main_k_norm_rope_flashmla_module(
|
|
dtype: torch.dtype,
|
|
head_dim: int,
|
|
rope_dim: int,
|
|
page_size: int,
|
|
layout: KVLayout,
|
|
):
|
|
"""Main MLA path K kernel: rmsnorm + RoPE + write to FlashMLA paged cache."""
|
|
args = make_cpp_args(
|
|
dtype, head_dim, rope_dim, page_size, layout.cpp_name, is_arch_support_pdl()
|
|
)
|
|
return load_jit(
|
|
make_name("main_k_norm_rope_flashmla"),
|
|
*args,
|
|
cuda_files=["deepseek_v4/main_norm_rope.cuh"],
|
|
cuda_wrappers=[
|
|
("forward", f"FusedKNormRopeFlashMLAKernel<{args}>::forward"),
|
|
],
|
|
)
|
|
|
|
|
|
@cache_once
|
|
def _jit_main_k_norm_rope_q_flashmla_module(
|
|
dtype: torch.dtype,
|
|
head_dim: int,
|
|
rope_dim: int,
|
|
page_size: int,
|
|
layout: KVLayout,
|
|
):
|
|
"""ROCm only: the K kernel above with the in-place query rope in the same launch."""
|
|
args = make_cpp_args(
|
|
dtype, head_dim, rope_dim, page_size, layout.cpp_name, is_arch_support_pdl()
|
|
)
|
|
return load_jit(
|
|
make_name("main_k_norm_rope_q_flashmla"),
|
|
*args,
|
|
cuda_files=["deepseek_v4/main_norm_rope.cuh"],
|
|
cuda_wrappers=[
|
|
("forward_with_q", f"FusedKNormRopeFlashMLAKernel<{args}>::forward_with_q"),
|
|
],
|
|
)
|
|
|
|
|
|
@cache_once
|
|
def _jit_main_q_indexer_rope_hadamard_quant_module(dtype: torch.dtype):
|
|
"""C4 indexer Q kernel: RoPE + 128-pt Hadamard + fp8 act-quant"""
|
|
args = make_cpp_args(dtype, is_arch_support_pdl())
|
|
return load_jit(
|
|
make_name("main_q_indexer_rope_hadamard_quant"),
|
|
*args,
|
|
cuda_files=["deepseek_v4/main_norm_rope.cuh"],
|
|
cuda_wrappers=[
|
|
("forward", f"FusedQIndexerRopeHadamardQuantKernel<{args}>::forward"),
|
|
],
|
|
)
|
|
|
|
|
|
# V3.2 lays q out as [rope | nope] (V4 is [nope | rope]) -> kRopeFirst=true, and
|
|
# drops the Hadamard rotation (kHadamard=false).
|
|
@cache_once
|
|
def _jit_main_q_indexer_rope_first_quant_module(dtype: torch.dtype):
|
|
args = make_cpp_args(dtype, is_arch_support_pdl(), True, False)
|
|
return load_jit(
|
|
make_name("main_q_indexer_rope_first_quant"),
|
|
*args,
|
|
cuda_files=["deepseek_v4/main_norm_rope.cuh"],
|
|
cuda_wrappers=[
|
|
("forward", f"FusedQIndexerRopeHadamardQuantKernel<{args}>::forward"),
|
|
],
|
|
)
|
|
|
|
|
|
@cache_once
|
|
def _jit_main_q_indexer_rope_hadamard_fp4_quant_module(dtype: torch.dtype):
|
|
args = make_cpp_args(dtype, is_arch_support_pdl())
|
|
return load_jit(
|
|
make_name("main_q_indexer_rope_hadamard_fp4_quant"),
|
|
*args,
|
|
cuda_files=["deepseek_v4/main_norm_rope.cuh"],
|
|
cuda_wrappers=[
|
|
("forward", f"FusedQIndexerRopeHadamardFp4QuantKernel<{args}>::forward"),
|
|
],
|
|
)
|
|
|
|
|
|
def fused_rope_inplace(
|
|
q: torch.Tensor,
|
|
k: Optional[torch.Tensor],
|
|
freqs_cis: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
inverse: bool = False,
|
|
) -> None:
|
|
"""Apply rotary embeddings to both Q and K in a single fused CUDA kernel.
|
|
|
|
Args:
|
|
q: [batch_size, num_q_heads, rope_dim] bfloat16
|
|
k: [batch_size, num_k_heads, rope_dim] bfloat16 or None
|
|
freqs_cis: [max_seq_len, rope_dim // 2] complex64 (full table)
|
|
positions: [batch_size] int32 or int64, indices into freqs_cis
|
|
inverse: if True, apply inverse rotation (conjugate freqs)
|
|
"""
|
|
if _is_hip or _is_xpu:
|
|
from sglang.kernels.ops.attention.deepseek_v4_rope import (
|
|
apply_rotary_emb_triton,
|
|
)
|
|
|
|
apply_rotary_emb_triton(q, freqs_cis, positions=positions, inverse=inverse)
|
|
if k is not None:
|
|
apply_rotary_emb_triton(k, freqs_cis, positions=positions, inverse=inverse)
|
|
return
|
|
|
|
freqs_real = torch.view_as_real(freqs_cis).flatten(-2).contiguous()
|
|
module = _jit_fused_rope_module()
|
|
module.forward(q, k, freqs_real, positions, inverse)
|
|
|
|
|
|
def fused_q_norm_rope(
|
|
q_input: torch.Tensor,
|
|
q_output: torch.Tensor,
|
|
eps: Optional[float],
|
|
freqs_cis: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
) -> None:
|
|
freqs_real = torch.view_as_real(freqs_cis).flatten(-2)
|
|
head_dim = q_input.shape[-1]
|
|
rope_dim = freqs_real.shape[-1]
|
|
if _is_xpu:
|
|
assert eps is not None
|
|
fused_q_norm_rope_xpu(q_input, q_output, freqs_real, positions, eps)
|
|
else:
|
|
module = _jit_main_q_norm_rope_module(
|
|
q_input.dtype, head_dim, rope_dim, eps is not None
|
|
)
|
|
module.forward(q_input, q_output, freqs_real, positions, eps or 0.0)
|
|
|
|
|
|
def fused_q_indexer_rope_hadamard_quant(
|
|
q_input: torch.Tensor,
|
|
weight: torch.Tensor,
|
|
weight_scale: float,
|
|
freqs_cis: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
freqs_real = torch.view_as_real(freqs_cis).flatten(-2)
|
|
q_fp8 = torch.empty(q_input.shape, dtype=torch.float8_e4m3fn, device=q_input.device)
|
|
weights_out = torch.empty(
|
|
(*q_input.shape[:-1], 1), dtype=torch.float32, device=q_input.device
|
|
)
|
|
if _is_hip:
|
|
torch.ops.sgl_kernel.dsv4_fused_q_indexer_rope_hadamard_quant(
|
|
q_input,
|
|
q_fp8,
|
|
weight,
|
|
weights_out,
|
|
float(weight_scale),
|
|
freqs_real,
|
|
positions,
|
|
)
|
|
elif _is_xpu:
|
|
from sgl_kernel import fused_q_indexer_rope_hadamard_quant
|
|
|
|
fused_q_indexer_rope_hadamard_quant(
|
|
q_input,
|
|
q_fp8,
|
|
weight,
|
|
weights_out,
|
|
float(weight_scale),
|
|
freqs_real,
|
|
positions,
|
|
)
|
|
else:
|
|
module = _jit_main_q_indexer_rope_hadamard_quant_module(q_input.dtype)
|
|
module.forward(
|
|
q_input,
|
|
q_fp8,
|
|
weight,
|
|
weights_out,
|
|
float(weight_scale),
|
|
freqs_real,
|
|
positions,
|
|
)
|
|
return q_fp8, weights_out
|
|
|
|
|
|
def fused_q_indexer_rope_first_quant(
|
|
q_input: torch.Tensor,
|
|
weight: torch.Tensor,
|
|
weight_scale: float,
|
|
cos_sin_cache: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
"""DeepSeek-V3.2 only. Indexer Q: RoPE on the leading dims + fp8 act-quant. CUDA only."""
|
|
q_fp8 = torch.empty(q_input.shape, dtype=torch.float8_e4m3fn, device=q_input.device)
|
|
weights_out = torch.empty(
|
|
(*q_input.shape[:-1], 1), dtype=torch.float32, device=q_input.device
|
|
)
|
|
module = _jit_main_q_indexer_rope_first_quant_module(q_input.dtype)
|
|
module.forward(
|
|
q_input,
|
|
q_fp8,
|
|
weight,
|
|
weights_out,
|
|
float(weight_scale),
|
|
cos_sin_cache,
|
|
positions,
|
|
)
|
|
return q_fp8, weights_out
|
|
|
|
|
|
def fused_q_indexer_rope_hadamard_fp4_quant(
|
|
q_input: torch.Tensor,
|
|
weight: torch.Tensor,
|
|
weight_scale: float,
|
|
freqs_cis: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
) -> Tuple[Tuple[torch.Tensor, torch.Tensor], torch.Tensor]:
|
|
if _is_hip:
|
|
raise RuntimeError("DeepSeek V4 FP4 indexer requires the CUDA fused Q path.")
|
|
freqs_real = torch.view_as_real(freqs_cis).flatten(-2)
|
|
q_fp4 = torch.empty(
|
|
(*q_input.shape[:-1], q_input.shape[-1] // 2),
|
|
dtype=torch.int8,
|
|
device=q_input.device,
|
|
)
|
|
q_sf = torch.empty(q_input.shape[:-1], dtype=torch.int32, device=q_input.device)
|
|
weights_out = torch.empty(
|
|
(*q_input.shape[:-1], 1), dtype=torch.float32, device=q_input.device
|
|
)
|
|
module = _jit_main_q_indexer_rope_hadamard_fp4_quant_module(q_input.dtype)
|
|
module.forward(
|
|
q_input,
|
|
q_fp4,
|
|
q_sf,
|
|
weight,
|
|
weights_out,
|
|
float(weight_scale),
|
|
freqs_real,
|
|
positions,
|
|
)
|
|
return (q_fp4, q_sf), weights_out
|
|
|
|
|
|
def fused_k_norm_rope_flashmla(
|
|
kv: torch.Tensor,
|
|
kv_weight: torch.Tensor,
|
|
eps: float,
|
|
freqs_cis: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
out_loc: torch.Tensor,
|
|
kvcache: torch.Tensor,
|
|
page_size: int,
|
|
layout: Union[KVLayout, str] = KVLayout.V4,
|
|
q: Optional[torch.Tensor] = None,
|
|
) -> None:
|
|
"""RMSNorm + RoPE ``kv`` and write it into the ``layout`` paged FlashMLA
|
|
cache at ``out_loc``."""
|
|
layout = KVLayout.parse(layout)
|
|
freqs_real = torch.view_as_real(freqs_cis).flatten(-2)
|
|
head_dim = kv.shape[-1]
|
|
rope_dim = freqs_real.shape[-1]
|
|
if _is_xpu:
|
|
assert layout is KVLayout.V4, "the V4.1 KV layouts are CUDA (sm100) only"
|
|
assert q is None, "the XPU K kernel does not rope q"
|
|
fused_k_norm_rope_flashmla_xpu(
|
|
kv, kv_weight, freqs_real, positions, out_loc, kvcache, eps, page_size
|
|
)
|
|
elif q is not None:
|
|
assert _is_hip, "only the ROCm K launch ropes q"
|
|
module = _jit_main_k_norm_rope_q_flashmla_module(
|
|
kv.dtype, head_dim, rope_dim, page_size, layout
|
|
)
|
|
module.forward_with_q(
|
|
kv, kv_weight, freqs_real, positions, out_loc, kvcache, eps, q
|
|
)
|
|
else:
|
|
module = _jit_main_k_norm_rope_flashmla_module(
|
|
kv.dtype, head_dim, rope_dim, page_size, layout
|
|
)
|
|
module.forward(kv, kv_weight, freqs_real, positions, out_loc, kvcache, eps)
|