[NPU] Enable piecewise CUDA graph support on NPU (#28417)

This commit is contained in:
hanwlax
2026-09-23 17:30:30 +08:00
committed by GitHub
parent 58f622ebc4
commit 2d25767759
12 changed files with 414 additions and 35 deletions
@@ -8,6 +8,7 @@
from contextlib import nullcontext
from functools import lru_cache
from typing import Optional
import torch
import torch.nn.functional as F
@@ -30,6 +31,7 @@ from sglang.srt.utils import (
is_npu,
next_power_of_2,
)
from sglang.srt.utils.custom_op import register_custom_op
_is_npu = is_npu()
_use_cpu = is_cpu() and cpu_has_amx_support()
@@ -323,7 +325,70 @@ def _layer_norm_fwd(
if _is_npu:
from sgl_kernel_npu.fla.layernorm_gated import layer_norm_fwd_npu as _layer_norm_fwd
from sgl_kernel_npu.fla.layernorm_gated import (
layer_norm_fwd_npu as _layer_norm_fwd_npu_raw,
)
@register_custom_op(out_shape="x", mutates_args=["out"])
def _layer_norm_fwd_custom(
x: torch.Tensor,
weight: torch.Tensor,
bias: Optional[torch.Tensor],
eps: float,
z: Optional[torch.Tensor],
out: Optional[torch.Tensor],
group_size: Optional[int],
norm_before_gate: bool,
is_rms_norm: bool,
activation: Optional[str],
) -> torch.Tensor:
_out, _mean, _rstd = _layer_norm_fwd_npu_raw(
x=x,
weight=weight,
bias=bias,
eps=eps,
z=z,
out=out,
group_size=group_size,
norm_before_gate=norm_before_gate,
is_rms_norm=is_rms_norm,
activation=activation,
)
return _out
def _layer_norm_fwd(
x,
weight,
bias,
eps,
z=None,
out=None,
group_size=None,
norm_before_gate=True,
is_rms_norm=False,
activation=None,
):
if group_size is None:
group_size = x.shape[-1]
if out is not None:
assert out.shape == x.shape
else:
out = torch.empty_like(x)
y = _layer_norm_fwd_custom(
x=x,
weight=weight,
bias=bias,
eps=eps,
z=z,
out=out,
group_size=group_size,
norm_before_gate=norm_before_gate,
is_rms_norm=is_rms_norm,
activation=activation,
)
# mean/rstd are unused by all callers on the NPU path (rms_norm_gated
# discards them); return None to keep the (y, mean, rstd) unpack contract.
return y, None, None
def rms_norm_gated(
@@ -3,9 +3,10 @@ import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.utils import is_hip
from sglang.srt.utils import is_hip, is_npu
_is_hip = is_hip()
_is_npu = is_npu()
rmsnorm_autotune = triton.autotune(
@@ -443,6 +444,15 @@ def fused_sigmoid_mul(
gate_stride_row = hidden_dim
gate_stride_head = hidden_dim
if _is_npu and torch.compiler.is_compiling():
gate_reshaped = (
gate.view_as(attn_output) if gate.numel() == attn_output.numel() else gate
)
if inplace:
attn_output.mul_(torch.sigmoid(gate_reshaped))
return attn_output
return attn_output * torch.sigmoid(gate_reshaped)
out = attn_output if inplace else torch.empty_like(attn_output)
block_h = 1024 if num_tokens < 1024 else 2048
grid = (num_tokens, triton.cdiv(hidden_dim, block_h))
@@ -14,6 +14,7 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
)
from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import (
ALLOWED_BACKENDS_PER_PHASE,
Backend,
@@ -193,7 +194,10 @@ def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any):
"non-CUDA hardware (HIP/NPU/CPU/MPS/XPU)",
lambda: (
get_platform().is_hip
or get_platform().is_npu
or (
get_platform().is_npu
and not envs.SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH.get()
)
or is_cpu()
or is_mps()
or get_platform().is_xpu
@@ -7,6 +7,10 @@ import torch.fx as fx
from sglang.srt.compilation.compilation_config import CompilationConfig
from sglang.srt.compilation.compilation_counter import compilation_counter
from sglang.srt.compilation.compile_phase import (
get_pcg_capture_stream,
is_in_torch_compile_warmup,
)
from sglang.srt.compilation.cuda_piecewise_backend import (
CUDAPiecewiseBackend,
weak_ref_tensors,
@@ -39,6 +43,12 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend):
)
def __call__(self, *args):
if is_in_torch_compile_warmup():
return self.compiled_graph_for_general_shape(*args)
if len(self.sym_shape_indices) == 0:
return self.compiled_graph_for_general_shape(*args)
runtime_shape = args[self.sym_shape_indices[0]]
if runtime_shape not in self.concrete_size_entries:
# we don't need to do anything for this shape
@@ -73,7 +83,12 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend):
stack.enter_context(patch("torch.npu.empty_cache", lambda: None))
# mind-exploding: carefully manage the reference and memory.
with torch.npu.graph(npugraph, pool=self.graph_pool):
with torch.npu.graph(
npugraph,
pool=self.graph_pool,
stream=get_pcg_capture_stream(),
auto_dispatch_capture=True,
):
# `output` is managed by pytorch's cudagraph pool
output = entry.runnable(*args)
if self.is_last_graph:
+1
View File
@@ -976,6 +976,7 @@ class Envs:
DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False)
SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0)
SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("")
SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH = EnvBool(False) # prefill cuda graph on NPU
# ===================================================================
# MUSA
+48 -21
View File
@@ -1,5 +1,9 @@
from typing import List
import torch
from sglang.srt.utils.custom_op import register_custom_op
cmo_stream = None
share_stream = None
@@ -19,12 +23,10 @@ def set_cmo_stream(stream):
cmo_stream = stream
def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000):
"""
PREFETCH_MAX_SIZE: maximum size (bytes) for each prefetch operation.
This affects the time spent in prefetch:
time ≈ PREFETCH_MAX_SIZE / system_bandwidth
"""
@register_custom_op()
def _prepare_weight_cache_list(
handle: torch.Tensor, cache: List[torch.Tensor], max_size: int
) -> None:
import torch_npu
stream = get_cmo_stream()
@@ -33,26 +35,44 @@ def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000):
set_cmo_stream(stream)
stream.wait_stream(torch.npu.current_stream())
with torch.npu.stream(stream):
if isinstance(cache, list):
for weight in cache:
torch_npu.npu_prefetch(
weight,
handle,
PREFETCH_MAX_SIZE,
)
else:
torch_npu.npu_prefetch(
cache,
handle,
PREFETCH_MAX_SIZE,
)
for weight in cache:
torch_npu.npu_prefetch(weight, handle, max_size)
def wait_cmo_stream():
@register_custom_op()
def _prepare_weight_cache_single(
handle: torch.Tensor, cache: torch.Tensor, max_size: int
) -> None:
import torch_npu
stream = get_cmo_stream()
if stream is None:
stream = torch.npu.Stream()
set_cmo_stream(stream)
stream.wait_stream(torch.npu.current_stream())
with torch.npu.stream(stream):
torch_npu.npu_prefetch(cache, handle, max_size)
def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000):
"""
PREFETCH_MAX_SIZE: maximum size (bytes) for each prefetch operation.
This affects the time spent in prefetch:
time ≈ PREFETCH_MAX_SIZE / system_bandwidth
"""
if isinstance(cache, list):
_prepare_weight_cache_list(handle, cache, PREFETCH_MAX_SIZE)
else:
_prepare_weight_cache_single(handle, cache, PREFETCH_MAX_SIZE)
@register_custom_op(out_shape=0)
def wait_cmo_stream(dummy: torch.Tensor) -> torch.Tensor:
stream = get_cmo_stream()
if stream is not None:
cur_stream = torch.npu.current_stream()
cur_stream.wait_stream(stream)
return dummy
def get_share_stream():
@@ -65,14 +85,21 @@ def set_share_stream(stream):
share_stream = stream
def wait_share_stream():
@register_custom_op(out_shape=0)
def wait_share_stream(dummy: torch.Tensor) -> torch.Tensor:
stream = get_share_stream()
if stream is not None:
cur_stream = torch.npu.current_stream()
cur_stream.wait_stream(stream)
return dummy
def shared_expert_on_independent_stream(hidden_states, forward_func):
import torch.compiler
if torch.compiler.is_compiling():
return forward_func(hidden_states)
stream = get_share_stream()
if stream is None:
stream = torch.npu.Stream()
@@ -1,7 +1,16 @@
from typing import TYPE_CHECKING, Optional
import torch
from sgl_kernel_npu.norm.l1_norm import l1_norm
from sglang.srt.utils.custom_op import register_custom_op
@register_custom_op(out_shape=0)
def _l1_norm(x: torch.Tensor) -> torch.Tensor:
from sgl_kernel_npu.norm.l1_norm import l1_norm
return l1_norm(x)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical
@@ -59,7 +68,7 @@ def fused_topk_npu(
)
if renormalize:
topk_weights = l1_norm(
topk_weights = _l1_norm(
topk_weights
if topk_config.num_fused_shared_experts == 0
else topk_weights[:, :-1]
+1 -1
View File
@@ -625,7 +625,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
)
trace_sync("post_experts")
if enable_dual_stream:
wait_share_stream()
final_hidden_states = wait_share_stream(final_hidden_states)
elif enable_cuda_shared_overlap:
torch.cuda.current_stream().wait_event(shared_event)
+53 -3
View File
@@ -33,6 +33,7 @@ from sglang.srt.models.utils import apply_qk_norm
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import add_prefix, get_bool_env_var, is_cuda, is_hip, is_npu
from sglang.srt.utils.custom_op import register_custom_op
Qwen3Config = None
@@ -57,6 +58,55 @@ if _is_npu:
from sglang.srt.hardware_backend.npu.cmo import get_cmo_stream, wait_cmo_stream
def _split_qkv_rmsnorm_rope_fake_impl(
qkv: torch.Tensor,
sin: torch.Tensor,
cos: torch.Tensor,
q_size: int,
kv_size: int,
head_dim: int,
eps: float = 1e-6,
q_weight: Optional[torch.Tensor] = None,
k_weight: Optional[torch.Tensor] = None,
q_bias: Optional[torch.Tensor] = None,
k_bias: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias
output_shape = qkv.shape[:-1]
return (
qkv.new_empty((*output_shape, q_size)),
qkv.new_empty((*output_shape, kv_size)),
qkv.new_empty((*output_shape, kv_size)),
)
@register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl)
def _split_qkv_rmsnorm_rope_custom(
qkv: torch.Tensor,
sin: torch.Tensor,
cos: torch.Tensor,
q_size: int,
kv_size: int,
head_dim: int,
eps: float = 1e-6,
q_weight: Optional[torch.Tensor] = None,
k_weight: Optional[torch.Tensor] = None,
q_bias: Optional[torch.Tensor] = None,
k_bias: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return split_qkv_rmsnorm_rope(
qkv,
sin,
cos,
q_size,
kv_size,
head_dim,
eps=eps,
q_weight=q_weight,
k_weight=k_weight,
q_bias=q_bias,
k_bias=k_bias,
)
class Qwen3Attention(nn.Module):
def __init__(
@@ -186,7 +236,7 @@ class Qwen3Attention(nn.Module):
if self.attn.layer_id == self.start_layer:
self.rotary_emb.get_cos_sin_with_position(positions)
q, k, v = split_qkv_rmsnorm_rope(
q, k, v = _split_qkv_rmsnorm_rope_custom(
qkv,
self.rotary_emb.position_sin,
self.rotary_emb.position_cos,
@@ -422,8 +472,8 @@ class Qwen3DecoderLayer(nn.Module):
),
)
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
if _is_npu and get_cmo_stream():
wait_cmo_stream()
if _is_npu and get_cmo_stream() is not None:
hidden_states = wait_cmo_stream(hidden_states)
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
+9 -3
View File
@@ -16,6 +16,7 @@
import logging
import os
from contextlib import nullcontext
from functools import lru_cache
from typing import Iterable, Optional, Set, Tuple, Union
@@ -1936,9 +1937,14 @@ class Qwen3_5ForCausalLM(nn.Module):
# Pass through decoder layers
for layer_idx in range(self.start_layer, self.end_layer):
layer = self.layers[layer_idx]
with get_global_expert_distribution_recorder().with_current_layer(
layer_idx
):
ctx = (
nullcontext()
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
else get_global_expert_distribution_recorder().with_current_layer(
layer_idx
)
)
with ctx:
hidden_states, residual = layer(
positions=positions,
hidden_states=hidden_states,
+51 -1
View File
@@ -73,6 +73,7 @@ from sglang.srt.utils import (
is_non_idle_and_non_empty,
is_npu,
)
from sglang.srt.utils.custom_op import register_custom_op
from sglang.srt.utils.hf_transformers_utils import get_rope_config
_is_cuda = is_cuda()
@@ -103,6 +104,55 @@ _is_npu = is_npu()
if _is_npu:
from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import split_qkv_rmsnorm_rope
def _split_qkv_rmsnorm_rope_fake_impl(
qkv: torch.Tensor,
sin: torch.Tensor,
cos: torch.Tensor,
q_size: int,
kv_size: int,
head_dim: int,
eps: float = 1e-6,
q_weight: Optional[torch.Tensor] = None,
k_weight: Optional[torch.Tensor] = None,
q_bias: Optional[torch.Tensor] = None,
k_bias: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias
output_shape = qkv.shape[:-1]
return (
qkv.new_empty((*output_shape, q_size)),
qkv.new_empty((*output_shape, kv_size)),
qkv.new_empty((*output_shape, kv_size)),
)
@register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl)
def _split_qkv_rmsnorm_rope_custom(
qkv: torch.Tensor,
sin: torch.Tensor,
cos: torch.Tensor,
q_size: int,
kv_size: int,
head_dim: int,
eps: float = 1e-6,
q_weight: Optional[torch.Tensor] = None,
k_weight: Optional[torch.Tensor] = None,
q_bias: Optional[torch.Tensor] = None,
k_bias: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return split_qkv_rmsnorm_rope(
qkv,
sin,
cos,
q_size,
kv_size,
head_dim,
eps=eps,
q_weight=q_weight,
k_weight=k_weight,
q_bias=q_bias,
k_bias=k_bias,
)
def compute_yarn_parameters(
config: PretrainedConfig,
@@ -622,7 +672,7 @@ class Qwen3MoeAttention(nn.Module):
qkv, _ = self.qkv_proj(hidden_states)
if self.attn.layer_id == self.start_layer:
self.rotary_emb.get_cos_sin_with_position(positions)
q, k, v = split_qkv_rmsnorm_rope(
q, k, v = _split_qkv_rmsnorm_rope_custom(
qkv,
self.rotary_emb.position_sin,
self.rotary_emb.position_cos,