[NPU] Enable piecewise CUDA graph support on NPU (#28417)
This commit is contained in:
@@ -8,6 +8,7 @@
|
||||
|
||||
from contextlib import nullcontext
|
||||
from functools import lru_cache
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -30,6 +31,7 @@ from sglang.srt.utils import (
|
||||
is_npu,
|
||||
next_power_of_2,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
_is_npu = is_npu()
|
||||
_use_cpu = is_cpu() and cpu_has_amx_support()
|
||||
@@ -323,7 +325,70 @@ def _layer_norm_fwd(
|
||||
|
||||
|
||||
if _is_npu:
|
||||
from sgl_kernel_npu.fla.layernorm_gated import layer_norm_fwd_npu as _layer_norm_fwd
|
||||
from sgl_kernel_npu.fla.layernorm_gated import (
|
||||
layer_norm_fwd_npu as _layer_norm_fwd_npu_raw,
|
||||
)
|
||||
|
||||
@register_custom_op(out_shape="x", mutates_args=["out"])
|
||||
def _layer_norm_fwd_custom(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: Optional[torch.Tensor],
|
||||
eps: float,
|
||||
z: Optional[torch.Tensor],
|
||||
out: Optional[torch.Tensor],
|
||||
group_size: Optional[int],
|
||||
norm_before_gate: bool,
|
||||
is_rms_norm: bool,
|
||||
activation: Optional[str],
|
||||
) -> torch.Tensor:
|
||||
_out, _mean, _rstd = _layer_norm_fwd_npu_raw(
|
||||
x=x,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
eps=eps,
|
||||
z=z,
|
||||
out=out,
|
||||
group_size=group_size,
|
||||
norm_before_gate=norm_before_gate,
|
||||
is_rms_norm=is_rms_norm,
|
||||
activation=activation,
|
||||
)
|
||||
return _out
|
||||
|
||||
def _layer_norm_fwd(
|
||||
x,
|
||||
weight,
|
||||
bias,
|
||||
eps,
|
||||
z=None,
|
||||
out=None,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
is_rms_norm=False,
|
||||
activation=None,
|
||||
):
|
||||
if group_size is None:
|
||||
group_size = x.shape[-1]
|
||||
if out is not None:
|
||||
assert out.shape == x.shape
|
||||
else:
|
||||
out = torch.empty_like(x)
|
||||
y = _layer_norm_fwd_custom(
|
||||
x=x,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
eps=eps,
|
||||
z=z,
|
||||
out=out,
|
||||
group_size=group_size,
|
||||
norm_before_gate=norm_before_gate,
|
||||
is_rms_norm=is_rms_norm,
|
||||
activation=activation,
|
||||
)
|
||||
# mean/rstd are unused by all callers on the NPU path (rms_norm_gated
|
||||
# discards them); return None to keep the (y, mean, rstd) unpack contract.
|
||||
return y, None, None
|
||||
|
||||
|
||||
def rms_norm_gated(
|
||||
|
||||
@@ -3,9 +3,10 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.srt.utils import is_hip, is_npu
|
||||
|
||||
_is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
|
||||
|
||||
rmsnorm_autotune = triton.autotune(
|
||||
@@ -443,6 +444,15 @@ def fused_sigmoid_mul(
|
||||
gate_stride_row = hidden_dim
|
||||
gate_stride_head = hidden_dim
|
||||
|
||||
if _is_npu and torch.compiler.is_compiling():
|
||||
gate_reshaped = (
|
||||
gate.view_as(attn_output) if gate.numel() == attn_output.numel() else gate
|
||||
)
|
||||
if inplace:
|
||||
attn_output.mul_(torch.sigmoid(gate_reshaped))
|
||||
return attn_output
|
||||
return attn_output * torch.sigmoid(gate_reshaped)
|
||||
|
||||
out = attn_output if inplace else torch.empty_like(attn_output)
|
||||
block_h = 1024 if num_tokens < 1024 else 2048
|
||||
grid = (num_tokens, triton.cdiv(hidden_dim, block_h))
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
ALLOWED_BACKENDS_PER_PHASE,
|
||||
Backend,
|
||||
@@ -193,7 +194,10 @@ def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any):
|
||||
"non-CUDA hardware (HIP/NPU/CPU/MPS/XPU)",
|
||||
lambda: (
|
||||
get_platform().is_hip
|
||||
or get_platform().is_npu
|
||||
or (
|
||||
get_platform().is_npu
|
||||
and not envs.SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH.get()
|
||||
)
|
||||
or is_cpu()
|
||||
or is_mps()
|
||||
or get_platform().is_xpu
|
||||
|
||||
@@ -7,6 +7,10 @@ import torch.fx as fx
|
||||
|
||||
from sglang.srt.compilation.compilation_config import CompilationConfig
|
||||
from sglang.srt.compilation.compilation_counter import compilation_counter
|
||||
from sglang.srt.compilation.compile_phase import (
|
||||
get_pcg_capture_stream,
|
||||
is_in_torch_compile_warmup,
|
||||
)
|
||||
from sglang.srt.compilation.cuda_piecewise_backend import (
|
||||
CUDAPiecewiseBackend,
|
||||
weak_ref_tensors,
|
||||
@@ -39,6 +43,12 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend):
|
||||
)
|
||||
|
||||
def __call__(self, *args):
|
||||
if is_in_torch_compile_warmup():
|
||||
return self.compiled_graph_for_general_shape(*args)
|
||||
|
||||
if len(self.sym_shape_indices) == 0:
|
||||
return self.compiled_graph_for_general_shape(*args)
|
||||
|
||||
runtime_shape = args[self.sym_shape_indices[0]]
|
||||
if runtime_shape not in self.concrete_size_entries:
|
||||
# we don't need to do anything for this shape
|
||||
@@ -73,7 +83,12 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend):
|
||||
stack.enter_context(patch("torch.npu.empty_cache", lambda: None))
|
||||
|
||||
# mind-exploding: carefully manage the reference and memory.
|
||||
with torch.npu.graph(npugraph, pool=self.graph_pool):
|
||||
with torch.npu.graph(
|
||||
npugraph,
|
||||
pool=self.graph_pool,
|
||||
stream=get_pcg_capture_stream(),
|
||||
auto_dispatch_capture=True,
|
||||
):
|
||||
# `output` is managed by pytorch's cudagraph pool
|
||||
output = entry.runnable(*args)
|
||||
if self.is_last_graph:
|
||||
|
||||
@@ -976,6 +976,7 @@ class Envs:
|
||||
DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False)
|
||||
SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0)
|
||||
SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("")
|
||||
SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH = EnvBool(False) # prefill cuda graph on NPU
|
||||
|
||||
# ===================================================================
|
||||
# MUSA
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
cmo_stream = None
|
||||
share_stream = None
|
||||
|
||||
@@ -19,12 +23,10 @@ def set_cmo_stream(stream):
|
||||
cmo_stream = stream
|
||||
|
||||
|
||||
def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000):
|
||||
"""
|
||||
PREFETCH_MAX_SIZE: maximum size (bytes) for each prefetch operation.
|
||||
This affects the time spent in prefetch:
|
||||
time ≈ PREFETCH_MAX_SIZE / system_bandwidth
|
||||
"""
|
||||
@register_custom_op()
|
||||
def _prepare_weight_cache_list(
|
||||
handle: torch.Tensor, cache: List[torch.Tensor], max_size: int
|
||||
) -> None:
|
||||
import torch_npu
|
||||
|
||||
stream = get_cmo_stream()
|
||||
@@ -33,26 +35,44 @@ def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000):
|
||||
set_cmo_stream(stream)
|
||||
stream.wait_stream(torch.npu.current_stream())
|
||||
with torch.npu.stream(stream):
|
||||
if isinstance(cache, list):
|
||||
for weight in cache:
|
||||
torch_npu.npu_prefetch(
|
||||
weight,
|
||||
handle,
|
||||
PREFETCH_MAX_SIZE,
|
||||
)
|
||||
else:
|
||||
torch_npu.npu_prefetch(
|
||||
cache,
|
||||
handle,
|
||||
PREFETCH_MAX_SIZE,
|
||||
)
|
||||
for weight in cache:
|
||||
torch_npu.npu_prefetch(weight, handle, max_size)
|
||||
|
||||
|
||||
def wait_cmo_stream():
|
||||
@register_custom_op()
|
||||
def _prepare_weight_cache_single(
|
||||
handle: torch.Tensor, cache: torch.Tensor, max_size: int
|
||||
) -> None:
|
||||
import torch_npu
|
||||
|
||||
stream = get_cmo_stream()
|
||||
if stream is None:
|
||||
stream = torch.npu.Stream()
|
||||
set_cmo_stream(stream)
|
||||
stream.wait_stream(torch.npu.current_stream())
|
||||
with torch.npu.stream(stream):
|
||||
torch_npu.npu_prefetch(cache, handle, max_size)
|
||||
|
||||
|
||||
def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000):
|
||||
"""
|
||||
PREFETCH_MAX_SIZE: maximum size (bytes) for each prefetch operation.
|
||||
This affects the time spent in prefetch:
|
||||
time ≈ PREFETCH_MAX_SIZE / system_bandwidth
|
||||
"""
|
||||
if isinstance(cache, list):
|
||||
_prepare_weight_cache_list(handle, cache, PREFETCH_MAX_SIZE)
|
||||
else:
|
||||
_prepare_weight_cache_single(handle, cache, PREFETCH_MAX_SIZE)
|
||||
|
||||
|
||||
@register_custom_op(out_shape=0)
|
||||
def wait_cmo_stream(dummy: torch.Tensor) -> torch.Tensor:
|
||||
stream = get_cmo_stream()
|
||||
if stream is not None:
|
||||
cur_stream = torch.npu.current_stream()
|
||||
cur_stream.wait_stream(stream)
|
||||
return dummy
|
||||
|
||||
|
||||
def get_share_stream():
|
||||
@@ -65,14 +85,21 @@ def set_share_stream(stream):
|
||||
share_stream = stream
|
||||
|
||||
|
||||
def wait_share_stream():
|
||||
@register_custom_op(out_shape=0)
|
||||
def wait_share_stream(dummy: torch.Tensor) -> torch.Tensor:
|
||||
stream = get_share_stream()
|
||||
if stream is not None:
|
||||
cur_stream = torch.npu.current_stream()
|
||||
cur_stream.wait_stream(stream)
|
||||
return dummy
|
||||
|
||||
|
||||
def shared_expert_on_independent_stream(hidden_states, forward_func):
|
||||
import torch.compiler
|
||||
|
||||
if torch.compiler.is_compiling():
|
||||
return forward_func(hidden_states)
|
||||
|
||||
stream = get_share_stream()
|
||||
if stream is None:
|
||||
stream = torch.npu.Stream()
|
||||
|
||||
@@ -1,7 +1,16 @@
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
from sgl_kernel_npu.norm.l1_norm import l1_norm
|
||||
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
@register_custom_op(out_shape=0)
|
||||
def _l1_norm(x: torch.Tensor) -> torch.Tensor:
|
||||
from sgl_kernel_npu.norm.l1_norm import l1_norm
|
||||
|
||||
return l1_norm(x)
|
||||
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical
|
||||
@@ -59,7 +68,7 @@ def fused_topk_npu(
|
||||
)
|
||||
|
||||
if renormalize:
|
||||
topk_weights = l1_norm(
|
||||
topk_weights = _l1_norm(
|
||||
topk_weights
|
||||
if topk_config.num_fused_shared_experts == 0
|
||||
else topk_weights[:, :-1]
|
||||
|
||||
@@ -625,7 +625,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
)
|
||||
trace_sync("post_experts")
|
||||
if enable_dual_stream:
|
||||
wait_share_stream()
|
||||
final_hidden_states = wait_share_stream(final_hidden_states)
|
||||
elif enable_cuda_shared_overlap:
|
||||
torch.cuda.current_stream().wait_event(shared_event)
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.srt.models.utils import apply_qk_norm
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import add_prefix, get_bool_env_var, is_cuda, is_hip, is_npu
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
Qwen3Config = None
|
||||
|
||||
@@ -57,6 +58,55 @@ if _is_npu:
|
||||
|
||||
from sglang.srt.hardware_backend.npu.cmo import get_cmo_stream, wait_cmo_stream
|
||||
|
||||
def _split_qkv_rmsnorm_rope_fake_impl(
|
||||
qkv: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
head_dim: int,
|
||||
eps: float = 1e-6,
|
||||
q_weight: Optional[torch.Tensor] = None,
|
||||
k_weight: Optional[torch.Tensor] = None,
|
||||
q_bias: Optional[torch.Tensor] = None,
|
||||
k_bias: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias
|
||||
output_shape = qkv.shape[:-1]
|
||||
return (
|
||||
qkv.new_empty((*output_shape, q_size)),
|
||||
qkv.new_empty((*output_shape, kv_size)),
|
||||
qkv.new_empty((*output_shape, kv_size)),
|
||||
)
|
||||
|
||||
@register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl)
|
||||
def _split_qkv_rmsnorm_rope_custom(
|
||||
qkv: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
head_dim: int,
|
||||
eps: float = 1e-6,
|
||||
q_weight: Optional[torch.Tensor] = None,
|
||||
k_weight: Optional[torch.Tensor] = None,
|
||||
q_bias: Optional[torch.Tensor] = None,
|
||||
k_bias: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return split_qkv_rmsnorm_rope(
|
||||
qkv,
|
||||
sin,
|
||||
cos,
|
||||
q_size,
|
||||
kv_size,
|
||||
head_dim,
|
||||
eps=eps,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
q_bias=q_bias,
|
||||
k_bias=k_bias,
|
||||
)
|
||||
|
||||
|
||||
class Qwen3Attention(nn.Module):
|
||||
def __init__(
|
||||
@@ -186,7 +236,7 @@ class Qwen3Attention(nn.Module):
|
||||
|
||||
if self.attn.layer_id == self.start_layer:
|
||||
self.rotary_emb.get_cos_sin_with_position(positions)
|
||||
q, k, v = split_qkv_rmsnorm_rope(
|
||||
q, k, v = _split_qkv_rmsnorm_rope_custom(
|
||||
qkv,
|
||||
self.rotary_emb.position_sin,
|
||||
self.rotary_emb.position_cos,
|
||||
@@ -422,8 +472,8 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
),
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
|
||||
if _is_npu and get_cmo_stream():
|
||||
wait_cmo_stream()
|
||||
if _is_npu and get_cmo_stream() is not None:
|
||||
hidden_states = wait_cmo_stream(hidden_states)
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
import logging
|
||||
import os
|
||||
from contextlib import nullcontext
|
||||
from functools import lru_cache
|
||||
from typing import Iterable, Optional, Set, Tuple, Union
|
||||
|
||||
@@ -1936,9 +1937,14 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
# Pass through decoder layers
|
||||
for layer_idx in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[layer_idx]
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
layer_idx
|
||||
):
|
||||
ctx = (
|
||||
nullcontext()
|
||||
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
||||
else get_global_expert_distribution_recorder().with_current_layer(
|
||||
layer_idx
|
||||
)
|
||||
)
|
||||
with ctx:
|
||||
hidden_states, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -73,6 +73,7 @@ from sglang.srt.utils import (
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
@@ -103,6 +104,55 @@ _is_npu = is_npu()
|
||||
if _is_npu:
|
||||
from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import split_qkv_rmsnorm_rope
|
||||
|
||||
def _split_qkv_rmsnorm_rope_fake_impl(
|
||||
qkv: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
head_dim: int,
|
||||
eps: float = 1e-6,
|
||||
q_weight: Optional[torch.Tensor] = None,
|
||||
k_weight: Optional[torch.Tensor] = None,
|
||||
q_bias: Optional[torch.Tensor] = None,
|
||||
k_bias: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias
|
||||
output_shape = qkv.shape[:-1]
|
||||
return (
|
||||
qkv.new_empty((*output_shape, q_size)),
|
||||
qkv.new_empty((*output_shape, kv_size)),
|
||||
qkv.new_empty((*output_shape, kv_size)),
|
||||
)
|
||||
|
||||
@register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl)
|
||||
def _split_qkv_rmsnorm_rope_custom(
|
||||
qkv: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
head_dim: int,
|
||||
eps: float = 1e-6,
|
||||
q_weight: Optional[torch.Tensor] = None,
|
||||
k_weight: Optional[torch.Tensor] = None,
|
||||
q_bias: Optional[torch.Tensor] = None,
|
||||
k_bias: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return split_qkv_rmsnorm_rope(
|
||||
qkv,
|
||||
sin,
|
||||
cos,
|
||||
q_size,
|
||||
kv_size,
|
||||
head_dim,
|
||||
eps=eps,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
q_bias=q_bias,
|
||||
k_bias=k_bias,
|
||||
)
|
||||
|
||||
|
||||
def compute_yarn_parameters(
|
||||
config: PretrainedConfig,
|
||||
@@ -622,7 +672,7 @@ class Qwen3MoeAttention(nn.Module):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
if self.attn.layer_id == self.start_layer:
|
||||
self.rotary_emb.get_cos_sin_with_position(positions)
|
||||
q, k, v = split_qkv_rmsnorm_rope(
|
||||
q, k, v = _split_qkv_rmsnorm_rope_custom(
|
||||
qkv,
|
||||
self.rotary_emb.position_sin,
|
||||
self.rotary_emb.position_cos,
|
||||
|
||||
Reference in New Issue
Block a user