Revert " [NPU] Enable piecewise CUDA graph support on NPU" (#40895)

This commit is contained in:
Even Zhou
2026-09-23 23:38:14 +08:00
committed by GitHub
parent c3e985203a
commit 5c154c214d
12 changed files with 37 additions and 416 deletions
@@ -8,7 +8,6 @@
from contextlib import nullcontext
from functools import lru_cache
from typing import Optional
import torch
import torch.nn.functional as F
@@ -31,7 +30,6 @@ from sglang.srt.utils import (
is_npu,
next_power_of_2,
)
from sglang.srt.utils.custom_op import register_custom_op
_is_npu = is_npu()
_use_cpu = is_cpu() and cpu_has_amx_support()
@@ -325,70 +323,7 @@ def _layer_norm_fwd(
if _is_npu:
from sgl_kernel_npu.fla.layernorm_gated import (
layer_norm_fwd_npu as _layer_norm_fwd_npu_raw,
)
@register_custom_op(out_shape="x", mutates_args=["out"])
def _layer_norm_fwd_custom(
x: torch.Tensor,
weight: torch.Tensor,
bias: Optional[torch.Tensor],
eps: float,
z: Optional[torch.Tensor],
out: Optional[torch.Tensor],
group_size: Optional[int],
norm_before_gate: bool,
is_rms_norm: bool,
activation: Optional[str],
) -> torch.Tensor:
_out, _mean, _rstd = _layer_norm_fwd_npu_raw(
x=x,
weight=weight,
bias=bias,
eps=eps,
z=z,
out=out,
group_size=group_size,
norm_before_gate=norm_before_gate,
is_rms_norm=is_rms_norm,
activation=activation,
)
return _out
def _layer_norm_fwd(
x,
weight,
bias,
eps,
z=None,
out=None,
group_size=None,
norm_before_gate=True,
is_rms_norm=False,
activation=None,
):
if group_size is None:
group_size = x.shape[-1]
if out is not None:
assert out.shape == x.shape
else:
out = torch.empty_like(x)
y = _layer_norm_fwd_custom(
x=x,
weight=weight,
bias=bias,
eps=eps,
z=z,
out=out,
group_size=group_size,
norm_before_gate=norm_before_gate,
is_rms_norm=is_rms_norm,
activation=activation,
)
# mean/rstd are unused by all callers on the NPU path (rms_norm_gated
# discards them); return None to keep the (y, mean, rstd) unpack contract.
return y, None, None
from sgl_kernel_npu.fla.layernorm_gated import layer_norm_fwd_npu as _layer_norm_fwd
def rms_norm_gated(
@@ -3,10 +3,9 @@ import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.utils import is_hip, is_npu
from sglang.srt.utils import is_hip
_is_hip = is_hip()
_is_npu = is_npu()
rmsnorm_autotune = triton.autotune(
@@ -444,15 +443,6 @@ def fused_sigmoid_mul(
gate_stride_row = hidden_dim
gate_stride_head = hidden_dim
if _is_npu and torch.compiler.is_compiling():
gate_reshaped = (
gate.view_as(attn_output) if gate.numel() == attn_output.numel() else gate
)
if inplace:
attn_output.mul_(torch.sigmoid(gate_reshaped))
return attn_output
return attn_output * torch.sigmoid(gate_reshaped)
out = attn_output if inplace else torch.empty_like(attn_output)
block_h = 1024 if num_tokens < 1024 else 2048
grid = (num_tokens, triton.cdiv(hidden_dim, block_h))
@@ -14,7 +14,6 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
)
from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import (
ALLOWED_BACKENDS_PER_PHASE,
Backend,
@@ -194,10 +193,7 @@ def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any):
"non-CUDA hardware (HIP/NPU/CPU/MPS/XPU)",
lambda: (
get_platform().is_hip
or (
get_platform().is_npu
and not envs.SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH.get()
)
or get_platform().is_npu
or is_cpu()
or is_mps()
or get_platform().is_xpu
@@ -7,10 +7,6 @@ import torch.fx as fx
from sglang.srt.compilation.compilation_config import CompilationConfig
from sglang.srt.compilation.compilation_counter import compilation_counter
from sglang.srt.compilation.compile_phase import (
get_pcg_capture_stream,
is_in_torch_compile_warmup,
)
from sglang.srt.compilation.cuda_piecewise_backend import (
CUDAPiecewiseBackend,
weak_ref_tensors,
@@ -43,12 +39,6 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend):
)
def __call__(self, *args):
if is_in_torch_compile_warmup():
return self.compiled_graph_for_general_shape(*args)
if len(self.sym_shape_indices) == 0:
return self.compiled_graph_for_general_shape(*args)
runtime_shape = args[self.sym_shape_indices[0]]
if runtime_shape not in self.concrete_size_entries:
# we don't need to do anything for this shape
@@ -83,12 +73,7 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend):
stack.enter_context(patch("torch.npu.empty_cache", lambda: None))
# mind-exploding: carefully manage the reference and memory.
with torch.npu.graph(
npugraph,
pool=self.graph_pool,
stream=get_pcg_capture_stream(),
auto_dispatch_capture=True,
):
with torch.npu.graph(npugraph, pool=self.graph_pool):
# `output` is managed by pytorch's cudagraph pool
output = entry.runnable(*args)
if self.is_last_graph:
-1
View File
@@ -980,7 +980,6 @@ class Envs:
DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False)
SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0)
SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("")
SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH = EnvBool(False) # prefill cuda graph on NPU
# ===================================================================
# MUSA
+23 -50
View File
@@ -1,9 +1,5 @@
from typing import List
import torch
from sglang.srt.utils.custom_op import register_custom_op
cmo_stream = None
share_stream = None
@@ -23,56 +19,40 @@ def set_cmo_stream(stream):
cmo_stream = stream
@register_custom_op()
def _prepare_weight_cache_list(
handle: torch.Tensor, cache: List[torch.Tensor], max_size: int
) -> None:
import torch_npu
stream = get_cmo_stream()
if stream is None:
stream = torch.npu.Stream()
set_cmo_stream(stream)
stream.wait_stream(torch.npu.current_stream())
with torch.npu.stream(stream):
for weight in cache:
torch_npu.npu_prefetch(weight, handle, max_size)
@register_custom_op()
def _prepare_weight_cache_single(
handle: torch.Tensor, cache: torch.Tensor, max_size: int
) -> None:
import torch_npu
stream = get_cmo_stream()
if stream is None:
stream = torch.npu.Stream()
set_cmo_stream(stream)
stream.wait_stream(torch.npu.current_stream())
with torch.npu.stream(stream):
torch_npu.npu_prefetch(cache, handle, max_size)
def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000):
"""
PREFETCH_MAX_SIZE: maximum size (bytes) for each prefetch operation.
This affects the time spent in prefetch:
time ≈ PREFETCH_MAX_SIZE / system_bandwidth
"""
if isinstance(cache, list):
_prepare_weight_cache_list(handle, cache, PREFETCH_MAX_SIZE)
else:
_prepare_weight_cache_single(handle, cache, PREFETCH_MAX_SIZE)
import torch_npu
stream = get_cmo_stream()
if stream is None:
stream = torch.npu.Stream()
set_cmo_stream(stream)
stream.wait_stream(torch.npu.current_stream())
with torch.npu.stream(stream):
if isinstance(cache, list):
for weight in cache:
torch_npu.npu_prefetch(
weight,
handle,
PREFETCH_MAX_SIZE,
)
else:
torch_npu.npu_prefetch(
cache,
handle,
PREFETCH_MAX_SIZE,
)
@register_custom_op(out_shape=0)
def wait_cmo_stream(dummy: torch.Tensor) -> torch.Tensor:
def wait_cmo_stream():
stream = get_cmo_stream()
if stream is not None:
cur_stream = torch.npu.current_stream()
cur_stream.wait_stream(stream)
return dummy
def get_share_stream():
@@ -85,21 +65,14 @@ def set_share_stream(stream):
share_stream = stream
@register_custom_op(out_shape=0)
def wait_share_stream(dummy: torch.Tensor) -> torch.Tensor:
def wait_share_stream():
stream = get_share_stream()
if stream is not None:
cur_stream = torch.npu.current_stream()
cur_stream.wait_stream(stream)
return dummy
def shared_expert_on_independent_stream(hidden_states, forward_func):
import torch.compiler
if torch.compiler.is_compiling():
return forward_func(hidden_states)
stream = get_share_stream()
if stream is None:
stream = torch.npu.Stream()
@@ -1,16 +1,7 @@
from typing import TYPE_CHECKING, Optional
import torch
from sglang.srt.utils.custom_op import register_custom_op
@register_custom_op(out_shape=0)
def _l1_norm(x: torch.Tensor) -> torch.Tensor:
from sgl_kernel_npu.norm.l1_norm import l1_norm
return l1_norm(x)
from sgl_kernel_npu.norm.l1_norm import l1_norm
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical
@@ -68,7 +59,7 @@ def fused_topk_npu(
)
if renormalize:
topk_weights = _l1_norm(
topk_weights = l1_norm(
topk_weights
if topk_config.num_fused_shared_experts == 0
else topk_weights[:, :-1]
+1 -1
View File
@@ -625,7 +625,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
)
trace_sync("post_experts")
if enable_dual_stream:
final_hidden_states = wait_share_stream(final_hidden_states)
wait_share_stream()
elif enable_cuda_shared_overlap:
torch.cuda.current_stream().wait_event(shared_event)
+3 -53
View File
@@ -33,7 +33,6 @@ from sglang.srt.models.utils import apply_qk_norm
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import add_prefix, get_bool_env_var, is_cuda, is_hip, is_npu
from sglang.srt.utils.custom_op import register_custom_op
Qwen3Config = None
@@ -58,55 +57,6 @@ if _is_npu:
from sglang.srt.hardware_backend.npu.cmo import get_cmo_stream, wait_cmo_stream
def _split_qkv_rmsnorm_rope_fake_impl(
qkv: torch.Tensor,
sin: torch.Tensor,
cos: torch.Tensor,
q_size: int,
kv_size: int,
head_dim: int,
eps: float = 1e-6,
q_weight: Optional[torch.Tensor] = None,
k_weight: Optional[torch.Tensor] = None,
q_bias: Optional[torch.Tensor] = None,
k_bias: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias
output_shape = qkv.shape[:-1]
return (
qkv.new_empty((*output_shape, q_size)),
qkv.new_empty((*output_shape, kv_size)),
qkv.new_empty((*output_shape, kv_size)),
)
@register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl)
def _split_qkv_rmsnorm_rope_custom(
qkv: torch.Tensor,
sin: torch.Tensor,
cos: torch.Tensor,
q_size: int,
kv_size: int,
head_dim: int,
eps: float = 1e-6,
q_weight: Optional[torch.Tensor] = None,
k_weight: Optional[torch.Tensor] = None,
q_bias: Optional[torch.Tensor] = None,
k_bias: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return split_qkv_rmsnorm_rope(
qkv,
sin,
cos,
q_size,
kv_size,
head_dim,
eps=eps,
q_weight=q_weight,
k_weight=k_weight,
q_bias=q_bias,
k_bias=k_bias,
)
class Qwen3Attention(nn.Module):
def __init__(
@@ -236,7 +186,7 @@ class Qwen3Attention(nn.Module):
if self.attn.layer_id == self.start_layer:
self.rotary_emb.get_cos_sin_with_position(positions)
q, k, v = _split_qkv_rmsnorm_rope_custom(
q, k, v = split_qkv_rmsnorm_rope(
qkv,
self.rotary_emb.position_sin,
self.rotary_emb.position_cos,
@@ -472,8 +422,8 @@ class Qwen3DecoderLayer(nn.Module):
),
)
hidden_states = self.mlp(hidden_states, forward_batch=forward_batch)
if _is_npu and get_cmo_stream() is not None:
hidden_states = wait_cmo_stream(hidden_states)
if _is_npu and get_cmo_stream():
wait_cmo_stream()
hidden_states, residual = self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
+3 -9
View File
@@ -16,7 +16,6 @@
import logging
import os
from contextlib import nullcontext
from functools import lru_cache
from typing import Iterable, Optional, Set, Tuple, Union
@@ -1937,14 +1936,9 @@ class Qwen3_5ForCausalLM(nn.Module):
# Pass through decoder layers
for layer_idx in range(self.start_layer, self.end_layer):
layer = self.layers[layer_idx]
ctx = (
nullcontext()
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
else get_global_expert_distribution_recorder().with_current_layer(
layer_idx
)
)
with ctx:
with get_global_expert_distribution_recorder().with_current_layer(
layer_idx
):
hidden_states, residual = layer(
positions=positions,
hidden_states=hidden_states,
+1 -51
View File
@@ -73,7 +73,6 @@ from sglang.srt.utils import (
is_non_idle_and_non_empty,
is_npu,
)
from sglang.srt.utils.custom_op import register_custom_op
from sglang.srt.utils.hf_transformers_utils import get_rope_config
_is_cuda = is_cuda()
@@ -104,55 +103,6 @@ _is_npu = is_npu()
if _is_npu:
from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import split_qkv_rmsnorm_rope
def _split_qkv_rmsnorm_rope_fake_impl(
qkv: torch.Tensor,
sin: torch.Tensor,
cos: torch.Tensor,
q_size: int,
kv_size: int,
head_dim: int,
eps: float = 1e-6,
q_weight: Optional[torch.Tensor] = None,
k_weight: Optional[torch.Tensor] = None,
q_bias: Optional[torch.Tensor] = None,
k_bias: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias
output_shape = qkv.shape[:-1]
return (
qkv.new_empty((*output_shape, q_size)),
qkv.new_empty((*output_shape, kv_size)),
qkv.new_empty((*output_shape, kv_size)),
)
@register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl)
def _split_qkv_rmsnorm_rope_custom(
qkv: torch.Tensor,
sin: torch.Tensor,
cos: torch.Tensor,
q_size: int,
kv_size: int,
head_dim: int,
eps: float = 1e-6,
q_weight: Optional[torch.Tensor] = None,
k_weight: Optional[torch.Tensor] = None,
q_bias: Optional[torch.Tensor] = None,
k_bias: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return split_qkv_rmsnorm_rope(
qkv,
sin,
cos,
q_size,
kv_size,
head_dim,
eps=eps,
q_weight=q_weight,
k_weight=k_weight,
q_bias=q_bias,
k_bias=k_bias,
)
def compute_yarn_parameters(
config: PretrainedConfig,
@@ -672,7 +622,7 @@ class Qwen3MoeAttention(nn.Module):
qkv, _ = self.qkv_proj(hidden_states)
if self.attn.layer_id == self.start_layer:
self.rotary_emb.get_cos_sin_with_position(positions)
q, k, v = _split_qkv_rmsnorm_rope_custom(
q, k, v = split_qkv_rmsnorm_rope(
qkv,
self.rotary_emb.position_sin,
self.rotary_emb.position_cos,