[Perf] [NPU] attn: add not needs_cpu_seq_lens for dsa modules (#38461)

This commit is contained in:
Kurkur
2026-10-20 19:19:34 +08:00
committed by GitHub
parent ca473ef3e4
commit 07eb671b6a
4 changed files with 197 additions and 94 deletions
+2
View File
@@ -1078,6 +1078,8 @@ class Envs:
DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False)
SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0)
SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("")
# Set to 0 to allow AscendAttnBackend to set needs_cpu_seq_lens to False for dsa models.
SGLANG_NPU_ATTN_BACKEND_NEEDS_CPU_SEQ_LENS = EnvBool(True)
# ===================================================================
# MUSA
@@ -12,6 +12,7 @@ from sgl_kernel_npu.attention.sinks_attention import (
from sglang.srt.configs.model_config import AttentionArch
from sglang.srt.dllm.config import DllmConfig
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.npu.attention.ascend_torch_native_backend import (
AscendTorchNativeAttnBackend,
)
@@ -417,6 +418,12 @@ class AscendAttnBackend(AttentionBackend):
and self.token_to_kv_pool.swa_layer_nums > 0
)
if not envs.SGLANG_NPU_ATTN_BACKEND_NEEDS_CPU_SEQ_LENS.get():
assert not self.is_hybrid_swa, (
"Hybrid swa is not supply with `needs_cpu_seq_lens=False`."
)
self.needs_cpu_seq_lens = False
# head num padding
self.padding_size_list = [1, 2, 4, 8, 16, 32, 64, 128]
self.q_head_num_padding = None
@@ -523,51 +530,59 @@ class AscendAttnBackend(AttentionBackend):
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Init the metadata for a forward pass."""
self.forward_metadata = ForwardMetadata()
seq_lens_max = forward_batch.seq_lens.max()
if forward_batch.forward_mode.is_target_verify():
if (
forward_batch.spec_algorithm is not None
and forward_batch.spec_algorithm.is_dflash()
if self.needs_cpu_seq_lens:
seq_lens_max = forward_batch.seq_lens.max()
if forward_batch.forward_mode.is_target_verify():
if (
forward_batch.spec_algorithm is not None
and forward_batch.spec_algorithm.is_dflash()
):
# dflash_worker_v2 already publishes seq_lens_cpu as prefix +
# one verify block, which already covers the draft block.
seq_lens_max = forward_batch.seq_lens_cpu.max().item()
else:
# Overlap scheduling can publish the CPU sequence length one
# step ahead of the device tensor. FIA consumes seq_lens_cpu
# below, so derive the block-table width from the same source.
# Otherwise a page-aligned request can expose KV_S=N while
# asking FIA for N+1.
spec_tokens_per_req = int(forward_batch.spec_info.draft_token_num)
seq_lens_max = (
forward_batch.seq_lens_cpu.max().item() + spec_tokens_per_req
)
elif (
forward_batch.forward_mode.is_decode_or_idle()
and forward_batch.spec_info is not None
):
# dflash_worker_v2 already publishes seq_lens_cpu as prefix +
# one verify block, which already covers the draft block.
seq_lens_max = forward_batch.seq_lens_cpu.max().item()
seq_lens_max = forward_batch.seq_lens.max()
seq_lens_max += self.speculative_step_id + 1
else:
# Overlap scheduling can publish the CPU sequence length one
# step ahead of the device tensor. FIA consumes seq_lens_cpu
# below, so derive the block-table width from the same source.
# Otherwise a page-aligned request can expose KV_S=N while
# asking FIA for N+1.
spec_tokens_per_req = int(forward_batch.spec_info.draft_token_num)
seq_lens_max = (
forward_batch.seq_lens_cpu.max().item() + spec_tokens_per_req
seq_lens_max = forward_batch.seq_lens.max()
self.forward_metadata.block_tables = (
self.req_to_token_pool.req_to_token[
forward_batch.req_pool_indices, :seq_lens_max
][:, :: self.page_size]
// self.page_size
)
if self.is_hybrid_swa:
self.forward_metadata.block_tables_swa = (
(
self.full_to_swa_index_mapping[
self.req_to_token_pool.req_to_token[
forward_batch.req_pool_indices, :seq_lens_max
]
][:, :: self.page_size]
// self.page_size
)
.to(torch.int32)
.contiguous()
)
elif (
forward_batch.forward_mode.is_decode_or_idle()
and forward_batch.spec_info is not None
):
seq_lens_max = forward_batch.seq_lens.max()
seq_lens_max += self.speculative_step_id + 1
else:
seq_lens_max = forward_batch.seq_lens.max()
self.forward_metadata.block_tables = (
self.req_to_token_pool.req_to_token[
forward_batch.req_pool_indices, :seq_lens_max
][:, :: self.page_size]
// self.page_size
)
if self.is_hybrid_swa:
self.forward_metadata.block_tables_swa = (
(
self.full_to_swa_index_mapping[
self.req_to_token_pool.req_to_token[
forward_batch.req_pool_indices, :seq_lens_max
]
][:, :: self.page_size]
// self.page_size
)
.to(torch.int32)
.contiguous()
self.forward_metadata.block_tables = (
self.req_to_token_pool.req_to_token[
forward_batch.req_pool_indices, :: self.page_size
]
// self.page_size
)
if forward_batch.extend_seq_lens is not None:
self.forward_metadata.extend_seq_lens = forward_batch.extend_seq_lens
@@ -581,7 +596,8 @@ class AscendAttnBackend(AttentionBackend):
self.device
).int()
self.forward_metadata.seq_lens_cpu_int = forward_batch.seq_lens_cpu.int()
if self.needs_cpu_seq_lens:
self.forward_metadata.seq_lens_cpu_int = forward_batch.seq_lens_cpu.int()
# In graph mode (see _init_cuda_graph_metadata) seq_lens_cpu_int stays
# None so forward_mtp binds seq_lens_cpu_list instead: graph.update can
# only rebind the Host-side IntArray when captured as a Python list.
@@ -893,7 +909,7 @@ class AscendAttnBackend(AttentionBackend):
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: torch.Tensor,
seq_lens_cpu: Optional[torch.Tensor],
forward_mode: ForwardMode,
spec_info: Optional[SpecInput],
out_cache_loc: Optional[torch.Tensor] = None,
@@ -912,59 +928,75 @@ class AscendAttnBackend(AttentionBackend):
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc)
)
# Compute the host-side KV lengths visible to this attention step. Target
# verify adds the draft block except for DFlash, whose seq_lens_cpu already
# includes it; speculative decode adds tokens through the current draft step.
attention_kv_lens_cpu = seq_lens_cpu[:bs].int()
if forward_mode.is_target_verify() and not _is_dflash_verify(spec_info):
attention_kv_lens_cpu = (
attention_kv_lens_cpu + self.speculative_num_draft_tokens
)
elif forward_mode.is_decode_or_idle() and spec_info is not None:
attention_kv_lens_cpu = attention_kv_lens_cpu + self.speculative_step_id + 1
max_len = attention_kv_lens_cpu.max().item()
max_seq_pages = (max_len + self.page_size - 1) // self.page_size
if self.needs_cpu_seq_lens:
# Compute the host-side KV lengths visible to this attention step. Target
# verify adds the draft block except for DFlash, whose seq_lens_cpu already
# includes it; speculative decode adds tokens through the current draft step.
attention_kv_lens_cpu = seq_lens_cpu[:bs].int()
if forward_mode.is_target_verify() and not _is_dflash_verify(spec_info):
attention_kv_lens_cpu = (
attention_kv_lens_cpu + self.speculative_num_draft_tokens
)
elif forward_mode.is_decode_or_idle() and spec_info is not None:
attention_kv_lens_cpu = (
attention_kv_lens_cpu + self.speculative_step_id + 1
)
max_len = attention_kv_lens_cpu.max().item()
max_seq_pages = (max_len + self.page_size - 1) // self.page_size
if self.is_hybrid_swa:
full_page_locs = self.req_to_token[
req_pool_indices[:bs],
0 : max_len : self.page_size,
]
swa_page_table = (
self.full_to_swa_index_mapping[full_page_locs] // self.page_size
if self.is_hybrid_swa:
full_page_locs = self.req_to_token[
req_pool_indices[:bs],
0 : max_len : self.page_size,
]
swa_page_table = (
self.full_to_swa_index_mapping[full_page_locs] // self.page_size
)
metadata.block_tables_swa[:bs, :max_seq_pages].copy_(swa_page_table)
metadata.block_tables_swa[:bs, max_seq_pages:].fill_(0)
metadata.block_tables_swa[bs:, :].fill_(0)
# Update SWA mask: True = masked out (don't attend), False = attend.
# DFlash verify seq_lens is prefix-only, so use seq_lens_cpu
# (= prefix + block_size) to keep draft KV inside the mask window.
if (
forward_mode.is_target_verify()
and _is_dflash_verify(spec_info)
and seq_lens_cpu is not None
):
# seq_lens_cpu may be a CPU tensor; swa_indices is on-device.
seq_lens_int = seq_lens_cpu[:bs].int().to(self.device)
else:
seq_lens_int = seq_lens[:bs].int()
starts = torch.clamp(seq_lens_int - self.sliding_window_size, min=0)
indices = self.graph_metadata["swa_indices"]
start_exp = starts.unsqueeze(1)
seq_exp = seq_lens_int.unsqueeze(1)
mask = (indices.unsqueeze(0) < start_exp) | (
indices.unsqueeze(0) >= seq_exp
)
metadata.swa_mask[:bs, 0, :].copy_(mask)
metadata.swa_mask[bs:, :, :].fill_(True)
metadata.block_tables[:bs, :max_seq_pages].copy_(
self.req_to_token[req_pool_indices[:bs], 0 : max_len : self.page_size]
// self.page_size
)
metadata.block_tables_swa[:bs, :max_seq_pages].copy_(swa_page_table)
metadata.block_tables_swa[:bs, max_seq_pages:].fill_(0)
metadata.block_tables_swa[bs:, :].fill_(0)
# Update SWA mask: True = masked out (don't attend), False = attend.
# DFlash verify seq_lens is prefix-only, so use seq_lens_cpu
# (= prefix + block_size) to keep draft KV inside the mask window.
if (
forward_mode.is_target_verify()
and _is_dflash_verify(spec_info)
and seq_lens_cpu is not None
):
# seq_lens_cpu may be a CPU tensor; swa_indices is on-device.
seq_lens_int = seq_lens_cpu[:bs].int().to(self.device)
else:
seq_lens_int = seq_lens[:bs].int()
starts = torch.clamp(seq_lens_int - self.sliding_window_size, min=0)
indices = self.graph_metadata["swa_indices"]
start_exp = starts.unsqueeze(1)
seq_exp = seq_lens_int.unsqueeze(1)
mask = (indices.unsqueeze(0) < start_exp) | (
indices.unsqueeze(0) >= seq_exp
metadata.block_tables[:bs, max_seq_pages:].fill_(0)
else:
total_pages = min(
metadata.block_tables.shape[1],
(self.req_to_token.shape[1] + self.page_size - 1) // self.page_size,
)
metadata.swa_mask[:bs, 0, :].copy_(mask)
metadata.swa_mask[bs:, :, :].fill_(True)
metadata.block_tables[:bs, :max_seq_pages].copy_(
self.req_to_token[req_pool_indices[:bs], 0 : max_len : self.page_size]
// self.page_size
)
metadata.block_tables[:bs, max_seq_pages:].fill_(0)
metadata.block_tables[:bs, :total_pages].copy_(
self.req_to_token[
req_pool_indices[:bs],
0 : total_pages * self.page_size : self.page_size,
]
// self.page_size
)
metadata.block_tables[:bs, total_pages:].fill_(0)
if get_parallel().dcp_enabled and not self.is_draft_worker:
if "dcp_origin_out_cache_loc" in self.graph_metadata:
@@ -3430,6 +3462,7 @@ class AscendAttnMultiStepDraftBackend:
self.attn_backends.append(
AscendAttnBackend(model_runner, speculative_step_id=step_id)
)
self.needs_cpu_seq_lens = any(b.needs_cpu_seq_lens for b in self.attn_backends)
def common_template(self, forward_batch: ForwardBatch, call_fn: int):
assert forward_batch.spec_info is not None
@@ -78,12 +78,12 @@ class EAGLEDraftNpuGraphRunner(EAGLEDraftCudaGraphRunner):
decision = torch.tensor(
int(can_run_graph and seed_ready),
dtype=torch.int32,
device=self.device,
device="cpu",
)
torch.distributed.all_reduce(
decision,
op=torch.distributed.ReduceOp.MIN,
group=get_parallel().tp_group.device_group,
group=get_parallel().tp_group.cpu_group,
)
return bool(decision.item())