[Perf] [NPU] attn: add not needs_cpu_seq_lens for dsa modules (#38461)
This commit is contained in:
@@ -1078,6 +1078,8 @@ class Envs:
|
||||
DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False)
|
||||
SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0)
|
||||
SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("")
|
||||
# Set to 0 to allow AscendAttnBackend to set needs_cpu_seq_lens to False for dsa models.
|
||||
SGLANG_NPU_ATTN_BACKEND_NEEDS_CPU_SEQ_LENS = EnvBool(True)
|
||||
|
||||
# ===================================================================
|
||||
# MUSA
|
||||
|
||||
@@ -12,6 +12,7 @@ from sgl_kernel_npu.attention.sinks_attention import (
|
||||
|
||||
from sglang.srt.configs.model_config import AttentionArch
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.hardware_backend.npu.attention.ascend_torch_native_backend import (
|
||||
AscendTorchNativeAttnBackend,
|
||||
)
|
||||
@@ -417,6 +418,12 @@ class AscendAttnBackend(AttentionBackend):
|
||||
and self.token_to_kv_pool.swa_layer_nums > 0
|
||||
)
|
||||
|
||||
if not envs.SGLANG_NPU_ATTN_BACKEND_NEEDS_CPU_SEQ_LENS.get():
|
||||
assert not self.is_hybrid_swa, (
|
||||
"Hybrid swa is not supply with `needs_cpu_seq_lens=False`."
|
||||
)
|
||||
self.needs_cpu_seq_lens = False
|
||||
|
||||
# head num padding
|
||||
self.padding_size_list = [1, 2, 4, 8, 16, 32, 64, 128]
|
||||
self.q_head_num_padding = None
|
||||
@@ -523,51 +530,59 @@ class AscendAttnBackend(AttentionBackend):
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
"""Init the metadata for a forward pass."""
|
||||
self.forward_metadata = ForwardMetadata()
|
||||
seq_lens_max = forward_batch.seq_lens.max()
|
||||
if forward_batch.forward_mode.is_target_verify():
|
||||
if (
|
||||
forward_batch.spec_algorithm is not None
|
||||
and forward_batch.spec_algorithm.is_dflash()
|
||||
if self.needs_cpu_seq_lens:
|
||||
seq_lens_max = forward_batch.seq_lens.max()
|
||||
if forward_batch.forward_mode.is_target_verify():
|
||||
if (
|
||||
forward_batch.spec_algorithm is not None
|
||||
and forward_batch.spec_algorithm.is_dflash()
|
||||
):
|
||||
# dflash_worker_v2 already publishes seq_lens_cpu as prefix +
|
||||
# one verify block, which already covers the draft block.
|
||||
seq_lens_max = forward_batch.seq_lens_cpu.max().item()
|
||||
else:
|
||||
# Overlap scheduling can publish the CPU sequence length one
|
||||
# step ahead of the device tensor. FIA consumes seq_lens_cpu
|
||||
# below, so derive the block-table width from the same source.
|
||||
# Otherwise a page-aligned request can expose KV_S=N while
|
||||
# asking FIA for N+1.
|
||||
spec_tokens_per_req = int(forward_batch.spec_info.draft_token_num)
|
||||
seq_lens_max = (
|
||||
forward_batch.seq_lens_cpu.max().item() + spec_tokens_per_req
|
||||
)
|
||||
elif (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
and forward_batch.spec_info is not None
|
||||
):
|
||||
# dflash_worker_v2 already publishes seq_lens_cpu as prefix +
|
||||
# one verify block, which already covers the draft block.
|
||||
seq_lens_max = forward_batch.seq_lens_cpu.max().item()
|
||||
seq_lens_max = forward_batch.seq_lens.max()
|
||||
seq_lens_max += self.speculative_step_id + 1
|
||||
else:
|
||||
# Overlap scheduling can publish the CPU sequence length one
|
||||
# step ahead of the device tensor. FIA consumes seq_lens_cpu
|
||||
# below, so derive the block-table width from the same source.
|
||||
# Otherwise a page-aligned request can expose KV_S=N while
|
||||
# asking FIA for N+1.
|
||||
spec_tokens_per_req = int(forward_batch.spec_info.draft_token_num)
|
||||
seq_lens_max = (
|
||||
forward_batch.seq_lens_cpu.max().item() + spec_tokens_per_req
|
||||
seq_lens_max = forward_batch.seq_lens.max()
|
||||
self.forward_metadata.block_tables = (
|
||||
self.req_to_token_pool.req_to_token[
|
||||
forward_batch.req_pool_indices, :seq_lens_max
|
||||
][:, :: self.page_size]
|
||||
// self.page_size
|
||||
)
|
||||
if self.is_hybrid_swa:
|
||||
self.forward_metadata.block_tables_swa = (
|
||||
(
|
||||
self.full_to_swa_index_mapping[
|
||||
self.req_to_token_pool.req_to_token[
|
||||
forward_batch.req_pool_indices, :seq_lens_max
|
||||
]
|
||||
][:, :: self.page_size]
|
||||
// self.page_size
|
||||
)
|
||||
.to(torch.int32)
|
||||
.contiguous()
|
||||
)
|
||||
elif (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
and forward_batch.spec_info is not None
|
||||
):
|
||||
seq_lens_max = forward_batch.seq_lens.max()
|
||||
seq_lens_max += self.speculative_step_id + 1
|
||||
else:
|
||||
seq_lens_max = forward_batch.seq_lens.max()
|
||||
self.forward_metadata.block_tables = (
|
||||
self.req_to_token_pool.req_to_token[
|
||||
forward_batch.req_pool_indices, :seq_lens_max
|
||||
][:, :: self.page_size]
|
||||
// self.page_size
|
||||
)
|
||||
if self.is_hybrid_swa:
|
||||
self.forward_metadata.block_tables_swa = (
|
||||
(
|
||||
self.full_to_swa_index_mapping[
|
||||
self.req_to_token_pool.req_to_token[
|
||||
forward_batch.req_pool_indices, :seq_lens_max
|
||||
]
|
||||
][:, :: self.page_size]
|
||||
// self.page_size
|
||||
)
|
||||
.to(torch.int32)
|
||||
.contiguous()
|
||||
self.forward_metadata.block_tables = (
|
||||
self.req_to_token_pool.req_to_token[
|
||||
forward_batch.req_pool_indices, :: self.page_size
|
||||
]
|
||||
// self.page_size
|
||||
)
|
||||
if forward_batch.extend_seq_lens is not None:
|
||||
self.forward_metadata.extend_seq_lens = forward_batch.extend_seq_lens
|
||||
@@ -581,7 +596,8 @@ class AscendAttnBackend(AttentionBackend):
|
||||
self.device
|
||||
).int()
|
||||
|
||||
self.forward_metadata.seq_lens_cpu_int = forward_batch.seq_lens_cpu.int()
|
||||
if self.needs_cpu_seq_lens:
|
||||
self.forward_metadata.seq_lens_cpu_int = forward_batch.seq_lens_cpu.int()
|
||||
# In graph mode (see _init_cuda_graph_metadata) seq_lens_cpu_int stays
|
||||
# None so forward_mtp binds seq_lens_cpu_list instead: graph.update can
|
||||
# only rebind the Host-side IntArray when captured as a Python list.
|
||||
@@ -893,7 +909,7 @@ class AscendAttnBackend(AttentionBackend):
|
||||
bs: int,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
forward_mode: ForwardMode,
|
||||
spec_info: Optional[SpecInput],
|
||||
out_cache_loc: Optional[torch.Tensor] = None,
|
||||
@@ -912,59 +928,75 @@ class AscendAttnBackend(AttentionBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc)
|
||||
)
|
||||
# Compute the host-side KV lengths visible to this attention step. Target
|
||||
# verify adds the draft block except for DFlash, whose seq_lens_cpu already
|
||||
# includes it; speculative decode adds tokens through the current draft step.
|
||||
attention_kv_lens_cpu = seq_lens_cpu[:bs].int()
|
||||
if forward_mode.is_target_verify() and not _is_dflash_verify(spec_info):
|
||||
attention_kv_lens_cpu = (
|
||||
attention_kv_lens_cpu + self.speculative_num_draft_tokens
|
||||
)
|
||||
elif forward_mode.is_decode_or_idle() and spec_info is not None:
|
||||
attention_kv_lens_cpu = attention_kv_lens_cpu + self.speculative_step_id + 1
|
||||
max_len = attention_kv_lens_cpu.max().item()
|
||||
max_seq_pages = (max_len + self.page_size - 1) // self.page_size
|
||||
if self.needs_cpu_seq_lens:
|
||||
# Compute the host-side KV lengths visible to this attention step. Target
|
||||
# verify adds the draft block except for DFlash, whose seq_lens_cpu already
|
||||
# includes it; speculative decode adds tokens through the current draft step.
|
||||
attention_kv_lens_cpu = seq_lens_cpu[:bs].int()
|
||||
if forward_mode.is_target_verify() and not _is_dflash_verify(spec_info):
|
||||
attention_kv_lens_cpu = (
|
||||
attention_kv_lens_cpu + self.speculative_num_draft_tokens
|
||||
)
|
||||
elif forward_mode.is_decode_or_idle() and spec_info is not None:
|
||||
attention_kv_lens_cpu = (
|
||||
attention_kv_lens_cpu + self.speculative_step_id + 1
|
||||
)
|
||||
max_len = attention_kv_lens_cpu.max().item()
|
||||
max_seq_pages = (max_len + self.page_size - 1) // self.page_size
|
||||
|
||||
if self.is_hybrid_swa:
|
||||
full_page_locs = self.req_to_token[
|
||||
req_pool_indices[:bs],
|
||||
0 : max_len : self.page_size,
|
||||
]
|
||||
swa_page_table = (
|
||||
self.full_to_swa_index_mapping[full_page_locs] // self.page_size
|
||||
if self.is_hybrid_swa:
|
||||
full_page_locs = self.req_to_token[
|
||||
req_pool_indices[:bs],
|
||||
0 : max_len : self.page_size,
|
||||
]
|
||||
swa_page_table = (
|
||||
self.full_to_swa_index_mapping[full_page_locs] // self.page_size
|
||||
)
|
||||
|
||||
metadata.block_tables_swa[:bs, :max_seq_pages].copy_(swa_page_table)
|
||||
metadata.block_tables_swa[:bs, max_seq_pages:].fill_(0)
|
||||
metadata.block_tables_swa[bs:, :].fill_(0)
|
||||
|
||||
# Update SWA mask: True = masked out (don't attend), False = attend.
|
||||
# DFlash verify seq_lens is prefix-only, so use seq_lens_cpu
|
||||
# (= prefix + block_size) to keep draft KV inside the mask window.
|
||||
if (
|
||||
forward_mode.is_target_verify()
|
||||
and _is_dflash_verify(spec_info)
|
||||
and seq_lens_cpu is not None
|
||||
):
|
||||
# seq_lens_cpu may be a CPU tensor; swa_indices is on-device.
|
||||
seq_lens_int = seq_lens_cpu[:bs].int().to(self.device)
|
||||
else:
|
||||
seq_lens_int = seq_lens[:bs].int()
|
||||
starts = torch.clamp(seq_lens_int - self.sliding_window_size, min=0)
|
||||
indices = self.graph_metadata["swa_indices"]
|
||||
start_exp = starts.unsqueeze(1)
|
||||
seq_exp = seq_lens_int.unsqueeze(1)
|
||||
mask = (indices.unsqueeze(0) < start_exp) | (
|
||||
indices.unsqueeze(0) >= seq_exp
|
||||
)
|
||||
metadata.swa_mask[:bs, 0, :].copy_(mask)
|
||||
metadata.swa_mask[bs:, :, :].fill_(True)
|
||||
metadata.block_tables[:bs, :max_seq_pages].copy_(
|
||||
self.req_to_token[req_pool_indices[:bs], 0 : max_len : self.page_size]
|
||||
// self.page_size
|
||||
)
|
||||
|
||||
metadata.block_tables_swa[:bs, :max_seq_pages].copy_(swa_page_table)
|
||||
metadata.block_tables_swa[:bs, max_seq_pages:].fill_(0)
|
||||
metadata.block_tables_swa[bs:, :].fill_(0)
|
||||
|
||||
# Update SWA mask: True = masked out (don't attend), False = attend.
|
||||
# DFlash verify seq_lens is prefix-only, so use seq_lens_cpu
|
||||
# (= prefix + block_size) to keep draft KV inside the mask window.
|
||||
if (
|
||||
forward_mode.is_target_verify()
|
||||
and _is_dflash_verify(spec_info)
|
||||
and seq_lens_cpu is not None
|
||||
):
|
||||
# seq_lens_cpu may be a CPU tensor; swa_indices is on-device.
|
||||
seq_lens_int = seq_lens_cpu[:bs].int().to(self.device)
|
||||
else:
|
||||
seq_lens_int = seq_lens[:bs].int()
|
||||
starts = torch.clamp(seq_lens_int - self.sliding_window_size, min=0)
|
||||
indices = self.graph_metadata["swa_indices"]
|
||||
start_exp = starts.unsqueeze(1)
|
||||
seq_exp = seq_lens_int.unsqueeze(1)
|
||||
mask = (indices.unsqueeze(0) < start_exp) | (
|
||||
indices.unsqueeze(0) >= seq_exp
|
||||
metadata.block_tables[:bs, max_seq_pages:].fill_(0)
|
||||
else:
|
||||
total_pages = min(
|
||||
metadata.block_tables.shape[1],
|
||||
(self.req_to_token.shape[1] + self.page_size - 1) // self.page_size,
|
||||
)
|
||||
metadata.swa_mask[:bs, 0, :].copy_(mask)
|
||||
metadata.swa_mask[bs:, :, :].fill_(True)
|
||||
metadata.block_tables[:bs, :max_seq_pages].copy_(
|
||||
self.req_to_token[req_pool_indices[:bs], 0 : max_len : self.page_size]
|
||||
// self.page_size
|
||||
)
|
||||
|
||||
metadata.block_tables[:bs, max_seq_pages:].fill_(0)
|
||||
metadata.block_tables[:bs, :total_pages].copy_(
|
||||
self.req_to_token[
|
||||
req_pool_indices[:bs],
|
||||
0 : total_pages * self.page_size : self.page_size,
|
||||
]
|
||||
// self.page_size
|
||||
)
|
||||
metadata.block_tables[:bs, total_pages:].fill_(0)
|
||||
|
||||
if get_parallel().dcp_enabled and not self.is_draft_worker:
|
||||
if "dcp_origin_out_cache_loc" in self.graph_metadata:
|
||||
@@ -3430,6 +3462,7 @@ class AscendAttnMultiStepDraftBackend:
|
||||
self.attn_backends.append(
|
||||
AscendAttnBackend(model_runner, speculative_step_id=step_id)
|
||||
)
|
||||
self.needs_cpu_seq_lens = any(b.needs_cpu_seq_lens for b in self.attn_backends)
|
||||
|
||||
def common_template(self, forward_batch: ForwardBatch, call_fn: int):
|
||||
assert forward_batch.spec_info is not None
|
||||
|
||||
@@ -78,12 +78,12 @@ class EAGLEDraftNpuGraphRunner(EAGLEDraftCudaGraphRunner):
|
||||
decision = torch.tensor(
|
||||
int(can_run_graph and seed_ready),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
device="cpu",
|
||||
)
|
||||
torch.distributed.all_reduce(
|
||||
decision,
|
||||
op=torch.distributed.ReduceOp.MIN,
|
||||
group=get_parallel().tp_group.device_group,
|
||||
group=get_parallel().tp_group.cpu_group,
|
||||
)
|
||||
return bool(decision.item())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user