[NPU] Add decode context parallel support for dsa models (#37787)

Co-authored-by: sglang-npu-bot <sglangnpu@163.com>
This commit is contained in:
heziiop
2026-09-30 15:22:16 +08:00
committed by GitHub
co-authored by sglang-npu-bot
parent 8c75ad9e62
commit ec0006673a
21 changed files with 1572 additions and 57 deletions
@@ -164,6 +164,13 @@ def handle_model_specific_adjustments(server_args: Any):
hf_config = model_config.hf_config
model_arch = hf_config.architectures[0]
if get_platform().is_npu and cfg.dcp_size > 1 and not is_deepseek_dsa(hf_config):
raise ValueError(
"NPU decode context parallelism is currently implemented only for "
"DeepSeek DSA models; got "
f"{model_arch}. Set --decode-context-parallel-size=1 or use a DSA model."
)
if model_arch == "InternS2MobiusForConditionalGeneration":
unsupported = []
if cfg.pp_size != 1:
@@ -74,11 +74,12 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
if cfg.disaggregation_transfer_backend not in (
"mooncake",
"nixl",
"ascend",
"fake",
):
raise ValueError(
"PD decode DCP requires --disaggregation-transfer-backend "
"mooncake, nixl, or fake for synthetic benchmarking, got "
"mooncake, nixl, ascend, or fake for synthetic benchmarking, got "
f"{cfg.disaggregation_transfer_backend!r}."
)
@@ -34,7 +34,81 @@ class AscendStateType(str, enum.Enum):
_DSV4_KVCACHE_STATE_TYPES = tuple(AscendStateType)
def _build_page_interleaved_dcp_plan(
src_page_indices: npt.NDArray[np.int32],
dst_page_indices: npt.NDArray[np.int32],
*,
page_size: int,
dcp_size: int,
dcp_rank: int,
src_page_offset: int,
decode_prefix_len: int,
num_kv_tokens: int,
) -> Tuple[npt.NDArray[np.int64], ...]:
"""Map physical prefill pages to local/global decode page slots."""
if not 0 <= dcp_rank < dcp_size:
raise ValueError(f"Invalid DCP rank {dcp_rank} for size {dcp_size}")
virtual_page_size = page_size * dcp_size
if decode_prefix_len % virtual_page_size:
raise ValueError(
"Ascend PD DCP requires decode_prefix_len to align to the virtual "
f"page size ({virtual_page_size}), got {decode_prefix_len}"
)
if src_page_offset < 0 or num_kv_tokens < 0:
raise ValueError(
"Ascend PD DCP page offset and token count must be nonnegative"
)
src_pages = np.asarray(src_page_indices, dtype=np.int64)
dst_pages = np.asarray(dst_page_indices, dtype=np.int64)
max_src_pages = (num_kv_tokens + page_size - 1) // page_size
if src_pages.size > max_src_pages:
raise ValueError(
"Ascend PD DCP source page count exceeds the token count: "
f"pages={src_pages.size}, tokens={num_kv_tokens}, page_size={page_size}"
)
if src_pages.size == 0:
empty = np.empty((0,), dtype=np.int64)
return empty, empty.copy(), empty.copy(), empty.copy()
# CP may assign this sender only a contiguous subset of the chunk pages;
# index_slice.start still carries that subset's suffix-relative offset.
relative_pages = src_page_offset + np.arange(src_pages.size, dtype=np.int64)
dst_positions = relative_pages // dcp_size
if dst_positions[-1] >= dst_pages.size:
raise ValueError(
"Ascend PD DCP destination does not contain enough virtual pages: "
f"required={dst_positions[-1] + 1}, available={dst_pages.size}"
)
dst_super_pages = dst_pages[dst_positions]
owners = (decode_prefix_len // page_size + relative_pages) % dcp_size
local = owners == dcp_rank
return (
src_pages[local],
dst_super_pages[local],
src_pages,
dst_super_pages * dcp_size + owners,
)
class AscendKVManager(MooncakeKVManager):
def __init__(
self,
args,
disaggregation_mode,
server_args,
is_mla_backend: Optional[bool] = False,
dcp_remote_decode_layout=None,
):
self._dcp_remote_decode_layout = (
None if dcp_remote_decode_layout is None else list(dcp_remote_decode_layout)
)
super().__init__(args, disaggregation_mode, server_args, is_mla_backend)
def _is_npu_dsa_layout(self) -> bool:
return getattr(self.kv_args, "kv_buf_groups", 1) == 3
def _requires_exact_state_index_match(self, st: StateType) -> bool:
return (
super()._requires_exact_state_index_match(st)
@@ -67,6 +141,47 @@ class AscendKVManager(MooncakeKVManager):
if ptrs:
self.engine.batch_register(ptrs, lens)
def requires_dcp_relayout(self, dst_dcp_size: int, dst_dcp_rank: int) -> bool:
if self._is_npu_dsa_layout() and self.dcp_size != dst_dcp_size:
if self.dcp_size == 1 and dst_dcp_size > 1:
return True
raise RuntimeError(
"NPU DSA PD supports prefill DCP 1 -> decode DCP N only, got "
f"prefill={self.dcp_size}, decode={dst_dcp_size}"
)
return super().requires_dcp_relayout(dst_dcp_size, dst_dcp_rank)
def _get_dcp_remote_decode_layout(self) -> List[bool]:
layout = self._dcp_remote_decode_layout
if layout is None or len(layout) != len(self.kv_args.kv_data_ptrs):
raise RuntimeError(
"Ascend PD DCP layout does not match its source KV entries"
)
return layout
def prepare_dcp_token_item_lens(
self, dst_page_item_lens: List[Optional[int]], dst_dcp_size: int
) -> List[int]:
if not self._is_npu_dsa_layout():
return super().prepare_dcp_token_item_lens(dst_page_item_lens, dst_dcp_size)
self._get_dcp_remote_decode_layout()
page_size = self.kv_args.page_size
token_item_lens = []
for entry, item_len in enumerate(self.kv_args.kv_item_lens):
token_item_len, remainder = divmod(item_len, page_size)
if remainder:
raise RuntimeError(
f"Ascend PD DCP source entry {entry} is not page aligned"
)
token_item_lens.append(token_item_len)
return token_item_lens
def _init_dcp_pack_buffers_once(
self, dcp_size: int, *, include_draft: bool = False
) -> None:
# Page-interleaved NPU DCP transfers whole pages directly.
self._dcp_pack_buffers = []
def get_mla_kv_ptrs_with_pp(
self, src_kv_ptrs: List[int], dst_kv_ptrs: List[int], state_type=None
) -> Tuple[List[int], List[int], int]:
@@ -289,6 +404,109 @@ class AscendKVManager(MooncakeKVManager):
return 0
def send_kvcache_dcp(
self,
mooncake_session_id: str,
prefill_kv_indices: npt.NDArray[np.int32],
dst_kv_ptrs: list[int],
dst_kv_indices: npt.NDArray[np.int32],
*,
dcp_token_item_lens: List[int],
dst_dcp_size: int,
dst_dcp_rank: int,
src_page_offset: int,
decode_prefix_len: int,
num_kv_tokens: int,
executor: concurrent.futures.ThreadPoolExecutor,
dst_layer_ids: List[int],
pack_buffer=None,
dst_kv_item_lens: Optional[List[int]] = None,
dst_tp_rank: int = 0,
dst_attn_tp_size: Optional[int] = None,
) -> int:
if not self._is_npu_dsa_layout():
return super().send_kvcache_dcp(
mooncake_session_id,
prefill_kv_indices,
dst_kv_ptrs,
dst_kv_indices,
dcp_token_item_lens=dcp_token_item_lens,
dst_dcp_size=dst_dcp_size,
dst_dcp_rank=dst_dcp_rank,
src_page_offset=src_page_offset,
decode_prefix_len=decode_prefix_len,
num_kv_tokens=num_kv_tokens,
executor=executor,
dst_layer_ids=dst_layer_ids,
pack_buffer=pack_buffer,
dst_kv_item_lens=dst_kv_item_lens,
dst_tp_rank=dst_tp_rank,
dst_attn_tp_size=dst_attn_tp_size,
)
src_kv_ptrs = self.kv_args.kv_data_ptrs
_, dst_kv_ptrs, _ = self.get_mla_kv_ptrs_with_pp(src_kv_ptrs, dst_kv_ptrs)
if dst_kv_item_lens:
_, dst_kv_item_lens, _ = self.get_mla_kv_ptrs_with_pp(
self.kv_args.kv_item_lens, dst_kv_item_lens
)
layout = self._get_dcp_remote_decode_layout()
num_entries = len(src_kv_ptrs)
if not (
len(dst_kv_ptrs)
== len(self.kv_args.kv_item_lens)
== len(dcp_token_item_lens)
== len(layout)
== num_entries
):
raise RuntimeError("Ascend PD DCP KV entry metadata is inconsistent")
if dst_kv_item_lens and len(dst_kv_item_lens) != num_entries:
raise RuntimeError("Ascend PD DCP destination KV metadata is inconsistent")
local_src, local_dst, global_src, global_dst = _build_page_interleaved_dcp_plan(
prefill_kv_indices,
dst_kv_indices,
page_size=self.kv_args.page_size,
dcp_size=dst_dcp_size,
dcp_rank=dst_dcp_rank,
src_page_offset=src_page_offset,
decode_prefix_len=decode_prefix_len,
num_kv_tokens=num_kv_tokens,
)
transfer_blocks: List[Tuple[int, int, int]] = []
for entry, uses_global_slots in enumerate(layout):
page_bytes = self.kv_args.kv_item_lens[entry]
if page_bytes != dcp_token_item_lens[entry] * self.kv_args.page_size:
raise RuntimeError(
f"Ascend PD DCP source geometry differs at entry {entry}"
)
if dst_kv_item_lens:
expected_dst_bytes = page_bytes * (
dst_dcp_size if uses_global_slots else 1
)
if dst_kv_item_lens[entry] != expected_dst_bytes:
raise RuntimeError(
"Ascend PD DCP destination geometry differs at entry "
f"{entry}: expected={expected_dst_bytes}, "
f"actual={dst_kv_item_lens[entry]}"
)
src_pages = global_src if uses_global_slots else local_src
dst_pages = global_dst if uses_global_slots else local_dst
src_groups, dst_groups = group_concurrent_contiguous(src_pages, dst_pages)
for src_group, dst_group in zip(src_groups, dst_groups):
transfer_blocks.append(
(
src_kv_ptrs[entry] + int(src_group[0]) * page_bytes,
dst_kv_ptrs[entry] + int(dst_group[0]) * page_bytes,
len(src_group) * page_bytes,
)
)
return self._transfer_data(mooncake_session_id, transfer_blocks)
def _is_generic_kvcache_state_type(self, st) -> bool:
# DSV4 per-pool components also use the page-indexed send path.
return (
@@ -290,6 +290,13 @@ class PrefillBootstrapQueue:
kv_item_lens += draft_kv_item_lens
num_draft_entries = len(draft_kv_data_ptrs)
dcp_remote_decode_layout = []
if self.transfer_backend == TransferBackend.ASCEND:
for pool in (self.token_to_kv_pool, draft_kv_pool):
get_layout = getattr(pool, "get_dcp_remote_decode_layout", None)
if get_layout is not None:
dcp_remote_decode_layout.extend(get_layout())
kv_args.kv_data_ptrs = kv_data_ptrs
kv_args.kv_data_lens = kv_data_lens
kv_args.kv_item_lens = kv_item_lens
@@ -323,11 +330,17 @@ class PrefillBootstrapQueue:
)
kv_manager_class = get_kv_class(self.transfer_backend, KVClassType.MANAGER)
kv_manager_kwargs = (
{"dcp_remote_decode_layout": dcp_remote_decode_layout}
if self.transfer_backend == TransferBackend.ASCEND
else {}
)
kv_manager = kv_manager_class(
kv_args,
DisaggregationMode.PREFILL,
self.scheduler.server_args,
self.is_mla_backend,
**kv_manager_kwargs,
)
# Pass KV pool tensor refs to the manager for GPU gather (staging mode)
if (
+1 -1
View File
@@ -1558,7 +1558,7 @@ def setup_state_kv_args(
tail_item_lens = tail_item_lens + draft_tail_item_lens
if isinstance(token_to_kv_pool, NPUMLATokenToKVPool):
kv_args.kv_buf_groups = (
len(kv_args.kv_data_ptrs) // token_to_kv_pool.layer_num
3 if token_to_kv_pool.index_head_dim is not None else 2
)
kv_args.hidden_kv_layers = total_kv_layers
kv_args.draft_kv_layers = (
@@ -2639,12 +2639,12 @@ def initialize_model_parallel(
raise RuntimeError(
f"decode_context_parallel_size ({decode_context_parallel_size}) must be >= 1"
)
if decode_context_parallel_size > 1 and not (is_hip() or is_cuda()):
if decode_context_parallel_size > 1 and not (is_hip() or is_cuda() or _is_npu):
raise RuntimeError(
"Decode context parallel (decode_context_parallel_size > 1) is "
"currently only supported on the AMD HIP platform or CUDA platform, but got "
f"decode_context_parallel_size ({decode_context_parallel_size}) "
"on a non-HIP or non-CUDA platform."
"currently only supported on the AMD HIP, CUDA, or NPU "
"platform, but got decode_context_parallel_size "
f"({decode_context_parallel_size}) on an unsupported platform."
)
if tensor_model_parallel_size % decode_context_parallel_size != 0:
raise RuntimeError(
@@ -15,6 +15,9 @@ from sglang.srt.dllm.config import DllmConfig
from sglang.srt.hardware_backend.npu.attention.ascend_torch_native_backend import (
AscendTorchNativeAttnBackend,
)
from sglang.srt.hardware_backend.npu.attention.dsa_dcp import (
forward_dcp_sparse_attention,
)
from sglang.srt.hardware_backend.npu.attention.mla_cache import gather_mla_cache_pages
from sglang.srt.hardware_backend.npu.attention.mla_preprocess import (
is_fia_nz,
@@ -26,6 +29,10 @@ from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import (
)
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.dcp.layout import (
get_dcp_chain_spec_lens,
get_dcp_lens,
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.layers.utils.cp_utils import cp_all_gather_rerange_kv_cache
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
@@ -100,6 +107,16 @@ class ForwardMetadata:
actual_seq_lengths_q_pa_cpu: Optional[torch.Tensor] = None
actual_seq_lengths_kv: Optional[torch.Tensor] = None
# DCP decode inputs
dcp_seq_lens_cpu_int: Optional[torch.Tensor] = None
dcp_seq_lens: Optional[torch.Tensor] = None
dcp_block_tables: Optional[torch.Tensor] = None
dcp_origin_out_cache_loc: Optional[torch.Tensor] = None
# DCP target-verify inputs, expanded to one row per speculative query.
dcp_spec_seq_lens_cpu_int: Optional[torch.Tensor] = None
dcp_spec_seq_lens: Optional[torch.Tensor] = None
dcp_spec_block_tables: Optional[torch.Tensor] = None
# swa attention mask for graph mode decode
swa_mask: Optional[torch.Tensor] = None
@@ -309,6 +326,7 @@ class AscendAttnBackend(AttentionBackend):
super().__init__()
self.forward_metadata = None
self.device = model_runner.device
self.is_draft_worker = model_runner.is_draft_worker
self.speculative_step_id = speculative_step_id
self.speculative_step_offset_npu = torch.tensor(
speculative_step_id + 1, device="npu"
@@ -444,6 +462,40 @@ class AscendAttnBackend(AttentionBackend):
v = layer.v_head_dim
return (d == v and d in (128, 192, 256)) or (d == 192 and v == 128)
def _get_kv_lens_and_block_tables(
self,
kv_lens_cpu: torch.Tensor,
req_pool_indices: torch.Tensor,
is_spec: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Build rank-local paged KV metadata for NPU DSA DCP."""
parallel = get_parallel()
if is_spec:
local_kv_lens = get_dcp_chain_spec_lens(
kv_lens_cpu,
self.speculative_num_draft_tokens,
parallel.attn_dcp_size,
parallel.attn_dcp_rank,
interleave_size=self.page_size,
)
else:
local_kv_lens = get_dcp_lens(
kv_lens_cpu,
parallel.attn_dcp_size,
parallel.attn_dcp_rank,
interleave_size=self.page_size,
).int()
page_stride = self.page_size * parallel.attn_dcp_size
max_len = int(kv_lens_cpu.max().item()) if kv_lens_cpu.numel() else 0
block_tables = (
self.req_to_token[req_pool_indices, :max_len:page_stride] // page_stride
).to(torch.int32)
if is_spec:
block_tables = block_tables.repeat_interleave(
self.speculative_num_draft_tokens, dim=0
)
return local_kv_lens, block_tables
def update_verify_buffers_to_fill_after_draft(
self, spec_info: SpecInput, cuda_graph_bs: Optional[int]
):
@@ -474,6 +526,7 @@ class AscendAttnBackend(AttentionBackend):
forward_mode=forward_batch.forward_mode,
spec_info=forward_batch.spec_info,
out_cache_loc=forward_batch.out_cache_loc,
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
)
def init_forward_metadata(self, forward_batch: ForwardBatch):
@@ -591,6 +644,40 @@ class AscendAttnBackend(AttentionBackend):
device=self.device,
)
if get_parallel().dcp_enabled and not self.is_draft_worker:
# Draft workers retain allocator-global cache slots and use the non-DCP
# attention path, so only target workers need DCP-specific metadata.
self.forward_metadata.dcp_origin_out_cache_loc = (
forward_batch.origin_out_cache_loc
)
if forward_batch.forward_mode.is_target_verify():
(
self.forward_metadata.dcp_spec_seq_lens_cpu_int,
self.forward_metadata.dcp_spec_block_tables,
) = self._get_kv_lens_and_block_tables(
kv_lens_cpu=self.forward_metadata.seq_lens_cpu_int,
req_pool_indices=forward_batch.req_pool_indices,
is_spec=True,
)
self.forward_metadata.dcp_spec_seq_lens = (
self.forward_metadata.dcp_spec_seq_lens_cpu_int.to(
device=self.device, dtype=torch.int32
)
)
else:
(
self.forward_metadata.dcp_seq_lens_cpu_int,
self.forward_metadata.dcp_block_tables,
) = self._get_kv_lens_and_block_tables(
kv_lens_cpu=self.forward_metadata.seq_lens_cpu_int,
req_pool_indices=forward_batch.req_pool_indices,
)
self.forward_metadata.dcp_seq_lens = (
self.forward_metadata.dcp_seq_lens_cpu_int.to(
device=self.device, dtype=torch.int32
)
)
if (
self.use_mla
and forward_batch.forward_mode.is_extend()
@@ -639,6 +726,40 @@ class AscendAttnBackend(AttentionBackend):
device=self.device,
),
}
# DCP decode needs its own block_table (different stride: dcp_page_size).
if get_parallel().dcp_enabled and not self.is_draft_worker:
dcp_page_size = self.page_size * get_parallel().attn_dcp_size
max_dcp_seq_pages = (total_context_len + dcp_page_size - 1) // dcp_page_size
# The captured DSA indexer cache-write operation reads from this fixed
# buffer. Replay refreshes its allocator-global slot ids before execution.
self.graph_metadata["dcp_origin_out_cache_loc"] = torch.zeros(
max_num_tokens,
dtype=torch.int64,
device=self.device,
)
if self.speculative_num_draft_tokens is not None:
self.graph_metadata["dcp_spec_block_tables"] = torch.zeros(
(
max_bs * self.speculative_num_draft_tokens,
max_dcp_seq_pages,
),
dtype=torch.int32,
device=self.device,
)
self.graph_metadata["dcp_spec_seq_lens"] = torch.zeros(
max_bs * self.speculative_num_draft_tokens,
dtype=torch.int32,
device=self.device,
)
else:
self.graph_metadata["dcp_block_tables"] = torch.zeros(
(max_bs, max_dcp_seq_pages),
dtype=torch.int32,
device=self.device,
)
self.graph_metadata["dcp_seq_lens"] = torch.zeros(
max_bs, dtype=torch.int32, device=self.device
)
if self.is_hybrid_swa:
self.graph_metadata["block_tables_swa"] = torch.empty(
(max_bs, total_context_len // self.page_size),
@@ -694,6 +815,26 @@ class AscendAttnBackend(AttentionBackend):
torch.npu.current_stream(self.device), self.device
)
metadata.block_tables = self.graph_metadata["block_tables"][:bs, :]
if get_parallel().dcp_enabled:
if "dcp_origin_out_cache_loc" in self.graph_metadata:
metadata.dcp_origin_out_cache_loc = self.graph_metadata[
"dcp_origin_out_cache_loc"
][: out_cache_loc.shape[0]]
if not self.is_draft_worker and "dcp_block_tables" in self.graph_metadata:
metadata.dcp_block_tables = self.graph_metadata["dcp_block_tables"][:bs]
metadata.dcp_seq_lens = self.graph_metadata["dcp_seq_lens"][:bs]
if (
get_parallel().dcp_enabled
and "dcp_spec_block_tables" in self.graph_metadata
and forward_mode.is_target_verify()
):
num_spec_rows = bs * self.speculative_num_draft_tokens
metadata.dcp_spec_block_tables = self.graph_metadata[
"dcp_spec_block_tables"
][:num_spec_rows]
metadata.dcp_spec_seq_lens = self.graph_metadata["dcp_spec_seq_lens"][
:num_spec_rows
]
if self.is_hybrid_swa:
metadata.block_tables_swa = self.graph_metadata["block_tables_swa"][:bs, :]
metadata.swa_mask = self.graph_metadata["swa_mask"][:bs, :, :]
@@ -765,6 +906,7 @@ class AscendAttnBackend(AttentionBackend):
forward_mode: ForwardMode,
spec_info: Optional[SpecInput],
out_cache_loc: Optional[torch.Tensor] = None,
origin_out_cache_loc: Optional[torch.Tensor] = None,
):
"""Shared capture+replay body for the cuda-graph init path.
@@ -779,11 +921,17 @@ class AscendAttnBackend(AttentionBackend):
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc)
)
max_len = seq_lens_cpu[:bs].max().item()
# Compute the host-side KV lengths visible to this attention step. Target
# verify adds the draft block except for DFlash, whose seq_lens_cpu already
# includes it; speculative decode adds tokens through the current draft step.
attention_kv_lens_cpu = seq_lens_cpu[:bs].int()
if forward_mode.is_target_verify() and not _is_dflash_verify(spec_info):
max_len += self.speculative_num_draft_tokens
attention_kv_lens_cpu = (
attention_kv_lens_cpu + self.speculative_num_draft_tokens
)
elif forward_mode.is_decode_or_idle() and spec_info is not None:
max_len += self.speculative_step_id + 1
attention_kv_lens_cpu = attention_kv_lens_cpu + self.speculative_step_id + 1
max_len = attention_kv_lens_cpu.max().item()
max_seq_pages = (max_len + self.page_size - 1) // self.page_size
if self.is_hybrid_swa:
@@ -826,6 +974,58 @@ class AscendAttnBackend(AttentionBackend):
)
metadata.block_tables[:bs, max_seq_pages:].fill_(0)
if get_parallel().dcp_enabled and not self.is_draft_worker:
if "dcp_origin_out_cache_loc" in self.graph_metadata:
buffer = self.graph_metadata["dcp_origin_out_cache_loc"]
if origin_out_cache_loc is None:
buffer.zero_()
else:
num_tokens = origin_out_cache_loc.shape[0]
assert num_tokens <= buffer.shape[0], (
"NPU DSA+DCP origin_out_cache_loc exceeds its graph buffer: "
f"{num_tokens} > {buffer.shape[0]}"
)
buffer[:num_tokens].copy_(origin_out_cache_loc)
buffer[num_tokens:].zero_()
# DCP decode/speculative paths use rank-local KV lengths and a block
# table whose stride is page_size * dcp_world_size. Draft attention
# uses the ordinary full-KV metadata above instead.
if forward_mode.is_target_verify():
(
metadata.dcp_spec_seq_lens_cpu_int,
dcp_spec_block_tables,
) = self._get_kv_lens_and_block_tables(
kv_lens_cpu=attention_kv_lens_cpu,
req_pool_indices=req_pool_indices[:bs],
is_spec=True,
)
metadata.dcp_spec_seq_lens.copy_(
metadata.dcp_spec_seq_lens_cpu_int.to(
device=metadata.dcp_spec_seq_lens.device
)
)
dcp_pages = dcp_spec_block_tables.shape[1]
metadata.dcp_spec_block_tables[:, :dcp_pages].copy_(
dcp_spec_block_tables
)
metadata.dcp_spec_block_tables[:, dcp_pages:].fill_(0)
else:
(
metadata.dcp_seq_lens_cpu_int,
dcp_block_tables,
) = self._get_kv_lens_and_block_tables(
kv_lens_cpu=attention_kv_lens_cpu,
req_pool_indices=req_pool_indices[:bs],
)
metadata.dcp_seq_lens.copy_(
metadata.dcp_seq_lens_cpu_int.to(
device=metadata.dcp_seq_lens.device
)
)
dcp_pages = dcp_block_tables.shape[1]
metadata.dcp_block_tables[:bs, :dcp_pages].copy_(dcp_block_tables)
metadata.dcp_block_tables[:bs, dcp_pages:].fill_(0)
metadata.block_tables[bs:, :].fill_(0)
if forward_mode.is_target_verify():
@@ -1225,6 +1425,19 @@ class AscendAttnBackend(AttentionBackend):
if topk_indices is not None:
topk_indices = self._pad_topk_indices(topk_indices, q_nope.shape[0])
topk_indices = _expand_dsa_sparse_indices(topk_indices)
if get_parallel().dcp_enabled and not self.is_draft_worker:
return forward_dcp_sparse_attention(
q_nope=q_nope,
q_rope=q_pe,
k_nope=k_nope,
k_rope=k_pe,
topk_indices=topk_indices,
actual_seq_lengths_query=actual_seq_qlen,
forward_metadata=self.forward_metadata,
forward_batch=forward_batch,
speculative_num_draft_tokens=self.speculative_num_draft_tokens,
scaling=layer.scaling,
)
if self.kv_cache_dtype == torch.float8_e4m3fn:
assert q_nope.dtype == q_pe.dtype == torch.bfloat16
packed = k_nope.view(torch.float8_e4m3fn)
@@ -0,0 +1,84 @@
"""NPU DSA adaptation for token-sharded decode context parallelism."""
from __future__ import annotations
from typing import TYPE_CHECKING
import sgl_kernel_npu # noqa: F401 Registers torch.ops.sgl_kernel_npu.
import torch
if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
def forward_dcp_sparse_attention(
*,
q_nope: torch.Tensor,
q_rope: torch.Tensor,
k_nope: torch.Tensor,
k_rope: torch.Tensor,
topk_indices: torch.Tensor,
actual_seq_lengths_query: torch.Tensor,
forward_metadata,
forward_batch: ForwardBatch,
speculative_num_draft_tokens: int | None,
scaling: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Compute one rank's sparse partial attention and natural-log LSE."""
is_target_verify = forward_batch.forward_mode.is_target_verify()
if is_target_verify:
block_tables = forward_metadata.dcp_spec_block_tables
local_lens = forward_metadata.dcp_spec_seq_lens
else:
block_tables = forward_metadata.dcp_block_tables
local_lens = forward_metadata.dcp_seq_lens
assert block_tables is not None, (
"NPU DSA+DCP sparse attention requires rank-local paged-KV metadata"
)
if is_target_verify:
# DCP verify presents every speculative query as an independent
# single-query sequence. Its KV frontier and page-table row therefore
# stay expanded in request-major [bs * draft_token_num] order.
num_query_rows = q_nope.shape[0]
assert block_tables.shape[0] == num_query_rows, (
"NPU DSA+DCP verify block-table rows must match query rows: "
f"{block_tables.shape[0]} != {num_query_rows}"
)
assert local_lens.shape[0] == num_query_rows, (
"NPU DSA+DCP verify KV-length rows must match query rows: "
f"{local_lens.shape[0]} != {num_query_rows}"
)
actual_seq_lengths_query = torch.arange(
1,
num_query_rows + 1,
device=q_nope.device,
dtype=torch.int32,
)
else:
actual_seq_lengths_query = actual_seq_lengths_query.to(
device=q_nope.device, dtype=torch.int32
)
attn_out, softmax_max, softmax_sum = torch.ops.npu.sgl_sparse_flash_attention(
query=q_nope,
key=k_nope,
value=k_nope,
query_rope=q_rope,
key_rope=k_rope,
sparse_indices=topk_indices,
scale_value=scaling,
actual_seq_lengths_query=actual_seq_lengths_query,
actual_seq_lengths_kv=local_lens,
block_table=block_tables,
sparse_block_size=1,
layout_query="TND",
layout_kv="PA_BSND",
# Global top-k already enforces causal visibility. Local indices no
# longer carry enough information for the kernel's causal sparse mode.
sparse_mode=0,
attention_mode=2,
return_softmax_lse=True,
)
lse = softmax_max.float() + torch.log(softmax_sum.float())
return attn_out, lse.permute(1, 0, 2).reshape(lse.shape[1], -1)
@@ -4,6 +4,7 @@ import torch
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.environ import envs
from sglang.srt.layers.dcp.layout import localize_dcp_indices
from sglang.srt.mem_cache.memory_pool import (
MHATokenToKOnlyPool,
MHATokenToKVPool,
@@ -12,6 +13,7 @@ from sglang.srt.mem_cache.memory_pool import (
get_tensor_size_bytes,
unwrap_write_loc,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_bool_env_var
from sglang.srt.utils.common import is_npu
@@ -585,10 +587,13 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
device: str,
enable_memory_saver: bool,
index_head_dim: Optional[int] = None,
index_size: Optional[int] = None,
start_layer: Optional[int] = None,
end_layer: Optional[int] = None,
index_page_size: Optional[int] = None,
indexer_layer_ids: Optional[Sequence[int]] = None,
kv_cache_dim: Optional[int] = None,
is_draft_worker: bool = False,
):
# MLAPO historically owned NZ writes. Keep the allocation unchanged and
# write into the NZ-addressed view below so ordinary MLA (including
@@ -613,7 +618,6 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
)
if self.enable_sparsity_driven_kv_offload and self.index_head_dim is None:
raise ValueError("Sparsity-driven KV offload requires an index KV cache.")
if index_head_dim is None:
self.indexer_layer_ids = ()
elif indexer_layer_ids is None:
@@ -649,6 +653,19 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
self.kr_cache_dim = 0 if self.dsa_kv_cache_store_fp8 else qk_rope_head_dim
self.index_k_scale_buffer = None
self.indexer_hadamard_128 = None
self.index_page_size = page_size if index_page_size is None else index_page_size
self.index_size = size if index_size is None else index_size
parallel = get_parallel()
self.dcp_size = parallel.attn_dcp_size
self.dcp_rank = parallel.attn_dcp_rank
global_page_padding = self.dcp_size if self.dcp_size > 1 else 1
kv_page_padding = global_page_padding if is_draft_worker else 1
index_page_padding = global_page_padding if index_head_dim is not None else 1
if kv_page_padding < 1 or index_page_padding < 1:
raise ValueError("NPU MLA page padding must be positive")
self.kv_page_padding = kv_page_padding
self.index_page_padding = index_page_padding
self.is_draft_worker = is_draft_worker
self.custom_mem_pool = None
@@ -661,7 +678,7 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
self.k_buffer = torch.zeros(
(
layer_num,
self.size // self.page_size + 1,
self.size // self.page_size + self.kv_page_padding,
self.page_size,
1,
self.kv_cache_dim,
@@ -672,7 +689,7 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
self.v_buffer = torch.zeros(
(
layer_num,
self.size // self.page_size + 1,
self.size // self.page_size + self.kv_page_padding,
self.page_size,
1,
self.kr_cache_dim,
@@ -689,8 +706,9 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
self.index_k_buffer = torch.zeros(
(
self.num_indexer_layers,
self.size // self.page_size + 1,
self.page_size,
self.index_size // self.index_page_size
+ self.index_page_padding,
self.index_page_size,
1,
self.index_head_dim,
),
@@ -713,6 +731,17 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
self._finalize_allocation_log(size)
def _copy_indices_for_buffer(self, indices, uses_global_slots):
if uses_global_slots or self.dcp_size <= 1:
return indices
local_indices = localize_dcp_indices(
indices,
self.dcp_size,
self.dcp_rank,
self.page_size,
)
return local_indices[local_indices >= 0]
def get_kv_size_bytes(self):
kv_size_bytes = 0
@@ -805,25 +834,37 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
scale.view(-1, 1),
)
def _get_disagg_buffer_entries(self):
"""Return (buffer, uses_global_slots) entries in PD transfer order."""
self._raise_if_native_kv_cache_disabled()
global_kv = self.is_draft_worker
entries = [(buffer, global_kv) for buffer in self.k_buffer]
if not getattr(self, "dsa_kv_cache_store_fp8", False):
entries += [(buffer, global_kv) for buffer in self.v_buffer]
if self.index_head_dim is not None:
entries += [(buffer, True) for buffer in self.index_k_buffer]
if self.index_k_scale_buffer is not None:
entries += [(buffer, True) for buffer in self.index_k_scale_buffer]
return entries
# for disagg
def get_contiguous_buf_infos(self):
self._raise_if_native_kv_cache_disabled()
# MLA has only one kv_buffer, so only the information of this buffer needs to be returned.
kv_data_ptrs = [self.k_buffer[i].data_ptr() for i in range(self.layer_num)]
kv_data_lens = [self.k_buffer[i].nbytes for i in range(self.layer_num)]
kv_item_lens = [self.k_buffer[i][0].nbytes for i in range(self.layer_num)]
# When DSA KV cache is packed into the FP8 k_buffer, the v_buffer is
# intentionally empty (kr_cache_dim == 0). Its data_ptr() is null
if not getattr(self, "dsa_kv_cache_store_fp8", False):
kv_data_ptrs += [self.v_buffer[i].data_ptr() for i in range(self.layer_num)]
kv_data_lens += [self.v_buffer[i].nbytes for i in range(self.layer_num)]
kv_item_lens += [self.v_buffer[i][0].nbytes for i in range(self.layer_num)]
if self.index_head_dim is not None:
ptrs, lens, item_lens = self.get_state_buf_infos()
kv_data_ptrs += ptrs
kv_data_lens += lens
kv_item_lens += item_lens
return kv_data_ptrs, kv_data_lens, kv_item_lens
entries = self._get_disagg_buffer_entries()
return (
[buffer.data_ptr() for buffer, _ in entries],
[buffer.nbytes for buffer, _ in entries],
[
buffer[0].nbytes * (self.dcp_size if uses_global_slots else 1)
for buffer, uses_global_slots in entries
],
)
def get_dcp_remote_decode_layout(self) -> list[bool]:
"""Whether each PD entry uses allocator-global slots on decode."""
return [
uses_global_slots
for _, uses_global_slots in self._get_disagg_buffer_entries()
]
def get_kv_layer_ids(self):
return (
@@ -956,17 +997,25 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
index_k.view(-1, 1, self.index_head_dim),
)
def _chunk_copy_npu_to_cpu(self, buf_of_layers, indices):
def _chunk_copy_npu_to_cpu(
self, buf_of_layers, indices, uses_global_slots_per_layer
):
chunk_size = self.cpu_offloading_chunk_size
out = []
for tensors_per_layer in buf_of_layers: # [k_buf, v_buf, ik_buf/None]
for tensors_per_layer, uses_global_slots in zip(
buf_of_layers, uses_global_slots_per_layer, strict=True
): # [k_buf, v_buf, ik_buf/None]
layer_chunks = []
for i in range(0, len(indices), chunk_size):
ci = indices[i : i + chunk_size]
layer_chunks.append(
[
t[ci].to("cpu", non_blocking=True)
for t in tensors_per_layer
t[self._copy_indices_for_buffer(ci, uses_global)].to(
"cpu", non_blocking=True
)
for t, uses_global in zip(
tensors_per_layer, uses_global_slots, strict=True
)
if t is not None
]
)
@@ -998,8 +1047,18 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
buf_of_layers = [
self._get_cpu_offload_layer_buffers(i) for i in range(self.layer_num)
]
uses_global_slots_per_layer = []
for buffers in buf_of_layers:
# MLA K/V is rank-local under DCP. The replicated
# indexer buffers retain allocator-global slot identities.
uses_global_slots_per_layer.append(
[self.is_draft_worker, self.is_draft_worker]
+ [True] * (len(buffers) - 2)
)
kv_cache_cpu = self._chunk_copy_npu_to_cpu(buf_of_layers, indices)
kv_cache_cpu = self._chunk_copy_npu_to_cpu(
buf_of_layers, indices, uses_global_slots_per_layer
)
torch.npu.synchronize()
return kv_cache_cpu
@@ -1013,7 +1072,20 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
for i in range(0, len(indices), chunk_size):
chunk_indices = indices[i : i + chunk_size]
chunk = kv_cache_cpu[local_layer_id][i // chunk_size]
for buffer, cpu in zip(buffers, chunk, strict=True):
assert cpu.shape[0] == len(chunk_indices)
buffer[chunk_indices] = cpu.to(buffer.device, non_blocking=True)
cpu_index = 0
for buffer, uses_global_slots in zip(
buffers,
[self.is_draft_worker, self.is_draft_worker]
+ [True] * (len(buffers) - 2),
strict=True,
):
if buffer is None:
continue
cpu = chunk[cpu_index]
cpu_index += 1
target_indices = self._copy_indices_for_buffer(
chunk_indices, uses_global_slots
)
assert cpu.shape[0] == len(target_indices)
buffer[target_indices] = cpu.to(buffer.device, non_blocking=True)
torch.npu.synchronize()
@@ -4,6 +4,10 @@ import torch
import torch_npu
from sgl_kernel_npu.norm.fused_split_qk_norm import fused_split_qk_norm
import sglang.srt.layers.dcp.comm as dcp_comm
import sglang.srt.layers.dcp.layout as dcp_layout
import sglang.srt.model_executor.forward_context as forward_context
import sglang.srt.runtime_context as runtime_context
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.npu.attention.mla_preprocess import (
NPUFusedMLAPreprocess,
@@ -28,6 +32,15 @@ _use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
_is_npu_arch35 = is_npu_arch35()
def _use_dsa_dcp_partial_attention(forward_batch: "ForwardBatch") -> bool:
return (
runtime_context.get_parallel().dcp_enabled
and not forward_context.get_attn_backend().is_draft_worker
and not dsa_use_prefill_cp(forward_batch)
and not forward_batch.forward_mode.is_idle()
)
# region MHA
def forward_mha_prepare_npu(
m: "DeepseekV2AttentionMLA",
@@ -488,9 +501,23 @@ def forward_dsa_prepare_npu(
input_on_attn_tp_slices,
dynamic_scale,
)
# DSA layers that skip the indexer reuse ``prev_topk_indices``. Remap
# only when a fresh global top-k is produced so shared-index layers do
# not repeat the same DCP partitioning work.
if _use_dsa_dcp_partial_attention(forward_batch):
parallel = runtime_context.get_parallel()
topk_indices = dcp_layout.remap_dcp_sparse_indices(
topk_indices,
parallel.attn_dcp_size,
parallel.attn_dcp_rank,
interleave_size=forward_context.get_attn_backend().page_size,
)
else:
topk_indices = prev_topk_indices
if _use_dsa_dcp_partial_attention(forward_batch):
q_nope_out, q_pe = dcp_comm.all_gather_q_for_mla_decode(q_nope_out, q_pe)
topk_indices = maybe_capture_indexer_topk(m.layer_id, topk_indices)
return (
@@ -522,7 +549,9 @@ def forward_dsa_core_npu(
# a trailing arg. None everywhere else.
gate: Optional[torch.Tensor] = None,
) -> torch.Tensor:
attn_output = m.attn_mqa(
use_dcp = _use_dsa_dcp_partial_attention(forward_batch)
attn = m.attn_mqa_for_dcp_decode if use_dcp else m.attn_mqa
attn_output = attn(
q_nope_out.contiguous(),
k_nope.contiguous(),
k_nope.contiguous(),
@@ -532,6 +561,16 @@ def forward_dsa_core_npu(
k_rope=k_pe.contiguous(),
topk_indices=topk_indices,
)
if use_dcp:
attn_output, lse = attn_output
attn_output = attn_output.view(
-1,
m.num_local_heads * runtime_context.get_parallel().attn_dcp_size,
m.kv_lora_rank,
)
attn_output = dcp_comm.cp_lse_ag_out_rs_mla_npu(
attn_output, lse, runtime_context.get_parallel().dcp_group
)
attn_output = attn_output.view(-1, m.num_local_heads, m.kv_lora_rank)
if _is_npu_arch35 or (
@@ -12,6 +12,7 @@ from sglang.srt.model_executor.forward_context import (
get_attn_backend,
get_token_to_kv_pool,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_npu
if is_npu():
@@ -207,16 +208,31 @@ class DSANPUIndexerMixin:
torch.npu.current_stream(),
)
indexer_cache_loc = forward_batch.out_cache_loc
parallel = get_parallel()
if (
parallel.dcp_enabled
and parallel.attn_dcp_size > 1
and not get_attn_backend().is_draft_worker
):
indexer_cache_loc = (
get_attn_backend().forward_metadata.dcp_origin_out_cache_loc
)
assert indexer_cache_loc is not None, (
"NPU DSA+DCP requires allocator-global origin_out_cache_loc metadata"
)
assert indexer_cache_loc.shape[0] == positions.shape[0], (
"NPU DSA+DCP origin_out_cache_loc metadata has an incompatible "
f"length: {indexer_cache_loc.shape[0]} != {positions.shape[0]}"
)
pool = get_token_to_kv_pool()
use_quant_indexer = pool.index_k_scale_buffer is not None
if use_quant_indexer:
k, k_scale = _quantize_npu_indexer_activation(
k, pool.indexer_hadamard_128, pool.dtype
)
pool.set_index_k_scale_buffer(
layer_id, forward_batch.out_cache_loc, k_scale
)
pool.set_index_k_buffer(layer_id, forward_batch.out_cache_loc, k)
pool.set_index_k_scale_buffer(layer_id, indexer_cache_loc, k_scale)
pool.set_index_k_buffer(layer_id, indexer_cache_loc, k)
if is_prefill:
if (
self.dsa_enable_prefill_cp
+35
View File
@@ -123,6 +123,41 @@ def cp_lse_ag_out_rs_mla(
return out.to(cp_attn_out.dtype)
def cp_lse_ag_out_rs_mla_npu(
cp_attn_out: torch.Tensor,
cp_attn_lse: torch.Tensor,
cp_group: GroupCoordinator,
) -> torch.Tensor:
"""Merge NPU DCP partial outputs and return the local head slice."""
if cp_group.world_size == 1:
return cp_attn_out
import torch_npu
batch_size, total_heads, head_dim = cp_attn_out.shape
world_size = cp_group.world_size
local_heads = total_heads // world_size
packed = torch.cat([cp_attn_out.float(), cp_attn_lse.float().unsqueeze(-1)], dim=-1)
packed = packed.permute(1, 2, 0).contiguous()
gathered = torch.empty_like(packed)
cp_group.all_to_all_single(gathered, packed)
# all_to_all_single splits the leading head dimension. After the exchange,
# the heads are grouped by source rank inside every token. Move that source
# rank in front before flattening tokens and local heads for the update op.
gathered = gathered.permute(2, 0, 1).contiguous()
gathered = (
gathered.view(batch_size, world_size, local_heads, head_dim + 1)
.permute(1, 0, 2, 3)
.contiguous()
.view(world_size, batch_size * local_heads, head_dim + 1)
)
out_flat, lse_flat = torch.split(gathered, [head_dim, 1], dim=-1)
merged, _ = torch_npu.npu_attention_update(
lse_flat.squeeze(-1).unbind(0), out_flat.unbind(0), 0
)
return merged.view(batch_size, local_heads, head_dim).to(cp_attn_out.dtype)
def _all_gather_dcp_kv_cache(kv_a: torch.Tensor):
parallel = get_parallel()
dcp_world_size = parallel.dcp_size
+112 -8
View File
@@ -25,20 +25,53 @@ def get_dcp_lens(
dcp_size: int,
dcp_rank: int,
start: torch.Tensor | None = None,
interleave_size: int = 1,
) -> torch.Tensor:
"""Per-rank visible KV length under the owner rule pos % dcp_size == dcp_rank.
"""Return the KV length owned by one DCP rank.
Superset implementation (PR #25090): supports both start=None and a per-request
`start` offset. update_local_kv_lens_for_dcp is the start=None special case.
Consecutive runs of ``interleave_size`` positions rotate across ranks. The
default value preserves the token-interleaved layout used by the generic
CUDA/ROCm DCP paths; NPU DSA passes its physical KV page size.
"""
if dcp_size == 1:
return lens
if start is None:
return lens // dcp_size + (dcp_rank < lens % dcp_size)
if interleave_size < 1:
raise ValueError(f"interleave_size must be positive, got {interleave_size}")
first = start + torch.remainder(dcp_rank - start, dcp_size)
remaining = start + lens - first
return torch.clamp((remaining + dcp_size - 1) // dcp_size, min=0)
cycle_size = dcp_size * interleave_size
def _count_before(end: torch.Tensor) -> torch.Tensor:
full_cycles = end // cycle_size
remainder = end % cycle_size
rank_remainder = torch.clamp(
remainder - dcp_rank * interleave_size,
min=0,
max=interleave_size,
)
return full_cycles * interleave_size + rank_remainder
if start is None:
return _count_before(lens)
return _count_before(start + lens) - _count_before(start)
def localize_dcp_indices(
indices: torch.Tensor,
dcp_size: int,
dcp_rank: int,
interleave_size: int = 1,
) -> torch.Tensor:
"""Map global DCP indices to one rank, using ``-1`` for non-local rows."""
if dcp_size == 1:
return indices
if interleave_size < 1:
raise ValueError(f"interleave_size must be positive, got {interleave_size}")
interleave_block = torch.div(indices, interleave_size, rounding_mode="floor")
is_local = (indices >= 0) & (interleave_block % dcp_size == dcp_rank)
local_block = torch.div(interleave_block, dcp_size, rounding_mode="floor")
local_indices = local_block * interleave_size + indices % interleave_size
return torch.where(is_local, local_indices, torch.full_like(indices, -1))
def maybe_dcp_kernel_indices(
@@ -84,6 +117,77 @@ def filter_dcp_local_chunk_kv_indices(
return torch.cat(parts)
def remap_dcp_sparse_indices(
topk_indices: torch.Tensor,
dcp_size: int,
dcp_rank: int,
interleave_size: int = 1,
) -> torch.Tensor:
"""Map global sparse token indices to one rank's compact DCP KV layout.
Keep indices owned by this rank, convert them to local KV positions, and
stably move them before ``-1`` padding while preserving their score order.
"""
if dcp_size == 1:
return topk_indices
if interleave_size < 1:
raise ValueError(f"interleave_size must be positive, got {interleave_size}")
# Float32 is faster than integer division/remainder on Ascend for this hot
# path. Keep the math equivalent to localize_dcp_indices above.
topk_indices_fp32 = topk_indices.to(torch.float32)
interleave_blocks = torch.floor(topk_indices_fp32 / interleave_size)
local_owner_mask = (topk_indices_fp32 >= 0) & (
torch.remainder(interleave_blocks, dcp_size) == dcp_rank
)
local_offsets = torch.remainder(topk_indices_fp32, interleave_size)
local_indices = (
torch.floor(interleave_blocks / dcp_size) * interleave_size + local_offsets
)
remapped_indices = torch.where(
local_owner_mask,
local_indices,
torch.full_like(topk_indices_fp32, -1.0),
).to(topk_indices.dtype)
# Move valid entries before padding without changing their top-k order.
topk_count = topk_indices.shape[-1]
original_order = torch.arange(
topk_count, dtype=torch.float32, device=topk_indices.device
).expand_as(topk_indices_fp32)
pack_keys = original_order + (~local_owner_mask).to(torch.float32) * topk_count
pack_order = torch.argsort(pack_keys, dim=-1).to(torch.int64)
return torch.gather(remapped_indices, dim=-1, index=pack_order)
def get_dcp_chain_spec_lens(
total_kv_lens: torch.Tensor,
tokens_per_req: int,
dcp_size: int,
dcp_rank: int,
interleave_size: int = 1,
) -> torch.Tensor:
"""Return request-major local KV frontiers for a speculative chain."""
if tokens_per_req < 1:
raise ValueError(f"tokens_per_req must be >= 1, got {tokens_per_req}")
total_kv_lens = total_kv_lens.int()
steps = torch.arange(
1, tokens_per_req + 1, dtype=total_kv_lens.dtype, device=total_kv_lens.device
)
global_query_lens = total_kv_lens.unsqueeze(1) - tokens_per_req + steps
global_query_lens = torch.where(
total_kv_lens.unsqueeze(1) >= tokens_per_req,
global_query_lens,
torch.zeros_like(global_query_lens),
)
return get_dcp_lens(
global_query_lens.reshape(-1),
dcp_size,
dcp_rank,
interleave_size=interleave_size,
).int()
def update_local_kv_lens_for_dcp(kv_len_arr):
"""In-place per-rank KV length: the start=0 case of get_dcp_lens.
@@ -1555,6 +1555,12 @@ class KVCacheConfigurator:
)
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
# Indexer uses the allocator-global slot space on DCP target workers.
dcp_size = get_parallel().attn_dcp_size
if self.is_draft_worker:
index_size = max_total_num_tokens
else:
index_size = max_total_num_tokens * dcp_size
is_arch35 = is_npu_arch35()
use_compact_indexer_layout = (
is_dsa_model
@@ -1576,11 +1582,13 @@ class KVCacheConfigurator:
)
token_to_kv_pool = NPUMLATokenToKVPool(
max_total_num_tokens,
page_size=self.pool_page_size,
page_size=get_schedule().page_size,
dtype=self.kv_cache_dtype,
kv_lora_rank=self.model_config.kv_lora_rank,
qk_rope_head_dim=self.model_config.qk_rope_head_dim,
index_head_dim=(self.model_config.index_head_dim if is_dsa_model else None),
index_size=index_size,
index_page_size=get_schedule().page_size,
indexer_layer_ids=indexer_layer_ids,
kv_cache_dim=(
calculate_mla_kv_cache_dim(
@@ -1589,6 +1597,7 @@ class KVCacheConfigurator:
if use_dsa_fp8_kv_cache_storage
else None
),
is_draft_worker=self.is_draft_worker,
layer_num=self.layer_info.num_effective_layers,
device=self.device,
enable_memory_saver=get_exec().features.enable_memory_saver,
@@ -2057,8 +2066,13 @@ class KVCacheConfigurator:
)
token_to_kv_pool_allocator = NPUPagedTokenToKVPoolAllocator(
sizes.max_total_num_tokens,
page_size=get_schedule().page_size,
# DCP allocation is in the global virtual loc space.
# The target attention path localizes these locs when
# building rank-local metadata; draft/indexer paths
# consume the allocator locs directly.
sizes.max_total_num_tokens * get_parallel().attn_dcp_size,
page_size=get_schedule().page_size
* get_parallel().attn_dcp_size,
dtype=self.kv_cache_dtype,
device=self.device,
kvcache=token_to_kv_pool,
@@ -43,6 +43,7 @@ from sglang.srt.environ import envs
from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
compute_req_all_ids_info,
)
from sglang.srt.layers.dcp.layout import localize_dcp_indices
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
dp_slot_in,
@@ -191,6 +192,21 @@ def _elastic_should_preserve_local_token_counts(
return uneven_token_count
def _localize_npu_dcp_out_cache_loc(
out_cache_loc: torch.Tensor,
*,
interleave_size: int,
) -> torch.Tensor:
"""Map allocator-global NPU DCP slots to this target rank."""
parallel = get_parallel()
return localize_dcp_indices(
out_cache_loc,
parallel.dcp_size,
parallel.dcp_rank,
interleave_size,
)
class ForwardMode(IntEnum):
# Extend a sequence. The KV cache of the beginning part of the sequence is already computed (e.g., system prompt).
# It is also called "prefill" in common terminology.
@@ -505,6 +521,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# The sum of all sequence lengths
seq_lens_sum: int
# Allocator-global output slots before NPU DCP localizes ``out_cache_loc``.
# DSA's replicated indexer cache uses the global slot identity.
origin_out_cache_loc: Optional[torch.Tensor] = None
# === Borrowed from ScheduleBatch: GPU tensors (cross-stream; clone targets for stream isolation) ===
# FIXME(lsyin): these are currently aliased by reference from ScheduleBatch. Once
# they are cloned/relayed into FB-owned copies at the boundary, move them out of
@@ -1025,6 +1045,15 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
spec_info=batch.spec_info,
)
# ScheduleBatch and req_to_token keep allocator-global slot identities.
# Preserve that view before exposing rank-local NPU DCP write slots.
if _is_npu and get_parallel().dcp_enabled and not model_runner.is_draft_worker:
ret.origin_out_cache_loc = ret.out_cache_loc
if ret.out_cache_loc is not None:
ret.out_cache_loc = _localize_npu_dcp_out_cache_loc(
ret.out_cache_loc,
interleave_size=model_runner.page_size,
)
ret._maybe_init_non_generation_fields(batch)
device = model_runner.device
@@ -1820,6 +1849,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
)
self.out_cache_loc = self._pad_tensor_to_size(self.out_cache_loc, num_tokens)
if self.origin_out_cache_loc is not None:
self.origin_out_cache_loc = self._pad_tensor_to_size(
self.origin_out_cache_loc, num_tokens
)
if self.encoder_lens is not None:
self.encoder_lens = self._pad_tensor_to_size(self.encoder_lens, bs)
self.positions = self._pad_tensor_to_size(self.positions, num_tokens)
@@ -2086,6 +2119,7 @@ def build_inner_fb_view(
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
# A caller may hand in another view that does not carry this field.
out_cache_loc_virtual=getattr(forward_batch, "out_cache_loc_virtual", None),
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
spec_info=forward_batch.spec_info,
)
@@ -192,6 +192,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
dcp_size = get_parallel().attn_dcp_size
# Determine effective number of layers for KV cache
if mambaish := mambaish_config(kvc.model_config):
effective_layer_ids = [
@@ -233,6 +234,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
kvc=kvc,
num_layers=num_layers,
)
if _is_npu and dcp_size > 1:
target_indexer_size *= dcp_size
target_kv_size = self._cell_size - target_indexer_size
from sglang.srt.layers.cp.utils import (
get_glm_dsa_layer_split_effective_num_layers,
@@ -249,6 +252,11 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
num_layers=draft_num_layers,
allocate_all_layers=True,
)
# The draft pool is replicated and consumes the widened
# allocator-global slot space on NPU DCP.
if _is_npu and dcp_size > 1:
draft_kv_size *= dcp_size
draft_indexer_size *= dcp_size
self._cell_size += draft_kv_size + draft_indexer_size
else:
self._cell_size = int(
@@ -337,10 +345,13 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
# Add indexer KV cache overhead for DSA models (DeepSeek V3.2)
if is_deepseek_dsa(model_config.hf_config):
cell_size += self._compute_dsa_indexer_cell_size(
indexer_cell_size = self._compute_dsa_indexer_cell_size(
kvc=kvc,
num_layers=num_layers,
)
if _is_npu and not kvc.is_draft_worker and dcp_size > 1:
indexer_cell_size *= dcp_size
cell_size += indexer_cell_size
elif is_minimax_sparse(model_config.hf_config):
from sglang.srt.server_args import m3_fp8_attn_gemm_enabled
@@ -198,6 +198,7 @@ def build_replay_fb_view(
encoder_lens=buffers.encoder_lens[:bs] if is_encoder_decoder else None,
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
max_seq_len_override=forward_batch.max_seq_len_override,
# The mamba-track registry slot (VIRTUAL ids) is the v2p translate SOURCE
+2
View File
@@ -3420,6 +3420,8 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
kv_cache_device,
create_chunked_prefix_cache_kv_indices_fn,
):
if _is_npu:
return None
return prepare_decode_context_parallel_metadata(
seq_lens=seq_lens,
extend_prefix_lens=extend_prefix_lens,