From ec0006673abe8c2a89162d428ef9b68b67393797 Mon Sep 17 00:00:00 2001 From: heziiop Date: Wed, 30 Sep 2026 15:22:16 +0800 Subject: [PATCH] [NPU] Add decode context parallel support for dsa models (#37787) Co-authored-by: sglang-npu-bot --- python/sglang/srt/arg_groups/model_hook.py | 7 + .../srt/arg_groups/pd_disaggregation_hook.py | 3 +- .../sglang/srt/disaggregation/ascend/conn.py | 218 +++++++++ python/sglang/srt/disaggregation/prefill.py | 13 + python/sglang/srt/disaggregation/utils.py | 2 +- .../sglang/srt/distributed/parallel_state.py | 8 +- .../npu/attention/ascend_backend.py | 219 ++++++++- .../hardware_backend/npu/attention/dsa_dcp.py | 84 ++++ .../hardware_backend/npu/memory_pool_npu.py | 132 ++++-- .../modules/deepseek_v2_attention_mla_npu.py | 41 +- .../layers/attention/dsa/dsa_npu_indexer.py | 24 +- python/sglang/srt/layers/dcp/comm.py | 35 ++ python/sglang/srt/layers/dcp/layout.py | 120 ++++- .../srt/mem_cache/kv_cache_configurator.py | 20 +- .../srt/model_executor/forward_batch_info.py | 34 ++ .../srt/model_executor/pool_configurator.py | 13 +- .../runner/decode_cuda_graph_runner.py | 1 + python/sglang/srt/models/deepseek_v2.py | 2 + .../disaggregation/test_ascend_page_dcp.py | 201 ++++++++ .../unit/npu/attention/test_npu_dcp.py | 446 ++++++++++++++++++ .../unit/server_args/test_server_args.py | 6 +- 21 files changed, 1572 insertions(+), 57 deletions(-) create mode 100644 python/sglang/srt/hardware_backend/npu/attention/dsa_dcp.py create mode 100644 test/registered/unit/disaggregation/test_ascend_page_dcp.py create mode 100644 test/registered/unit/npu/attention/test_npu_dcp.py diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index edfafe900d..da31f4a1d2 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -164,6 +164,13 @@ def handle_model_specific_adjustments(server_args: Any): hf_config = model_config.hf_config model_arch = hf_config.architectures[0] + if get_platform().is_npu and cfg.dcp_size > 1 and not is_deepseek_dsa(hf_config): + raise ValueError( + "NPU decode context parallelism is currently implemented only for " + "DeepSeek DSA models; got " + f"{model_arch}. Set --decode-context-parallel-size=1 or use a DSA model." + ) + if model_arch == "InternS2MobiusForConditionalGeneration": unsupported = [] if cfg.pp_size != 1: diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index 4124b23c5d..b15044b252 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -74,11 +74,12 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: if cfg.disaggregation_transfer_backend not in ( "mooncake", "nixl", + "ascend", "fake", ): raise ValueError( "PD decode DCP requires --disaggregation-transfer-backend " - "mooncake, nixl, or fake for synthetic benchmarking, got " + "mooncake, nixl, ascend, or fake for synthetic benchmarking, got " f"{cfg.disaggregation_transfer_backend!r}." ) diff --git a/python/sglang/srt/disaggregation/ascend/conn.py b/python/sglang/srt/disaggregation/ascend/conn.py index c05cbdeb13..d6afd5f8a5 100644 --- a/python/sglang/srt/disaggregation/ascend/conn.py +++ b/python/sglang/srt/disaggregation/ascend/conn.py @@ -34,7 +34,81 @@ class AscendStateType(str, enum.Enum): _DSV4_KVCACHE_STATE_TYPES = tuple(AscendStateType) +def _build_page_interleaved_dcp_plan( + src_page_indices: npt.NDArray[np.int32], + dst_page_indices: npt.NDArray[np.int32], + *, + page_size: int, + dcp_size: int, + dcp_rank: int, + src_page_offset: int, + decode_prefix_len: int, + num_kv_tokens: int, +) -> Tuple[npt.NDArray[np.int64], ...]: + """Map physical prefill pages to local/global decode page slots.""" + if not 0 <= dcp_rank < dcp_size: + raise ValueError(f"Invalid DCP rank {dcp_rank} for size {dcp_size}") + virtual_page_size = page_size * dcp_size + if decode_prefix_len % virtual_page_size: + raise ValueError( + "Ascend PD DCP requires decode_prefix_len to align to the virtual " + f"page size ({virtual_page_size}), got {decode_prefix_len}" + ) + if src_page_offset < 0 or num_kv_tokens < 0: + raise ValueError( + "Ascend PD DCP page offset and token count must be nonnegative" + ) + + src_pages = np.asarray(src_page_indices, dtype=np.int64) + dst_pages = np.asarray(dst_page_indices, dtype=np.int64) + max_src_pages = (num_kv_tokens + page_size - 1) // page_size + if src_pages.size > max_src_pages: + raise ValueError( + "Ascend PD DCP source page count exceeds the token count: " + f"pages={src_pages.size}, tokens={num_kv_tokens}, page_size={page_size}" + ) + if src_pages.size == 0: + empty = np.empty((0,), dtype=np.int64) + return empty, empty.copy(), empty.copy(), empty.copy() + + # CP may assign this sender only a contiguous subset of the chunk pages; + # index_slice.start still carries that subset's suffix-relative offset. + relative_pages = src_page_offset + np.arange(src_pages.size, dtype=np.int64) + dst_positions = relative_pages // dcp_size + if dst_positions[-1] >= dst_pages.size: + raise ValueError( + "Ascend PD DCP destination does not contain enough virtual pages: " + f"required={dst_positions[-1] + 1}, available={dst_pages.size}" + ) + dst_super_pages = dst_pages[dst_positions] + owners = (decode_prefix_len // page_size + relative_pages) % dcp_size + local = owners == dcp_rank + + return ( + src_pages[local], + dst_super_pages[local], + src_pages, + dst_super_pages * dcp_size + owners, + ) + + class AscendKVManager(MooncakeKVManager): + def __init__( + self, + args, + disaggregation_mode, + server_args, + is_mla_backend: Optional[bool] = False, + dcp_remote_decode_layout=None, + ): + self._dcp_remote_decode_layout = ( + None if dcp_remote_decode_layout is None else list(dcp_remote_decode_layout) + ) + super().__init__(args, disaggregation_mode, server_args, is_mla_backend) + + def _is_npu_dsa_layout(self) -> bool: + return getattr(self.kv_args, "kv_buf_groups", 1) == 3 + def _requires_exact_state_index_match(self, st: StateType) -> bool: return ( super()._requires_exact_state_index_match(st) @@ -67,6 +141,47 @@ class AscendKVManager(MooncakeKVManager): if ptrs: self.engine.batch_register(ptrs, lens) + def requires_dcp_relayout(self, dst_dcp_size: int, dst_dcp_rank: int) -> bool: + if self._is_npu_dsa_layout() and self.dcp_size != dst_dcp_size: + if self.dcp_size == 1 and dst_dcp_size > 1: + return True + raise RuntimeError( + "NPU DSA PD supports prefill DCP 1 -> decode DCP N only, got " + f"prefill={self.dcp_size}, decode={dst_dcp_size}" + ) + return super().requires_dcp_relayout(dst_dcp_size, dst_dcp_rank) + + def _get_dcp_remote_decode_layout(self) -> List[bool]: + layout = self._dcp_remote_decode_layout + if layout is None or len(layout) != len(self.kv_args.kv_data_ptrs): + raise RuntimeError( + "Ascend PD DCP layout does not match its source KV entries" + ) + return layout + + def prepare_dcp_token_item_lens( + self, dst_page_item_lens: List[Optional[int]], dst_dcp_size: int + ) -> List[int]: + if not self._is_npu_dsa_layout(): + return super().prepare_dcp_token_item_lens(dst_page_item_lens, dst_dcp_size) + self._get_dcp_remote_decode_layout() + page_size = self.kv_args.page_size + token_item_lens = [] + for entry, item_len in enumerate(self.kv_args.kv_item_lens): + token_item_len, remainder = divmod(item_len, page_size) + if remainder: + raise RuntimeError( + f"Ascend PD DCP source entry {entry} is not page aligned" + ) + token_item_lens.append(token_item_len) + return token_item_lens + + def _init_dcp_pack_buffers_once( + self, dcp_size: int, *, include_draft: bool = False + ) -> None: + # Page-interleaved NPU DCP transfers whole pages directly. + self._dcp_pack_buffers = [] + def get_mla_kv_ptrs_with_pp( self, src_kv_ptrs: List[int], dst_kv_ptrs: List[int], state_type=None ) -> Tuple[List[int], List[int], int]: @@ -289,6 +404,109 @@ class AscendKVManager(MooncakeKVManager): return 0 + def send_kvcache_dcp( + self, + mooncake_session_id: str, + prefill_kv_indices: npt.NDArray[np.int32], + dst_kv_ptrs: list[int], + dst_kv_indices: npt.NDArray[np.int32], + *, + dcp_token_item_lens: List[int], + dst_dcp_size: int, + dst_dcp_rank: int, + src_page_offset: int, + decode_prefix_len: int, + num_kv_tokens: int, + executor: concurrent.futures.ThreadPoolExecutor, + dst_layer_ids: List[int], + pack_buffer=None, + dst_kv_item_lens: Optional[List[int]] = None, + dst_tp_rank: int = 0, + dst_attn_tp_size: Optional[int] = None, + ) -> int: + if not self._is_npu_dsa_layout(): + return super().send_kvcache_dcp( + mooncake_session_id, + prefill_kv_indices, + dst_kv_ptrs, + dst_kv_indices, + dcp_token_item_lens=dcp_token_item_lens, + dst_dcp_size=dst_dcp_size, + dst_dcp_rank=dst_dcp_rank, + src_page_offset=src_page_offset, + decode_prefix_len=decode_prefix_len, + num_kv_tokens=num_kv_tokens, + executor=executor, + dst_layer_ids=dst_layer_ids, + pack_buffer=pack_buffer, + dst_kv_item_lens=dst_kv_item_lens, + dst_tp_rank=dst_tp_rank, + dst_attn_tp_size=dst_attn_tp_size, + ) + + src_kv_ptrs = self.kv_args.kv_data_ptrs + _, dst_kv_ptrs, _ = self.get_mla_kv_ptrs_with_pp(src_kv_ptrs, dst_kv_ptrs) + if dst_kv_item_lens: + _, dst_kv_item_lens, _ = self.get_mla_kv_ptrs_with_pp( + self.kv_args.kv_item_lens, dst_kv_item_lens + ) + + layout = self._get_dcp_remote_decode_layout() + num_entries = len(src_kv_ptrs) + if not ( + len(dst_kv_ptrs) + == len(self.kv_args.kv_item_lens) + == len(dcp_token_item_lens) + == len(layout) + == num_entries + ): + raise RuntimeError("Ascend PD DCP KV entry metadata is inconsistent") + if dst_kv_item_lens and len(dst_kv_item_lens) != num_entries: + raise RuntimeError("Ascend PD DCP destination KV metadata is inconsistent") + + local_src, local_dst, global_src, global_dst = _build_page_interleaved_dcp_plan( + prefill_kv_indices, + dst_kv_indices, + page_size=self.kv_args.page_size, + dcp_size=dst_dcp_size, + dcp_rank=dst_dcp_rank, + src_page_offset=src_page_offset, + decode_prefix_len=decode_prefix_len, + num_kv_tokens=num_kv_tokens, + ) + + transfer_blocks: List[Tuple[int, int, int]] = [] + for entry, uses_global_slots in enumerate(layout): + page_bytes = self.kv_args.kv_item_lens[entry] + if page_bytes != dcp_token_item_lens[entry] * self.kv_args.page_size: + raise RuntimeError( + f"Ascend PD DCP source geometry differs at entry {entry}" + ) + if dst_kv_item_lens: + expected_dst_bytes = page_bytes * ( + dst_dcp_size if uses_global_slots else 1 + ) + if dst_kv_item_lens[entry] != expected_dst_bytes: + raise RuntimeError( + "Ascend PD DCP destination geometry differs at entry " + f"{entry}: expected={expected_dst_bytes}, " + f"actual={dst_kv_item_lens[entry]}" + ) + + src_pages = global_src if uses_global_slots else local_src + dst_pages = global_dst if uses_global_slots else local_dst + src_groups, dst_groups = group_concurrent_contiguous(src_pages, dst_pages) + for src_group, dst_group in zip(src_groups, dst_groups): + transfer_blocks.append( + ( + src_kv_ptrs[entry] + int(src_group[0]) * page_bytes, + dst_kv_ptrs[entry] + int(dst_group[0]) * page_bytes, + len(src_group) * page_bytes, + ) + ) + + return self._transfer_data(mooncake_session_id, transfer_blocks) + def _is_generic_kvcache_state_type(self, st) -> bool: # DSV4 per-pool components also use the page-indexed send path. return ( diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index b72dc2a296..498daeb3ce 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -290,6 +290,13 @@ class PrefillBootstrapQueue: kv_item_lens += draft_kv_item_lens num_draft_entries = len(draft_kv_data_ptrs) + dcp_remote_decode_layout = [] + if self.transfer_backend == TransferBackend.ASCEND: + for pool in (self.token_to_kv_pool, draft_kv_pool): + get_layout = getattr(pool, "get_dcp_remote_decode_layout", None) + if get_layout is not None: + dcp_remote_decode_layout.extend(get_layout()) + kv_args.kv_data_ptrs = kv_data_ptrs kv_args.kv_data_lens = kv_data_lens kv_args.kv_item_lens = kv_item_lens @@ -323,11 +330,17 @@ class PrefillBootstrapQueue: ) kv_manager_class = get_kv_class(self.transfer_backend, KVClassType.MANAGER) + kv_manager_kwargs = ( + {"dcp_remote_decode_layout": dcp_remote_decode_layout} + if self.transfer_backend == TransferBackend.ASCEND + else {} + ) kv_manager = kv_manager_class( kv_args, DisaggregationMode.PREFILL, self.scheduler.server_args, self.is_mla_backend, + **kv_manager_kwargs, ) # Pass KV pool tensor refs to the manager for GPU gather (staging mode) if ( diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index 859f01b93a..3e303f1c44 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -1558,7 +1558,7 @@ def setup_state_kv_args( tail_item_lens = tail_item_lens + draft_tail_item_lens if isinstance(token_to_kv_pool, NPUMLATokenToKVPool): kv_args.kv_buf_groups = ( - len(kv_args.kv_data_ptrs) // token_to_kv_pool.layer_num + 3 if token_to_kv_pool.index_head_dim is not None else 2 ) kv_args.hidden_kv_layers = total_kv_layers kv_args.draft_kv_layers = ( diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index d3277f4386..f5441452ae 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -2639,12 +2639,12 @@ def initialize_model_parallel( raise RuntimeError( f"decode_context_parallel_size ({decode_context_parallel_size}) must be >= 1" ) - if decode_context_parallel_size > 1 and not (is_hip() or is_cuda()): + if decode_context_parallel_size > 1 and not (is_hip() or is_cuda() or _is_npu): raise RuntimeError( "Decode context parallel (decode_context_parallel_size > 1) is " - "currently only supported on the AMD HIP platform or CUDA platform, but got " - f"decode_context_parallel_size ({decode_context_parallel_size}) " - "on a non-HIP or non-CUDA platform." + "currently only supported on the AMD HIP, CUDA, or NPU " + "platform, but got decode_context_parallel_size " + f"({decode_context_parallel_size}) on an unsupported platform." ) if tensor_model_parallel_size % decode_context_parallel_size != 0: raise RuntimeError( diff --git a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py index e0b8d14a2d..e0dcf0008c 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py +++ b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py @@ -15,6 +15,9 @@ from sglang.srt.dllm.config import DllmConfig from sglang.srt.hardware_backend.npu.attention.ascend_torch_native_backend import ( AscendTorchNativeAttnBackend, ) +from sglang.srt.hardware_backend.npu.attention.dsa_dcp import ( + forward_dcp_sparse_attention, +) from sglang.srt.hardware_backend.npu.attention.mla_cache import gather_mla_cache_pages from sglang.srt.hardware_backend.npu.attention.mla_preprocess import ( is_fia_nz, @@ -26,6 +29,10 @@ from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import ( ) from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp +from sglang.srt.layers.dcp.layout import ( + get_dcp_chain_spec_lens, + get_dcp_lens, +) from sglang.srt.layers.radix_attention import AttentionType from sglang.srt.layers.utils.cp_utils import cp_all_gather_rerange_kv_cache from sglang.srt.mem_cache.memory_pool import KVWriteLoc @@ -100,6 +107,16 @@ class ForwardMetadata: actual_seq_lengths_q_pa_cpu: Optional[torch.Tensor] = None actual_seq_lengths_kv: Optional[torch.Tensor] = None + # DCP decode inputs + dcp_seq_lens_cpu_int: Optional[torch.Tensor] = None + dcp_seq_lens: Optional[torch.Tensor] = None + dcp_block_tables: Optional[torch.Tensor] = None + dcp_origin_out_cache_loc: Optional[torch.Tensor] = None + # DCP target-verify inputs, expanded to one row per speculative query. + dcp_spec_seq_lens_cpu_int: Optional[torch.Tensor] = None + dcp_spec_seq_lens: Optional[torch.Tensor] = None + dcp_spec_block_tables: Optional[torch.Tensor] = None + # swa attention mask for graph mode decode swa_mask: Optional[torch.Tensor] = None @@ -309,6 +326,7 @@ class AscendAttnBackend(AttentionBackend): super().__init__() self.forward_metadata = None self.device = model_runner.device + self.is_draft_worker = model_runner.is_draft_worker self.speculative_step_id = speculative_step_id self.speculative_step_offset_npu = torch.tensor( speculative_step_id + 1, device="npu" @@ -444,6 +462,40 @@ class AscendAttnBackend(AttentionBackend): v = layer.v_head_dim return (d == v and d in (128, 192, 256)) or (d == 192 and v == 128) + def _get_kv_lens_and_block_tables( + self, + kv_lens_cpu: torch.Tensor, + req_pool_indices: torch.Tensor, + is_spec: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Build rank-local paged KV metadata for NPU DSA DCP.""" + parallel = get_parallel() + if is_spec: + local_kv_lens = get_dcp_chain_spec_lens( + kv_lens_cpu, + self.speculative_num_draft_tokens, + parallel.attn_dcp_size, + parallel.attn_dcp_rank, + interleave_size=self.page_size, + ) + else: + local_kv_lens = get_dcp_lens( + kv_lens_cpu, + parallel.attn_dcp_size, + parallel.attn_dcp_rank, + interleave_size=self.page_size, + ).int() + page_stride = self.page_size * parallel.attn_dcp_size + max_len = int(kv_lens_cpu.max().item()) if kv_lens_cpu.numel() else 0 + block_tables = ( + self.req_to_token[req_pool_indices, :max_len:page_stride] // page_stride + ).to(torch.int32) + if is_spec: + block_tables = block_tables.repeat_interleave( + self.speculative_num_draft_tokens, dim=0 + ) + return local_kv_lens, block_tables + def update_verify_buffers_to_fill_after_draft( self, spec_info: SpecInput, cuda_graph_bs: Optional[int] ): @@ -474,6 +526,7 @@ class AscendAttnBackend(AttentionBackend): forward_mode=forward_batch.forward_mode, spec_info=forward_batch.spec_info, out_cache_loc=forward_batch.out_cache_loc, + origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None), ) def init_forward_metadata(self, forward_batch: ForwardBatch): @@ -591,6 +644,40 @@ class AscendAttnBackend(AttentionBackend): device=self.device, ) + if get_parallel().dcp_enabled and not self.is_draft_worker: + # Draft workers retain allocator-global cache slots and use the non-DCP + # attention path, so only target workers need DCP-specific metadata. + self.forward_metadata.dcp_origin_out_cache_loc = ( + forward_batch.origin_out_cache_loc + ) + if forward_batch.forward_mode.is_target_verify(): + ( + self.forward_metadata.dcp_spec_seq_lens_cpu_int, + self.forward_metadata.dcp_spec_block_tables, + ) = self._get_kv_lens_and_block_tables( + kv_lens_cpu=self.forward_metadata.seq_lens_cpu_int, + req_pool_indices=forward_batch.req_pool_indices, + is_spec=True, + ) + self.forward_metadata.dcp_spec_seq_lens = ( + self.forward_metadata.dcp_spec_seq_lens_cpu_int.to( + device=self.device, dtype=torch.int32 + ) + ) + else: + ( + self.forward_metadata.dcp_seq_lens_cpu_int, + self.forward_metadata.dcp_block_tables, + ) = self._get_kv_lens_and_block_tables( + kv_lens_cpu=self.forward_metadata.seq_lens_cpu_int, + req_pool_indices=forward_batch.req_pool_indices, + ) + self.forward_metadata.dcp_seq_lens = ( + self.forward_metadata.dcp_seq_lens_cpu_int.to( + device=self.device, dtype=torch.int32 + ) + ) + if ( self.use_mla and forward_batch.forward_mode.is_extend() @@ -639,6 +726,40 @@ class AscendAttnBackend(AttentionBackend): device=self.device, ), } + # DCP decode needs its own block_table (different stride: dcp_page_size). + if get_parallel().dcp_enabled and not self.is_draft_worker: + dcp_page_size = self.page_size * get_parallel().attn_dcp_size + max_dcp_seq_pages = (total_context_len + dcp_page_size - 1) // dcp_page_size + # The captured DSA indexer cache-write operation reads from this fixed + # buffer. Replay refreshes its allocator-global slot ids before execution. + self.graph_metadata["dcp_origin_out_cache_loc"] = torch.zeros( + max_num_tokens, + dtype=torch.int64, + device=self.device, + ) + if self.speculative_num_draft_tokens is not None: + self.graph_metadata["dcp_spec_block_tables"] = torch.zeros( + ( + max_bs * self.speculative_num_draft_tokens, + max_dcp_seq_pages, + ), + dtype=torch.int32, + device=self.device, + ) + self.graph_metadata["dcp_spec_seq_lens"] = torch.zeros( + max_bs * self.speculative_num_draft_tokens, + dtype=torch.int32, + device=self.device, + ) + else: + self.graph_metadata["dcp_block_tables"] = torch.zeros( + (max_bs, max_dcp_seq_pages), + dtype=torch.int32, + device=self.device, + ) + self.graph_metadata["dcp_seq_lens"] = torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ) if self.is_hybrid_swa: self.graph_metadata["block_tables_swa"] = torch.empty( (max_bs, total_context_len // self.page_size), @@ -694,6 +815,26 @@ class AscendAttnBackend(AttentionBackend): torch.npu.current_stream(self.device), self.device ) metadata.block_tables = self.graph_metadata["block_tables"][:bs, :] + if get_parallel().dcp_enabled: + if "dcp_origin_out_cache_loc" in self.graph_metadata: + metadata.dcp_origin_out_cache_loc = self.graph_metadata[ + "dcp_origin_out_cache_loc" + ][: out_cache_loc.shape[0]] + if not self.is_draft_worker and "dcp_block_tables" in self.graph_metadata: + metadata.dcp_block_tables = self.graph_metadata["dcp_block_tables"][:bs] + metadata.dcp_seq_lens = self.graph_metadata["dcp_seq_lens"][:bs] + if ( + get_parallel().dcp_enabled + and "dcp_spec_block_tables" in self.graph_metadata + and forward_mode.is_target_verify() + ): + num_spec_rows = bs * self.speculative_num_draft_tokens + metadata.dcp_spec_block_tables = self.graph_metadata[ + "dcp_spec_block_tables" + ][:num_spec_rows] + metadata.dcp_spec_seq_lens = self.graph_metadata["dcp_spec_seq_lens"][ + :num_spec_rows + ] if self.is_hybrid_swa: metadata.block_tables_swa = self.graph_metadata["block_tables_swa"][:bs, :] metadata.swa_mask = self.graph_metadata["swa_mask"][:bs, :, :] @@ -765,6 +906,7 @@ class AscendAttnBackend(AttentionBackend): forward_mode: ForwardMode, spec_info: Optional[SpecInput], out_cache_loc: Optional[torch.Tensor] = None, + origin_out_cache_loc: Optional[torch.Tensor] = None, ): """Shared capture+replay body for the cuda-graph init path. @@ -779,11 +921,17 @@ class AscendAttnBackend(AttentionBackend): self.cuda_graph_swa_out_cache_loc[:n].copy_( self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc) ) - max_len = seq_lens_cpu[:bs].max().item() + # Compute the host-side KV lengths visible to this attention step. Target + # verify adds the draft block except for DFlash, whose seq_lens_cpu already + # includes it; speculative decode adds tokens through the current draft step. + attention_kv_lens_cpu = seq_lens_cpu[:bs].int() if forward_mode.is_target_verify() and not _is_dflash_verify(spec_info): - max_len += self.speculative_num_draft_tokens + attention_kv_lens_cpu = ( + attention_kv_lens_cpu + self.speculative_num_draft_tokens + ) elif forward_mode.is_decode_or_idle() and spec_info is not None: - max_len += self.speculative_step_id + 1 + attention_kv_lens_cpu = attention_kv_lens_cpu + self.speculative_step_id + 1 + max_len = attention_kv_lens_cpu.max().item() max_seq_pages = (max_len + self.page_size - 1) // self.page_size if self.is_hybrid_swa: @@ -826,6 +974,58 @@ class AscendAttnBackend(AttentionBackend): ) metadata.block_tables[:bs, max_seq_pages:].fill_(0) + + if get_parallel().dcp_enabled and not self.is_draft_worker: + if "dcp_origin_out_cache_loc" in self.graph_metadata: + buffer = self.graph_metadata["dcp_origin_out_cache_loc"] + if origin_out_cache_loc is None: + buffer.zero_() + else: + num_tokens = origin_out_cache_loc.shape[0] + assert num_tokens <= buffer.shape[0], ( + "NPU DSA+DCP origin_out_cache_loc exceeds its graph buffer: " + f"{num_tokens} > {buffer.shape[0]}" + ) + buffer[:num_tokens].copy_(origin_out_cache_loc) + buffer[num_tokens:].zero_() + # DCP decode/speculative paths use rank-local KV lengths and a block + # table whose stride is page_size * dcp_world_size. Draft attention + # uses the ordinary full-KV metadata above instead. + if forward_mode.is_target_verify(): + ( + metadata.dcp_spec_seq_lens_cpu_int, + dcp_spec_block_tables, + ) = self._get_kv_lens_and_block_tables( + kv_lens_cpu=attention_kv_lens_cpu, + req_pool_indices=req_pool_indices[:bs], + is_spec=True, + ) + metadata.dcp_spec_seq_lens.copy_( + metadata.dcp_spec_seq_lens_cpu_int.to( + device=metadata.dcp_spec_seq_lens.device + ) + ) + dcp_pages = dcp_spec_block_tables.shape[1] + metadata.dcp_spec_block_tables[:, :dcp_pages].copy_( + dcp_spec_block_tables + ) + metadata.dcp_spec_block_tables[:, dcp_pages:].fill_(0) + else: + ( + metadata.dcp_seq_lens_cpu_int, + dcp_block_tables, + ) = self._get_kv_lens_and_block_tables( + kv_lens_cpu=attention_kv_lens_cpu, + req_pool_indices=req_pool_indices[:bs], + ) + metadata.dcp_seq_lens.copy_( + metadata.dcp_seq_lens_cpu_int.to( + device=metadata.dcp_seq_lens.device + ) + ) + dcp_pages = dcp_block_tables.shape[1] + metadata.dcp_block_tables[:bs, :dcp_pages].copy_(dcp_block_tables) + metadata.dcp_block_tables[:bs, dcp_pages:].fill_(0) metadata.block_tables[bs:, :].fill_(0) if forward_mode.is_target_verify(): @@ -1225,6 +1425,19 @@ class AscendAttnBackend(AttentionBackend): if topk_indices is not None: topk_indices = self._pad_topk_indices(topk_indices, q_nope.shape[0]) topk_indices = _expand_dsa_sparse_indices(topk_indices) + if get_parallel().dcp_enabled and not self.is_draft_worker: + return forward_dcp_sparse_attention( + q_nope=q_nope, + q_rope=q_pe, + k_nope=k_nope, + k_rope=k_pe, + topk_indices=topk_indices, + actual_seq_lengths_query=actual_seq_qlen, + forward_metadata=self.forward_metadata, + forward_batch=forward_batch, + speculative_num_draft_tokens=self.speculative_num_draft_tokens, + scaling=layer.scaling, + ) if self.kv_cache_dtype == torch.float8_e4m3fn: assert q_nope.dtype == q_pe.dtype == torch.bfloat16 packed = k_nope.view(torch.float8_e4m3fn) diff --git a/python/sglang/srt/hardware_backend/npu/attention/dsa_dcp.py b/python/sglang/srt/hardware_backend/npu/attention/dsa_dcp.py new file mode 100644 index 0000000000..6515f74264 --- /dev/null +++ b/python/sglang/srt/hardware_backend/npu/attention/dsa_dcp.py @@ -0,0 +1,84 @@ +"""NPU DSA adaptation for token-sharded decode context parallelism.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sgl_kernel_npu # noqa: F401 Registers torch.ops.sgl_kernel_npu. +import torch + +if TYPE_CHECKING: + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + + +def forward_dcp_sparse_attention( + *, + q_nope: torch.Tensor, + q_rope: torch.Tensor, + k_nope: torch.Tensor, + k_rope: torch.Tensor, + topk_indices: torch.Tensor, + actual_seq_lengths_query: torch.Tensor, + forward_metadata, + forward_batch: ForwardBatch, + speculative_num_draft_tokens: int | None, + scaling: float, +) -> tuple[torch.Tensor, torch.Tensor]: + """Compute one rank's sparse partial attention and natural-log LSE.""" + + is_target_verify = forward_batch.forward_mode.is_target_verify() + if is_target_verify: + block_tables = forward_metadata.dcp_spec_block_tables + local_lens = forward_metadata.dcp_spec_seq_lens + else: + block_tables = forward_metadata.dcp_block_tables + local_lens = forward_metadata.dcp_seq_lens + + assert block_tables is not None, ( + "NPU DSA+DCP sparse attention requires rank-local paged-KV metadata" + ) + if is_target_verify: + # DCP verify presents every speculative query as an independent + # single-query sequence. Its KV frontier and page-table row therefore + # stay expanded in request-major [bs * draft_token_num] order. + num_query_rows = q_nope.shape[0] + assert block_tables.shape[0] == num_query_rows, ( + "NPU DSA+DCP verify block-table rows must match query rows: " + f"{block_tables.shape[0]} != {num_query_rows}" + ) + assert local_lens.shape[0] == num_query_rows, ( + "NPU DSA+DCP verify KV-length rows must match query rows: " + f"{local_lens.shape[0]} != {num_query_rows}" + ) + actual_seq_lengths_query = torch.arange( + 1, + num_query_rows + 1, + device=q_nope.device, + dtype=torch.int32, + ) + else: + actual_seq_lengths_query = actual_seq_lengths_query.to( + device=q_nope.device, dtype=torch.int32 + ) + attn_out, softmax_max, softmax_sum = torch.ops.npu.sgl_sparse_flash_attention( + query=q_nope, + key=k_nope, + value=k_nope, + query_rope=q_rope, + key_rope=k_rope, + sparse_indices=topk_indices, + scale_value=scaling, + actual_seq_lengths_query=actual_seq_lengths_query, + actual_seq_lengths_kv=local_lens, + block_table=block_tables, + sparse_block_size=1, + layout_query="TND", + layout_kv="PA_BSND", + # Global top-k already enforces causal visibility. Local indices no + # longer carry enough information for the kernel's causal sparse mode. + sparse_mode=0, + attention_mode=2, + return_softmax_lse=True, + ) + lse = softmax_max.float() + torch.log(softmax_sum.float()) + return attn_out, lse.permute(1, 0, 2).reshape(lse.shape[1], -1) diff --git a/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py b/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py index 14ea774717..013e9db066 100644 --- a/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py +++ b/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py @@ -4,6 +4,7 @@ import torch from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs +from sglang.srt.layers.dcp.layout import localize_dcp_indices from sglang.srt.mem_cache.memory_pool import ( MHATokenToKOnlyPool, MHATokenToKVPool, @@ -12,6 +13,7 @@ from sglang.srt.mem_cache.memory_pool import ( get_tensor_size_bytes, unwrap_write_loc, ) +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_bool_env_var from sglang.srt.utils.common import is_npu @@ -585,10 +587,13 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): device: str, enable_memory_saver: bool, index_head_dim: Optional[int] = None, + index_size: Optional[int] = None, start_layer: Optional[int] = None, end_layer: Optional[int] = None, + index_page_size: Optional[int] = None, indexer_layer_ids: Optional[Sequence[int]] = None, kv_cache_dim: Optional[int] = None, + is_draft_worker: bool = False, ): # MLAPO historically owned NZ writes. Keep the allocation unchanged and # write into the NZ-addressed view below so ordinary MLA (including @@ -613,7 +618,6 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): ) if self.enable_sparsity_driven_kv_offload and self.index_head_dim is None: raise ValueError("Sparsity-driven KV offload requires an index KV cache.") - if index_head_dim is None: self.indexer_layer_ids = () elif indexer_layer_ids is None: @@ -649,6 +653,19 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): self.kr_cache_dim = 0 if self.dsa_kv_cache_store_fp8 else qk_rope_head_dim self.index_k_scale_buffer = None self.indexer_hadamard_128 = None + self.index_page_size = page_size if index_page_size is None else index_page_size + self.index_size = size if index_size is None else index_size + parallel = get_parallel() + self.dcp_size = parallel.attn_dcp_size + self.dcp_rank = parallel.attn_dcp_rank + global_page_padding = self.dcp_size if self.dcp_size > 1 else 1 + kv_page_padding = global_page_padding if is_draft_worker else 1 + index_page_padding = global_page_padding if index_head_dim is not None else 1 + if kv_page_padding < 1 or index_page_padding < 1: + raise ValueError("NPU MLA page padding must be positive") + self.kv_page_padding = kv_page_padding + self.index_page_padding = index_page_padding + self.is_draft_worker = is_draft_worker self.custom_mem_pool = None @@ -661,7 +678,7 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): self.k_buffer = torch.zeros( ( layer_num, - self.size // self.page_size + 1, + self.size // self.page_size + self.kv_page_padding, self.page_size, 1, self.kv_cache_dim, @@ -672,7 +689,7 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): self.v_buffer = torch.zeros( ( layer_num, - self.size // self.page_size + 1, + self.size // self.page_size + self.kv_page_padding, self.page_size, 1, self.kr_cache_dim, @@ -689,8 +706,9 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): self.index_k_buffer = torch.zeros( ( self.num_indexer_layers, - self.size // self.page_size + 1, - self.page_size, + self.index_size // self.index_page_size + + self.index_page_padding, + self.index_page_size, 1, self.index_head_dim, ), @@ -713,6 +731,17 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): self._finalize_allocation_log(size) + def _copy_indices_for_buffer(self, indices, uses_global_slots): + if uses_global_slots or self.dcp_size <= 1: + return indices + local_indices = localize_dcp_indices( + indices, + self.dcp_size, + self.dcp_rank, + self.page_size, + ) + return local_indices[local_indices >= 0] + def get_kv_size_bytes(self): kv_size_bytes = 0 @@ -805,25 +834,37 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): scale.view(-1, 1), ) + def _get_disagg_buffer_entries(self): + """Return (buffer, uses_global_slots) entries in PD transfer order.""" + self._raise_if_native_kv_cache_disabled() + global_kv = self.is_draft_worker + entries = [(buffer, global_kv) for buffer in self.k_buffer] + if not getattr(self, "dsa_kv_cache_store_fp8", False): + entries += [(buffer, global_kv) for buffer in self.v_buffer] + if self.index_head_dim is not None: + entries += [(buffer, True) for buffer in self.index_k_buffer] + if self.index_k_scale_buffer is not None: + entries += [(buffer, True) for buffer in self.index_k_scale_buffer] + return entries + # for disagg def get_contiguous_buf_infos(self): - self._raise_if_native_kv_cache_disabled() - # MLA has only one kv_buffer, so only the information of this buffer needs to be returned. - kv_data_ptrs = [self.k_buffer[i].data_ptr() for i in range(self.layer_num)] - kv_data_lens = [self.k_buffer[i].nbytes for i in range(self.layer_num)] - kv_item_lens = [self.k_buffer[i][0].nbytes for i in range(self.layer_num)] - # When DSA KV cache is packed into the FP8 k_buffer, the v_buffer is - # intentionally empty (kr_cache_dim == 0). Its data_ptr() is null - if not getattr(self, "dsa_kv_cache_store_fp8", False): - kv_data_ptrs += [self.v_buffer[i].data_ptr() for i in range(self.layer_num)] - kv_data_lens += [self.v_buffer[i].nbytes for i in range(self.layer_num)] - kv_item_lens += [self.v_buffer[i][0].nbytes for i in range(self.layer_num)] - if self.index_head_dim is not None: - ptrs, lens, item_lens = self.get_state_buf_infos() - kv_data_ptrs += ptrs - kv_data_lens += lens - kv_item_lens += item_lens - return kv_data_ptrs, kv_data_lens, kv_item_lens + entries = self._get_disagg_buffer_entries() + return ( + [buffer.data_ptr() for buffer, _ in entries], + [buffer.nbytes for buffer, _ in entries], + [ + buffer[0].nbytes * (self.dcp_size if uses_global_slots else 1) + for buffer, uses_global_slots in entries + ], + ) + + def get_dcp_remote_decode_layout(self) -> list[bool]: + """Whether each PD entry uses allocator-global slots on decode.""" + return [ + uses_global_slots + for _, uses_global_slots in self._get_disagg_buffer_entries() + ] def get_kv_layer_ids(self): return ( @@ -956,17 +997,25 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): index_k.view(-1, 1, self.index_head_dim), ) - def _chunk_copy_npu_to_cpu(self, buf_of_layers, indices): + def _chunk_copy_npu_to_cpu( + self, buf_of_layers, indices, uses_global_slots_per_layer + ): chunk_size = self.cpu_offloading_chunk_size out = [] - for tensors_per_layer in buf_of_layers: # [k_buf, v_buf, ik_buf/None] + for tensors_per_layer, uses_global_slots in zip( + buf_of_layers, uses_global_slots_per_layer, strict=True + ): # [k_buf, v_buf, ik_buf/None] layer_chunks = [] for i in range(0, len(indices), chunk_size): ci = indices[i : i + chunk_size] layer_chunks.append( [ - t[ci].to("cpu", non_blocking=True) - for t in tensors_per_layer + t[self._copy_indices_for_buffer(ci, uses_global)].to( + "cpu", non_blocking=True + ) + for t, uses_global in zip( + tensors_per_layer, uses_global_slots, strict=True + ) if t is not None ] ) @@ -998,8 +1047,18 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): buf_of_layers = [ self._get_cpu_offload_layer_buffers(i) for i in range(self.layer_num) ] + uses_global_slots_per_layer = [] + for buffers in buf_of_layers: + # MLA K/V is rank-local under DCP. The replicated + # indexer buffers retain allocator-global slot identities. + uses_global_slots_per_layer.append( + [self.is_draft_worker, self.is_draft_worker] + + [True] * (len(buffers) - 2) + ) - kv_cache_cpu = self._chunk_copy_npu_to_cpu(buf_of_layers, indices) + kv_cache_cpu = self._chunk_copy_npu_to_cpu( + buf_of_layers, indices, uses_global_slots_per_layer + ) torch.npu.synchronize() return kv_cache_cpu @@ -1013,7 +1072,20 @@ class NPUMLATokenToKVPool(MLATokenToKVPool): for i in range(0, len(indices), chunk_size): chunk_indices = indices[i : i + chunk_size] chunk = kv_cache_cpu[local_layer_id][i // chunk_size] - for buffer, cpu in zip(buffers, chunk, strict=True): - assert cpu.shape[0] == len(chunk_indices) - buffer[chunk_indices] = cpu.to(buffer.device, non_blocking=True) + cpu_index = 0 + for buffer, uses_global_slots in zip( + buffers, + [self.is_draft_worker, self.is_draft_worker] + + [True] * (len(buffers) - 2), + strict=True, + ): + if buffer is None: + continue + cpu = chunk[cpu_index] + cpu_index += 1 + target_indices = self._copy_indices_for_buffer( + chunk_indices, uses_global_slots + ) + assert cpu.shape[0] == len(target_indices) + buffer[target_indices] = cpu.to(buffer.device, non_blocking=True) torch.npu.synchronize() diff --git a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py index 7b2ec443bc..0e1fa06d49 100644 --- a/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py +++ b/python/sglang/srt/hardware_backend/npu/modules/deepseek_v2_attention_mla_npu.py @@ -4,6 +4,10 @@ import torch import torch_npu from sgl_kernel_npu.norm.fused_split_qk_norm import fused_split_qk_norm +import sglang.srt.layers.dcp.comm as dcp_comm +import sglang.srt.layers.dcp.layout as dcp_layout +import sglang.srt.model_executor.forward_context as forward_context +import sglang.srt.runtime_context as runtime_context from sglang.srt.environ import envs from sglang.srt.hardware_backend.npu.attention.mla_preprocess import ( NPUFusedMLAPreprocess, @@ -28,6 +32,15 @@ _use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get() _is_npu_arch35 = is_npu_arch35() +def _use_dsa_dcp_partial_attention(forward_batch: "ForwardBatch") -> bool: + return ( + runtime_context.get_parallel().dcp_enabled + and not forward_context.get_attn_backend().is_draft_worker + and not dsa_use_prefill_cp(forward_batch) + and not forward_batch.forward_mode.is_idle() + ) + + # region MHA def forward_mha_prepare_npu( m: "DeepseekV2AttentionMLA", @@ -488,9 +501,23 @@ def forward_dsa_prepare_npu( input_on_attn_tp_slices, dynamic_scale, ) + # DSA layers that skip the indexer reuse ``prev_topk_indices``. Remap + # only when a fresh global top-k is produced so shared-index layers do + # not repeat the same DCP partitioning work. + if _use_dsa_dcp_partial_attention(forward_batch): + parallel = runtime_context.get_parallel() + topk_indices = dcp_layout.remap_dcp_sparse_indices( + topk_indices, + parallel.attn_dcp_size, + parallel.attn_dcp_rank, + interleave_size=forward_context.get_attn_backend().page_size, + ) else: topk_indices = prev_topk_indices + if _use_dsa_dcp_partial_attention(forward_batch): + q_nope_out, q_pe = dcp_comm.all_gather_q_for_mla_decode(q_nope_out, q_pe) + topk_indices = maybe_capture_indexer_topk(m.layer_id, topk_indices) return ( @@ -522,7 +549,9 @@ def forward_dsa_core_npu( # a trailing arg. None everywhere else. gate: Optional[torch.Tensor] = None, ) -> torch.Tensor: - attn_output = m.attn_mqa( + use_dcp = _use_dsa_dcp_partial_attention(forward_batch) + attn = m.attn_mqa_for_dcp_decode if use_dcp else m.attn_mqa + attn_output = attn( q_nope_out.contiguous(), k_nope.contiguous(), k_nope.contiguous(), @@ -532,6 +561,16 @@ def forward_dsa_core_npu( k_rope=k_pe.contiguous(), topk_indices=topk_indices, ) + if use_dcp: + attn_output, lse = attn_output + attn_output = attn_output.view( + -1, + m.num_local_heads * runtime_context.get_parallel().attn_dcp_size, + m.kv_lora_rank, + ) + attn_output = dcp_comm.cp_lse_ag_out_rs_mla_npu( + attn_output, lse, runtime_context.get_parallel().dcp_group + ) attn_output = attn_output.view(-1, m.num_local_heads, m.kv_lora_rank) if _is_npu_arch35 or ( diff --git a/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py index da832b28d7..64330f8073 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py @@ -12,6 +12,7 @@ from sglang.srt.model_executor.forward_context import ( get_attn_backend, get_token_to_kv_pool, ) +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import is_npu if is_npu(): @@ -207,16 +208,31 @@ class DSANPUIndexerMixin: torch.npu.current_stream(), ) + indexer_cache_loc = forward_batch.out_cache_loc + parallel = get_parallel() + if ( + parallel.dcp_enabled + and parallel.attn_dcp_size > 1 + and not get_attn_backend().is_draft_worker + ): + indexer_cache_loc = ( + get_attn_backend().forward_metadata.dcp_origin_out_cache_loc + ) + assert indexer_cache_loc is not None, ( + "NPU DSA+DCP requires allocator-global origin_out_cache_loc metadata" + ) + assert indexer_cache_loc.shape[0] == positions.shape[0], ( + "NPU DSA+DCP origin_out_cache_loc metadata has an incompatible " + f"length: {indexer_cache_loc.shape[0]} != {positions.shape[0]}" + ) pool = get_token_to_kv_pool() use_quant_indexer = pool.index_k_scale_buffer is not None if use_quant_indexer: k, k_scale = _quantize_npu_indexer_activation( k, pool.indexer_hadamard_128, pool.dtype ) - pool.set_index_k_scale_buffer( - layer_id, forward_batch.out_cache_loc, k_scale - ) - pool.set_index_k_buffer(layer_id, forward_batch.out_cache_loc, k) + pool.set_index_k_scale_buffer(layer_id, indexer_cache_loc, k_scale) + pool.set_index_k_buffer(layer_id, indexer_cache_loc, k) if is_prefill: if ( self.dsa_enable_prefill_cp diff --git a/python/sglang/srt/layers/dcp/comm.py b/python/sglang/srt/layers/dcp/comm.py index b06a49cbe0..37018f0397 100644 --- a/python/sglang/srt/layers/dcp/comm.py +++ b/python/sglang/srt/layers/dcp/comm.py @@ -123,6 +123,41 @@ def cp_lse_ag_out_rs_mla( return out.to(cp_attn_out.dtype) +def cp_lse_ag_out_rs_mla_npu( + cp_attn_out: torch.Tensor, + cp_attn_lse: torch.Tensor, + cp_group: GroupCoordinator, +) -> torch.Tensor: + """Merge NPU DCP partial outputs and return the local head slice.""" + if cp_group.world_size == 1: + return cp_attn_out + + import torch_npu + + batch_size, total_heads, head_dim = cp_attn_out.shape + world_size = cp_group.world_size + local_heads = total_heads // world_size + packed = torch.cat([cp_attn_out.float(), cp_attn_lse.float().unsqueeze(-1)], dim=-1) + packed = packed.permute(1, 2, 0).contiguous() + gathered = torch.empty_like(packed) + cp_group.all_to_all_single(gathered, packed) + # all_to_all_single splits the leading head dimension. After the exchange, + # the heads are grouped by source rank inside every token. Move that source + # rank in front before flattening tokens and local heads for the update op. + gathered = gathered.permute(2, 0, 1).contiguous() + gathered = ( + gathered.view(batch_size, world_size, local_heads, head_dim + 1) + .permute(1, 0, 2, 3) + .contiguous() + .view(world_size, batch_size * local_heads, head_dim + 1) + ) + out_flat, lse_flat = torch.split(gathered, [head_dim, 1], dim=-1) + merged, _ = torch_npu.npu_attention_update( + lse_flat.squeeze(-1).unbind(0), out_flat.unbind(0), 0 + ) + return merged.view(batch_size, local_heads, head_dim).to(cp_attn_out.dtype) + + def _all_gather_dcp_kv_cache(kv_a: torch.Tensor): parallel = get_parallel() dcp_world_size = parallel.dcp_size diff --git a/python/sglang/srt/layers/dcp/layout.py b/python/sglang/srt/layers/dcp/layout.py index 58697f443b..8781ff86b3 100644 --- a/python/sglang/srt/layers/dcp/layout.py +++ b/python/sglang/srt/layers/dcp/layout.py @@ -25,20 +25,53 @@ def get_dcp_lens( dcp_size: int, dcp_rank: int, start: torch.Tensor | None = None, + interleave_size: int = 1, ) -> torch.Tensor: - """Per-rank visible KV length under the owner rule pos % dcp_size == dcp_rank. + """Return the KV length owned by one DCP rank. - Superset implementation (PR #25090): supports both start=None and a per-request - `start` offset. update_local_kv_lens_for_dcp is the start=None special case. + Consecutive runs of ``interleave_size`` positions rotate across ranks. The + default value preserves the token-interleaved layout used by the generic + CUDA/ROCm DCP paths; NPU DSA passes its physical KV page size. """ if dcp_size == 1: return lens - if start is None: - return lens // dcp_size + (dcp_rank < lens % dcp_size) + if interleave_size < 1: + raise ValueError(f"interleave_size must be positive, got {interleave_size}") - first = start + torch.remainder(dcp_rank - start, dcp_size) - remaining = start + lens - first - return torch.clamp((remaining + dcp_size - 1) // dcp_size, min=0) + cycle_size = dcp_size * interleave_size + + def _count_before(end: torch.Tensor) -> torch.Tensor: + full_cycles = end // cycle_size + remainder = end % cycle_size + rank_remainder = torch.clamp( + remainder - dcp_rank * interleave_size, + min=0, + max=interleave_size, + ) + return full_cycles * interleave_size + rank_remainder + + if start is None: + return _count_before(lens) + return _count_before(start + lens) - _count_before(start) + + +def localize_dcp_indices( + indices: torch.Tensor, + dcp_size: int, + dcp_rank: int, + interleave_size: int = 1, +) -> torch.Tensor: + """Map global DCP indices to one rank, using ``-1`` for non-local rows.""" + if dcp_size == 1: + return indices + if interleave_size < 1: + raise ValueError(f"interleave_size must be positive, got {interleave_size}") + + interleave_block = torch.div(indices, interleave_size, rounding_mode="floor") + is_local = (indices >= 0) & (interleave_block % dcp_size == dcp_rank) + local_block = torch.div(interleave_block, dcp_size, rounding_mode="floor") + local_indices = local_block * interleave_size + indices % interleave_size + return torch.where(is_local, local_indices, torch.full_like(indices, -1)) def maybe_dcp_kernel_indices( @@ -84,6 +117,77 @@ def filter_dcp_local_chunk_kv_indices( return torch.cat(parts) +def remap_dcp_sparse_indices( + topk_indices: torch.Tensor, + dcp_size: int, + dcp_rank: int, + interleave_size: int = 1, +) -> torch.Tensor: + """Map global sparse token indices to one rank's compact DCP KV layout. + + Keep indices owned by this rank, convert them to local KV positions, and + stably move them before ``-1`` padding while preserving their score order. + """ + if dcp_size == 1: + return topk_indices + if interleave_size < 1: + raise ValueError(f"interleave_size must be positive, got {interleave_size}") + + # Float32 is faster than integer division/remainder on Ascend for this hot + # path. Keep the math equivalent to localize_dcp_indices above. + topk_indices_fp32 = topk_indices.to(torch.float32) + interleave_blocks = torch.floor(topk_indices_fp32 / interleave_size) + local_owner_mask = (topk_indices_fp32 >= 0) & ( + torch.remainder(interleave_blocks, dcp_size) == dcp_rank + ) + local_offsets = torch.remainder(topk_indices_fp32, interleave_size) + local_indices = ( + torch.floor(interleave_blocks / dcp_size) * interleave_size + local_offsets + ) + remapped_indices = torch.where( + local_owner_mask, + local_indices, + torch.full_like(topk_indices_fp32, -1.0), + ).to(topk_indices.dtype) + + # Move valid entries before padding without changing their top-k order. + topk_count = topk_indices.shape[-1] + original_order = torch.arange( + topk_count, dtype=torch.float32, device=topk_indices.device + ).expand_as(topk_indices_fp32) + pack_keys = original_order + (~local_owner_mask).to(torch.float32) * topk_count + pack_order = torch.argsort(pack_keys, dim=-1).to(torch.int64) + return torch.gather(remapped_indices, dim=-1, index=pack_order) + + +def get_dcp_chain_spec_lens( + total_kv_lens: torch.Tensor, + tokens_per_req: int, + dcp_size: int, + dcp_rank: int, + interleave_size: int = 1, +) -> torch.Tensor: + """Return request-major local KV frontiers for a speculative chain.""" + if tokens_per_req < 1: + raise ValueError(f"tokens_per_req must be >= 1, got {tokens_per_req}") + total_kv_lens = total_kv_lens.int() + steps = torch.arange( + 1, tokens_per_req + 1, dtype=total_kv_lens.dtype, device=total_kv_lens.device + ) + global_query_lens = total_kv_lens.unsqueeze(1) - tokens_per_req + steps + global_query_lens = torch.where( + total_kv_lens.unsqueeze(1) >= tokens_per_req, + global_query_lens, + torch.zeros_like(global_query_lens), + ) + return get_dcp_lens( + global_query_lens.reshape(-1), + dcp_size, + dcp_rank, + interleave_size=interleave_size, + ).int() + + def update_local_kv_lens_for_dcp(kv_len_arr): """In-place per-rank KV length: the start=0 case of get_dcp_lens. diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 65761b087e..7202fe89a8 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -1555,6 +1555,12 @@ class KVCacheConfigurator: ) from sglang.srt.hardware_backend.npu.utils import is_npu_arch35 + # Indexer uses the allocator-global slot space on DCP target workers. + dcp_size = get_parallel().attn_dcp_size + if self.is_draft_worker: + index_size = max_total_num_tokens + else: + index_size = max_total_num_tokens * dcp_size is_arch35 = is_npu_arch35() use_compact_indexer_layout = ( is_dsa_model @@ -1576,11 +1582,13 @@ class KVCacheConfigurator: ) token_to_kv_pool = NPUMLATokenToKVPool( max_total_num_tokens, - page_size=self.pool_page_size, + page_size=get_schedule().page_size, dtype=self.kv_cache_dtype, kv_lora_rank=self.model_config.kv_lora_rank, qk_rope_head_dim=self.model_config.qk_rope_head_dim, index_head_dim=(self.model_config.index_head_dim if is_dsa_model else None), + index_size=index_size, + index_page_size=get_schedule().page_size, indexer_layer_ids=indexer_layer_ids, kv_cache_dim=( calculate_mla_kv_cache_dim( @@ -1589,6 +1597,7 @@ class KVCacheConfigurator: if use_dsa_fp8_kv_cache_storage else None ), + is_draft_worker=self.is_draft_worker, layer_num=self.layer_info.num_effective_layers, device=self.device, enable_memory_saver=get_exec().features.enable_memory_saver, @@ -2057,8 +2066,13 @@ class KVCacheConfigurator: ) token_to_kv_pool_allocator = NPUPagedTokenToKVPoolAllocator( - sizes.max_total_num_tokens, - page_size=get_schedule().page_size, + # DCP allocation is in the global virtual loc space. + # The target attention path localizes these locs when + # building rank-local metadata; draft/indexer paths + # consume the allocator locs directly. + sizes.max_total_num_tokens * get_parallel().attn_dcp_size, + page_size=get_schedule().page_size + * get_parallel().attn_dcp_size, dtype=self.kv_cache_dtype, device=self.device, kvcache=token_to_kv_pool, diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 7c874a3948..cc6af6309d 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -43,6 +43,7 @@ from sglang.srt.environ import envs from sglang.srt.kv_canary.req_to_expected_token_ids_manager import ( compute_req_all_ids_info, ) +from sglang.srt.layers.dcp.layout import localize_dcp_indices from sglang.srt.layers.dp_attention import ( DpPaddingMode, dp_slot_in, @@ -191,6 +192,21 @@ def _elastic_should_preserve_local_token_counts( return uneven_token_count +def _localize_npu_dcp_out_cache_loc( + out_cache_loc: torch.Tensor, + *, + interleave_size: int, +) -> torch.Tensor: + """Map allocator-global NPU DCP slots to this target rank.""" + parallel = get_parallel() + return localize_dcp_indices( + out_cache_loc, + parallel.dcp_size, + parallel.dcp_rank, + interleave_size, + ) + + class ForwardMode(IntEnum): # Extend a sequence. The KV cache of the beginning part of the sequence is already computed (e.g., system prompt). # It is also called "prefill" in common terminology. @@ -505,6 +521,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # The sum of all sequence lengths seq_lens_sum: int + # Allocator-global output slots before NPU DCP localizes ``out_cache_loc``. + # DSA's replicated indexer cache uses the global slot identity. + origin_out_cache_loc: Optional[torch.Tensor] = None + # === Borrowed from ScheduleBatch: GPU tensors (cross-stream; clone targets for stream isolation) === # FIXME(lsyin): these are currently aliased by reference from ScheduleBatch. Once # they are cloned/relayed into FB-owned copies at the boundary, move them out of @@ -1025,6 +1045,15 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): spec_info=batch.spec_info, ) + # ScheduleBatch and req_to_token keep allocator-global slot identities. + # Preserve that view before exposing rank-local NPU DCP write slots. + if _is_npu and get_parallel().dcp_enabled and not model_runner.is_draft_worker: + ret.origin_out_cache_loc = ret.out_cache_loc + if ret.out_cache_loc is not None: + ret.out_cache_loc = _localize_npu_dcp_out_cache_loc( + ret.out_cache_loc, + interleave_size=model_runner.page_size, + ) ret._maybe_init_non_generation_fields(batch) device = model_runner.device @@ -1820,6 +1849,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): ) self.out_cache_loc = self._pad_tensor_to_size(self.out_cache_loc, num_tokens) + if self.origin_out_cache_loc is not None: + self.origin_out_cache_loc = self._pad_tensor_to_size( + self.origin_out_cache_loc, num_tokens + ) if self.encoder_lens is not None: self.encoder_lens = self._pad_tensor_to_size(self.encoder_lens, bs) self.positions = self._pad_tensor_to_size(self.positions, num_tokens) @@ -2086,6 +2119,7 @@ def build_inner_fb_view( out_cache_loc=getattr(forward_batch, "out_cache_loc", None), # A caller may hand in another view that does not carry this field. out_cache_loc_virtual=getattr(forward_batch, "out_cache_loc_virtual", None), + origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None), out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None), spec_info=forward_batch.spec_info, ) diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 4a1f60f8c2..74163e6830 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -192,6 +192,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): def __init__(self, kvc: KVCacheConfigurator): self.kv_cache_dtype_str = kvc.kv_cache_dtype_str + dcp_size = get_parallel().attn_dcp_size # Determine effective number of layers for KV cache if mambaish := mambaish_config(kvc.model_config): effective_layer_ids = [ @@ -233,6 +234,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): kvc=kvc, num_layers=num_layers, ) + if _is_npu and dcp_size > 1: + target_indexer_size *= dcp_size target_kv_size = self._cell_size - target_indexer_size from sglang.srt.layers.cp.utils import ( get_glm_dsa_layer_split_effective_num_layers, @@ -249,6 +252,11 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): num_layers=draft_num_layers, allocate_all_layers=True, ) + # The draft pool is replicated and consumes the widened + # allocator-global slot space on NPU DCP. + if _is_npu and dcp_size > 1: + draft_kv_size *= dcp_size + draft_indexer_size *= dcp_size self._cell_size += draft_kv_size + draft_indexer_size else: self._cell_size = int( @@ -337,10 +345,13 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): # Add indexer KV cache overhead for DSA models (DeepSeek V3.2) if is_deepseek_dsa(model_config.hf_config): - cell_size += self._compute_dsa_indexer_cell_size( + indexer_cell_size = self._compute_dsa_indexer_cell_size( kvc=kvc, num_layers=num_layers, ) + if _is_npu and not kvc.is_draft_worker and dcp_size > 1: + indexer_cell_size *= dcp_size + cell_size += indexer_cell_size elif is_minimax_sparse(model_config.hf_config): from sglang.srt.server_args import m3_fp8_attn_gemm_enabled diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 547ce91771..aa0314f742 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -198,6 +198,7 @@ def build_replay_fb_view( encoder_lens=buffers.encoder_lens[:bs] if is_encoder_decoder else None, out_cache_loc=getattr(forward_batch, "out_cache_loc", None), out_cache_loc_virtual=forward_batch.out_cache_loc_virtual, + origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None), out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None), max_seq_len_override=forward_batch.max_seq_len_override, # The mamba-track registry slot (VIRTUAL ids) is the v2p translate SOURCE diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 019173a448..6b60034ca5 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -3420,6 +3420,8 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): kv_cache_device, create_chunked_prefix_cache_kv_indices_fn, ): + if _is_npu: + return None return prepare_decode_context_parallel_metadata( seq_lens=seq_lens, extend_prefix_lens=extend_prefix_lens, diff --git a/test/registered/unit/disaggregation/test_ascend_page_dcp.py b/test/registered/unit/disaggregation/test_ascend_page_dcp.py new file mode 100644 index 0000000000..a743c95fd9 --- /dev/null +++ b/test/registered/unit/disaggregation/test_ascend_page_dcp.py @@ -0,0 +1,201 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import Mock + +import numpy as np + +from sglang.srt.disaggregation.ascend.conn import ( + AscendKVManager, + _build_page_interleaved_dcp_plan, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +class TestAscendPageDcpPlan(CustomTestCase): + def test_assigns_whole_pages_to_dcp_ranks(self): + src = np.array([5, 2, 11, 4], dtype=np.int32) + dst = np.array([7], dtype=np.int32) + + for rank, expected_src in enumerate(src): + local_src, local_dst, global_src, global_dst = ( + _build_page_interleaved_dcp_plan( + src, + dst, + page_size=2, + dcp_size=4, + dcp_rank=rank, + src_page_offset=0, + decode_prefix_len=0, + num_kv_tokens=8, + ) + ) + np.testing.assert_array_equal(local_src, [expected_src]) + np.testing.assert_array_equal(local_dst, [7]) + np.testing.assert_array_equal(global_src, src) + np.testing.assert_array_equal(global_dst, [28, 29, 30, 31]) + + def test_chunk_offset_crosses_virtual_pages(self): + args = { + "src_page_indices": np.array([9, 3], dtype=np.int32), + "dst_page_indices": np.array([4, 6], dtype=np.int32), + "page_size": 2, + "dcp_size": 2, + "src_page_offset": 1, + "decode_prefix_len": 4, + "num_kv_tokens": 4, + } + rank0 = _build_page_interleaved_dcp_plan(dcp_rank=0, **args) + rank1 = _build_page_interleaved_dcp_plan(dcp_rank=1, **args) + + np.testing.assert_array_equal(rank0[0], [3]) + np.testing.assert_array_equal(rank0[1], [6]) + np.testing.assert_array_equal(rank1[0], [9]) + np.testing.assert_array_equal(rank1[1], [4]) + np.testing.assert_array_equal(rank0[2], [9, 3]) + np.testing.assert_array_equal(rank0[3], [9, 12]) + + def test_partial_page_and_validation(self): + plan = _build_page_interleaved_dcp_plan( + np.array([3], dtype=np.int32), + np.array([8], dtype=np.int32), + page_size=4, + dcp_size=2, + dcp_rank=0, + src_page_offset=0, + decode_prefix_len=0, + num_kv_tokens=1, + ) + np.testing.assert_array_equal(plan[0], [3]) + with self.assertRaisesRegex(ValueError, "align"): + _build_page_interleaved_dcp_plan( + np.array([3], dtype=np.int32), + np.array([8], dtype=np.int32), + page_size=4, + dcp_size=2, + dcp_rank=0, + src_page_offset=0, + decode_prefix_len=4, + num_kv_tokens=1, + ) + + def test_accepts_cp_filtered_page_subset(self): + plan = _build_page_interleaved_dcp_plan( + np.array([3], dtype=np.int32), + np.array([4, 6], dtype=np.int32), + page_size=2, + dcp_size=2, + dcp_rank=0, + src_page_offset=2, + decode_prefix_len=0, + num_kv_tokens=4, + ) + np.testing.assert_array_equal(plan[0], [3]) + np.testing.assert_array_equal(plan[1], [6]) + + +class TestAscendPageDcpSend(CustomTestCase): + def test_same_dcp_uses_advertised_page_strides(self): + mgr = object.__new__(AscendKVManager) + mgr.kv_args = SimpleNamespace( + kv_data_ptrs=[1_000, 2_000, 3_000], + kv_item_lens=[20, 32, 16], + ) + mgr.dcp_size = 4 + mgr.pp_size = 1 + mgr.is_hybrid_mla_backend = False + mgr.enable_custom_mem_pool = False + mgr._validate_envelope_kv_layout = Mock() + mgr._transfer_data = Mock(return_value=0) + + result = mgr.send_kvcache( + "decode", + np.array([1], dtype=np.int32), + [10_000, 20_000, 30_000], + np.array([2], dtype=np.int32), + executor=None, + ) + + self.assertEqual(result, 0) + mgr._transfer_data.assert_called_once_with( + "decode", + [ + (1_020, 10_040, 20), + (2_032, 20_064, 32), + (3_016, 30_032, 16), + ], + ) + + def test_sends_local_and_global_entries_without_pack(self): + mgr = object.__new__(AscendKVManager) + mgr.kv_args = SimpleNamespace( + kv_buf_groups=3, + kv_data_ptrs=[1_000, 2_000, 3_000], + kv_item_lens=[20, 8, 4], + page_size=2, + mla_compression_ratios=None, + ) + mgr._dcp_remote_decode_layout = [False, True, True] + mgr._transfer_data = Mock(return_value=0) + + result = mgr.send_kvcache_dcp( + "decode", + np.array([5, 6, 7, 8], dtype=np.int32), + [10_000, 20_000, 30_000], + np.array([7], dtype=np.int32), + dcp_token_item_lens=[10, 4, 2], + dst_dcp_size=4, + dst_dcp_rank=2, + src_page_offset=0, + decode_prefix_len=0, + num_kv_tokens=8, + executor=None, + dst_layer_ids=[], + pack_buffer=None, + dst_kv_item_lens=[20, 32, 16], + ) + + self.assertEqual(result, 0) + mgr._transfer_data.assert_called_once_with( + "decode", + [ + (1_140, 10_140, 20), + (2_040, 20_224, 32), + (3_020, 30_112, 16), + ], + ) + + def test_rejects_incorrect_destination_geometry(self): + mgr = object.__new__(AscendKVManager) + mgr.kv_args = SimpleNamespace( + kv_buf_groups=3, + kv_data_ptrs=[1_000, 2_000, 3_000], + kv_item_lens=[20, 8, 4], + page_size=2, + mla_compression_ratios=None, + ) + mgr._dcp_remote_decode_layout = [False, True, True] + mgr._transfer_data = Mock(return_value=0) + + with self.assertRaisesRegex(RuntimeError, "destination geometry"): + mgr.send_kvcache_dcp( + "decode", + np.array([5], dtype=np.int32), + [10_000, 20_000, 30_000], + np.array([7], dtype=np.int32), + dcp_token_item_lens=[10, 4, 2], + dst_dcp_size=4, + dst_dcp_rank=0, + src_page_offset=0, + decode_prefix_len=0, + num_kv_tokens=2, + executor=None, + dst_layer_ids=[], + dst_kv_item_lens=[20, 8, 16], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/npu/attention/test_npu_dcp.py b/test/registered/unit/npu/attention/test_npu_dcp.py new file mode 100644 index 0000000000..fceb77f165 --- /dev/null +++ b/test/registered/unit/npu/attention/test_npu_dcp.py @@ -0,0 +1,446 @@ +"""CPU-only tests for the NPU DSA+DCP implementation. + +The index arithmetic, buffer geometry, and metadata contracts run on CPU. NPU +extension modules and operators are stubbed because their numerical kernels are +covered by the operator-library tests. +""" + +from __future__ import annotations + +import sys +import types +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import torch + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=8, suite="base-a-test-cpu") + +# Avoid importing the real extensions on CPU. ``layers.dcp`` performs optional +# kernel discovery during import, so temporarily force the platform probe to +# report CPU while loading the modules under test. +sys.modules.setdefault("sgl_kernel_npu", types.ModuleType("sgl_kernel_npu")) +sys.modules.setdefault("torch_npu", MagicMock(name="torch_npu")) +import sglang.srt.utils as _utils +from sglang.srt.utils import common as _common_utils + +_real_is_npu = _common_utils.is_npu +_real_utils_is_npu = _utils.is_npu +_common_utils.is_npu = lambda: False +_utils.is_npu = _common_utils.is_npu +try: + from sglang.srt import runtime_context as rc + from sglang.srt.hardware_backend.npu.attention.dsa_dcp import ( + forward_dcp_sparse_attention, + ) + from sglang.srt.hardware_backend.npu.memory_pool_npu import NPUMLATokenToKVPool + from sglang.srt.layers.dcp.comm import cp_lse_ag_out_rs_mla_npu + from sglang.srt.layers.dcp.layout import ( + filter_dcp_local_chunk_kv_indices, + get_dcp_chain_spec_lens, + get_dcp_lens, + localize_dcp_indices, + remap_dcp_sparse_indices, + ) +finally: + _common_utils.is_npu = _real_is_npu + _utils.is_npu = _real_utils_is_npu + + +class _Mode: + def __init__(self, target_verify: bool): + self._target_verify = target_verify + + def is_target_verify(self): + return self._target_verify + + +class _FakeGroup: + """Deterministic all-to-all transport around the real NPU LSE operator.""" + + def __init__(self, world_size: int, rank: int = 0): + self.world_size = world_size + self.rank_in_group = rank + self.calls = [] + + def all_to_all_single(self, output, input_tensor): + self.calls.append((output.shape, input_tensor.shape)) + # Exchange rank 0/1 chunks. The real npu_attention_update below then + # merges the exchanged partials. + if self.world_size == 2: + output.copy_(input_tensor.flip(0)) + else: + output.copy_(input_tensor) + + +class TestNpuDcpLengths(unittest.TestCase): + PAGE_SIZE = 4 + + def test_owner_counts_cover_every_position(self): + for size in (1, 2, 3, 4, 8): + for length in range(0, 65): + lens = [ + int( + get_dcp_lens( + torch.tensor([length]), + size, + rank, + interleave_size=self.PAGE_SIZE, + )[0] + ) + for rank in range(size) + ] + self.assertEqual(sum(lens), length) + + def test_owner_counts_follow_absolute_start(self): + for size in (2, 3, 4, 8): + for start in (0, 1, 2, 7, 31): + for length in (0, 1, 2, 5, 17, 64): + got = [ + int( + get_dcp_lens( + torch.tensor([length]), + size, + rank, + start=torch.tensor([start]), + interleave_size=self.PAGE_SIZE, + )[0] + ) + for rank in range(size) + ] + expected = [ + sum( + pos // self.PAGE_SIZE % size == rank + for pos in range(start, start + length) + ) + for rank in range(size) + ] + self.assertEqual(got, expected) + + def test_chain_lengths_are_request_major(self): + total = torch.tensor([8, 11], dtype=torch.int32) + got = get_dcp_chain_spec_lens( + total, + 3, + dcp_size=4, + dcp_rank=1, + interleave_size=self.PAGE_SIZE, + ) + # Global frontiers are [6, 7, 8] and [9, 10, 11]. Rank 1 owns the + # complete page at positions [4, 8), hence [2, 3, 4] and [4, 4, 4]. + self.assertEqual(got.tolist(), [2, 3, 4, 4, 4, 4]) + + def test_chain_lengths_zero_short_requests(self): + total = torch.tensor([0, 1, 2], dtype=torch.int64) + for rank in range(4): + got = get_dcp_chain_spec_lens( + total, 3, 4, rank, interleave_size=self.PAGE_SIZE + ) + self.assertEqual(got.tolist(), [0] * 9) + + def test_chain_lengths_reject_non_positive_speculation(self): + with self.assertRaisesRegex(ValueError, "tokens_per_req"): + get_dcp_chain_spec_lens(torch.tensor([4]), 0, 2, 0) + + def test_chunk_filter_uses_start_phase(self): + starts = torch.tensor([1, 7, 12]) + lengths = torch.tensor([5, 4, 6]) + values = torch.arange(15) + with rc.get_parallel().override(dcp_enabled=True, dcp_size=4, dcp_rank=2): + got = filter_dcp_local_chunk_kv_indices(values, starts, lengths) + # The first owned offset is (rank-start) mod size for each chunk. + expected = torch.tensor([1, 8, 11]) + self.assertTrue(torch.equal(got, expected)) + + +class TestNpuDcpSparseIndexRemap(unittest.TestCase): + PAGE_SIZE = 4 + + def test_rank_zero_compacts_valid_entries_stably(self): + topk = torch.tensor([[7, 4, 2, -1, 8, 0], [3, 1, -1, 6, 5, -1]]) + got = remap_dcp_sparse_indices( + topk, + dcp_size=2, + dcp_rank=0, + interleave_size=self.PAGE_SIZE, + ) + expected = torch.tensor([[2, 4, 0, -1, -1, -1], [3, 1, -1, -1, -1, -1]]) + self.assertTrue(torch.equal(got, expected)) + + def test_rank_one_preserves_score_order_before_padding(self): + topk = torch.tensor([[7, 4, 2, -1, 8, 0]]) + got = remap_dcp_sparse_indices( + topk, + dcp_size=2, + dcp_rank=1, + interleave_size=self.PAGE_SIZE, + ) + self.assertEqual(got.tolist(), [[3, 0, -1, -1, -1, -1]]) + + def test_negative_padding_never_becomes_a_valid_index(self): + topk = torch.tensor([[-1, -3, 2, 6, -1]], dtype=torch.int64) + for rank in range(3): + got = remap_dcp_sparse_indices( + topk, 3, rank, interleave_size=self.PAGE_SIZE + ) + self.assertTrue(torch.all((got == -1) | (got >= 0))) + expected_count = int( + ((topk >= 0) & (topk // self.PAGE_SIZE % 3 == rank)).sum() + ) + self.assertEqual(int((got >= 0).sum()), expected_count) + + def test_single_rank_is_identity_object(self): + topk = torch.tensor([[0, 3, -1]]) + got = remap_dcp_sparse_indices(topk, 1, 0) + self.assertTrue(torch.equal(got, topk)) + + def test_wide_integer_values_use_float32_owner_arithmetic(self): + # Values around 2**24 expose accidental float16 arithmetic. The + # implementation deliberately promotes to float32 before remainder. + topk = torch.tensor( + [[16_777_216, 16_777_220, 16_777_218, -1]], dtype=torch.int64 + ) + got = remap_dcp_sparse_indices(topk, 4, 0, interleave_size=self.PAGE_SIZE) + self.assertEqual(got.tolist(), [[4_194_304, 4_194_306, -1, -1]]) + + def test_shape_and_dtype_are_preserved(self): + topk = torch.tensor([[1, 2, 3], [4, 5, 6]], dtype=torch.int32) + got = remap_dcp_sparse_indices(topk, 4, 0, interleave_size=self.PAGE_SIZE) + self.assertEqual(got.shape, topk.shape) + self.assertEqual(got.dtype, topk.dtype) + + def test_global_indices_map_one_page_at_a_time(self): + indices = torch.arange(0, 24) + got = localize_dcp_indices( + indices, dcp_size=3, dcp_rank=1, interleave_size=self.PAGE_SIZE + ) + self.assertEqual( + got.tolist(), + [-1] * 4 + [0, 1, 2, 3] + [-1] * 8 + [4, 5, 6, 7] + [-1] * 4, + ) + + +class TestNpuDcpSparseAttentionContract(unittest.TestCase): + def setUp(self): + self.calls = [] + + def fake_sparse_attention(**kwargs): + self.calls.append(kwargs) + rows, heads, dim = kwargs["query"].shape + out = torch.arange(rows * heads * dim, dtype=torch.float32).view( + rows, heads, dim + ) + softmax_max = torch.zeros(heads, rows, 1) + softmax_sum = torch.full((heads, rows, 1), 2.0) + return out, softmax_max, softmax_sum + + self.fake_op = fake_sparse_attention + + def _metadata(self, rows=2): + return SimpleNamespace( + dcp_block_tables=torch.tensor([[10, 11]] * rows, dtype=torch.int32), + dcp_seq_lens=torch.arange(5, 5 + rows, dtype=torch.int32), + dcp_spec_block_tables=torch.arange(rows * 2).view(rows, 2), + dcp_spec_seq_lens=torch.arange(3, 3 + rows, dtype=torch.int32), + ) + + def _inputs(self, rows=2): + q_nope = torch.ones(rows, 2, 4, dtype=torch.float16) + q_rope = torch.zeros_like(q_nope) + k_nope = torch.ones(rows, 2, 4, dtype=torch.float16) + k_rope = torch.zeros_like(k_nope) + topk = torch.tensor([[0, 2, -1], [1, -1, -1]] * rows)[:rows] + return q_nope, q_rope, k_nope, k_rope, topk + + def test_decode_passes_rank_local_metadata_to_operator(self): + q, rope, k_nope, k_rope, topk = self._inputs() + batch = SimpleNamespace(forward_mode=_Mode(False)) + metadata = self._metadata() + with patch.object( + torch.ops.npu, + "sgl_sparse_flash_attention", + self.fake_op, + create=True, + ): + out, lse = forward_dcp_sparse_attention( + q_nope=q, + q_rope=rope, + k_nope=k_nope, + k_rope=k_rope, + topk_indices=topk, + actual_seq_lengths_query=torch.tensor([1, 2], dtype=torch.int64), + forward_metadata=metadata, + forward_batch=batch, + speculative_num_draft_tokens=None, + scaling=0.25, + ) + call = self.calls[0] + self.assertEqual(call["actual_seq_lengths_query"].dtype, torch.int32) + self.assertEqual(call["actual_seq_lengths_query"].tolist(), [1, 2]) + self.assertIs(call["block_table"], metadata.dcp_block_tables) + self.assertEqual(call["sparse_mode"], 0) + self.assertEqual(call["attention_mode"], 2) + self.assertEqual(call["layout_kv"], "PA_BSND") + self.assertEqual(out.shape, q.shape) + self.assertEqual(lse.shape, (2, 2)) + expected_lse = torch.full((2, 2), torch.log(torch.tensor(2.0))) + self.assertTrue(torch.allclose(lse, expected_lse)) + + def test_verify_replaces_query_lengths_and_uses_spec_metadata(self): + rows = 4 + q, rope, k_nope, k_rope, topk = self._inputs(rows) + batch = SimpleNamespace(forward_mode=_Mode(True)) + metadata = self._metadata(rows=rows) + with patch.object( + torch.ops.npu, + "sgl_sparse_flash_attention", + self.fake_op, + create=True, + ): + forward_dcp_sparse_attention( + q_nope=q, + q_rope=rope, + k_nope=k_nope, + k_rope=k_rope, + topk_indices=topk, + actual_seq_lengths_query=torch.tensor([99]), + forward_metadata=metadata, + forward_batch=batch, + speculative_num_draft_tokens=2, + scaling=1.0, + ) + call = self.calls[0] + self.assertEqual(call["actual_seq_lengths_query"].tolist(), [1, 2, 3, 4]) + self.assertTrue( + torch.equal(call["block_table"], metadata.dcp_spec_block_tables) + ) + self.assertTrue( + torch.equal(call["actual_seq_lengths_kv"], metadata.dcp_spec_seq_lens) + ) + + def test_missing_rank_local_metadata_fails_loudly(self): + q, rope, _, _, topk = self._inputs() + bad = SimpleNamespace(dcp_block_tables=None, dcp_seq_lens=None) + with self.assertRaisesRegex(AssertionError, "rank-local paged-KV"): + forward_dcp_sparse_attention( + q_nope=q, + q_rope=rope, + k_nope=q, + k_rope=rope, + topk_indices=topk, + actual_seq_lengths_query=torch.tensor([1, 2]), + forward_metadata=bad, + forward_batch=SimpleNamespace(forward_mode=_Mode(False)), + speculative_num_draft_tokens=None, + scaling=1.0, + ) + + def test_verify_row_mismatch_fails_before_operator(self): + q, rope, _, _, topk = self._inputs(2) + metadata = self._metadata(rows=3) + with self.assertRaisesRegex(AssertionError, "block-table rows"): + forward_dcp_sparse_attention( + q_nope=q, + q_rope=rope, + k_nope=q, + k_rope=rope, + topk_indices=topk, + actual_seq_lengths_query=torch.tensor([1]), + forward_metadata=metadata, + forward_batch=SimpleNamespace(forward_mode=_Mode(True)), + speculative_num_draft_tokens=2, + scaling=1.0, + ) + self.assertEqual(self.calls, []) + + +class TestNpuDcpBufferAndLseHelpers(unittest.TestCase): + def test_retraction_indices_map_target_kv_to_rank_local_slots(self): + pool = SimpleNamespace(dcp_size=4, dcp_rank=2, page_size=2) + indices = torch.tensor([0, 1, 4, 5, 6, 7, 12, 13], dtype=torch.int64) + got = NPUMLATokenToKVPool._copy_indices_for_buffer( + pool, indices, uses_global_slots=False + ) + self.assertEqual(got.tolist(), [0, 1, 2, 3]) + + def test_retraction_indices_keep_indexer_slots_global(self): + pool = SimpleNamespace(dcp_size=4, dcp_rank=2, page_size=2) + indices = torch.tensor([0, 1, 2, 7], dtype=torch.int64) + got = NPUMLATokenToKVPool._copy_indices_for_buffer( + pool, indices, uses_global_slots=True + ) + self.assertTrue(torch.equal(got, indices)) + + def test_retraction_indices_are_identity_without_dcp(self): + pool = SimpleNamespace(dcp_size=1, dcp_rank=0, page_size=2) + indices = torch.tensor([0, 3, 8], dtype=torch.int64) + got = NPUMLATokenToKVPool._copy_indices_for_buffer( + pool, indices, uses_global_slots=False + ) + self.assertTrue(torch.equal(got, indices)) + + def test_pd_buffer_infos_expand_only_global_page_strides(self): + pool = object.__new__(NPUMLATokenToKVPool) + pool.k_buffer = [torch.empty(4, 8, 1, 2)] + pool.v_buffer = [torch.empty(4, 8, 1, 1)] + pool.dcp_size = 4 + pool.is_draft_worker = False + pool.dsa_kv_cache_store_fp8 = False + pool.index_head_dim = 2 + pool.page_size = 8 + pool.index_page_size = 8 + pool.index_k_buffer = [torch.empty(4, 8, 1, 2)] + pool.index_k_scale_buffer = [torch.empty(4, 8, 1)] + + ptrs, lens, item_lens = pool.get_contiguous_buf_infos() + buffers = ( + pool.k_buffer + + pool.v_buffer + + pool.index_k_buffer + + pool.index_k_scale_buffer + ) + + self.assertEqual(ptrs, [buffer.data_ptr() for buffer in buffers]) + self.assertEqual(lens, [buffer.nbytes for buffer in buffers]) + self.assertEqual( + item_lens, + [ + pool.k_buffer[0][0].nbytes, + pool.v_buffer[0][0].nbytes, + pool.index_k_buffer[0][0].nbytes * pool.dcp_size, + pool.index_k_scale_buffer[0][0].nbytes * pool.dcp_size, + ], + ) + + def test_lse_merge_world_size_one_is_identity(self): + group = _FakeGroup(world_size=1) + out = torch.randn(2, 3, 4) + lse = torch.randn(2, 3) + got = cp_lse_ag_out_rs_mla_npu(out, lse, group) + self.assertIs(got, out) + self.assertEqual(group.calls, []) + + def test_lse_merge_calls_npu_update_for_multi_rank(self): + group = _FakeGroup(world_size=2) + out = torch.arange(2 * 4 * 3, dtype=torch.float16).view(2, 4, 3) + lse = torch.zeros(2, 4) + updates = [] + + def fake_update(lses, outputs, _zero): + updates.append((len(lses), len(outputs))) + return outputs[0], lses[0] + + fake_torch_npu = types.SimpleNamespace(npu_attention_update=fake_update) + with patch.dict(sys.modules, {"torch_npu": fake_torch_npu}): + got = cp_lse_ag_out_rs_mla_npu(out, lse, group) + self.assertEqual(updates, [(2, 2)]) + self.assertEqual(group.calls[0][0], group.calls[0][1]) + self.assertEqual(got.shape, (2, 2, 3)) + self.assertEqual(got.dtype, out.dtype) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 67010b1306..4949abe03a 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -1136,7 +1136,7 @@ class TestLoadBalanceMethod(unittest.TestCase): dcp_size=4, ) with self.assertRaisesRegex( - ValueError, "mooncake, nixl, or fake for synthetic benchmarking" + ValueError, "mooncake, nixl, ascend, or fake for synthetic benchmarking" ): handle_pd_disaggregation(server_args) @@ -4004,6 +4004,10 @@ class TestDcpCommBackendDefault(CustomTestCase): def test_no_dcp_is_ag_rs(self): self.assertEqual(self._resolved(dcp_size=1), "ag_rs") + @override_platform(is_npu=True, is_cuda=False, is_hip=False) + def test_a5_npu_supports_dcp(self): + self.assertEqual(self._resolved(dcp_size=2), "ag_rs") + @override_platform(is_cuda=True, is_hip=False) def test_fi_a2a_where_supported(self): with patch(