[NPU] Add decode context parallel support for dsa models (#37787)
Co-authored-by: sglang-npu-bot <sglangnpu@163.com>
This commit is contained in:
co-authored by
sglang-npu-bot
parent
8c75ad9e62
commit
ec0006673a
@@ -164,6 +164,13 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
hf_config = model_config.hf_config
|
||||
model_arch = hf_config.architectures[0]
|
||||
|
||||
if get_platform().is_npu and cfg.dcp_size > 1 and not is_deepseek_dsa(hf_config):
|
||||
raise ValueError(
|
||||
"NPU decode context parallelism is currently implemented only for "
|
||||
"DeepSeek DSA models; got "
|
||||
f"{model_arch}. Set --decode-context-parallel-size=1 or use a DSA model."
|
||||
)
|
||||
|
||||
if model_arch == "InternS2MobiusForConditionalGeneration":
|
||||
unsupported = []
|
||||
if cfg.pp_size != 1:
|
||||
|
||||
@@ -74,11 +74,12 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
if cfg.disaggregation_transfer_backend not in (
|
||||
"mooncake",
|
||||
"nixl",
|
||||
"ascend",
|
||||
"fake",
|
||||
):
|
||||
raise ValueError(
|
||||
"PD decode DCP requires --disaggregation-transfer-backend "
|
||||
"mooncake, nixl, or fake for synthetic benchmarking, got "
|
||||
"mooncake, nixl, ascend, or fake for synthetic benchmarking, got "
|
||||
f"{cfg.disaggregation_transfer_backend!r}."
|
||||
)
|
||||
|
||||
|
||||
@@ -34,7 +34,81 @@ class AscendStateType(str, enum.Enum):
|
||||
_DSV4_KVCACHE_STATE_TYPES = tuple(AscendStateType)
|
||||
|
||||
|
||||
def _build_page_interleaved_dcp_plan(
|
||||
src_page_indices: npt.NDArray[np.int32],
|
||||
dst_page_indices: npt.NDArray[np.int32],
|
||||
*,
|
||||
page_size: int,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
src_page_offset: int,
|
||||
decode_prefix_len: int,
|
||||
num_kv_tokens: int,
|
||||
) -> Tuple[npt.NDArray[np.int64], ...]:
|
||||
"""Map physical prefill pages to local/global decode page slots."""
|
||||
if not 0 <= dcp_rank < dcp_size:
|
||||
raise ValueError(f"Invalid DCP rank {dcp_rank} for size {dcp_size}")
|
||||
virtual_page_size = page_size * dcp_size
|
||||
if decode_prefix_len % virtual_page_size:
|
||||
raise ValueError(
|
||||
"Ascend PD DCP requires decode_prefix_len to align to the virtual "
|
||||
f"page size ({virtual_page_size}), got {decode_prefix_len}"
|
||||
)
|
||||
if src_page_offset < 0 or num_kv_tokens < 0:
|
||||
raise ValueError(
|
||||
"Ascend PD DCP page offset and token count must be nonnegative"
|
||||
)
|
||||
|
||||
src_pages = np.asarray(src_page_indices, dtype=np.int64)
|
||||
dst_pages = np.asarray(dst_page_indices, dtype=np.int64)
|
||||
max_src_pages = (num_kv_tokens + page_size - 1) // page_size
|
||||
if src_pages.size > max_src_pages:
|
||||
raise ValueError(
|
||||
"Ascend PD DCP source page count exceeds the token count: "
|
||||
f"pages={src_pages.size}, tokens={num_kv_tokens}, page_size={page_size}"
|
||||
)
|
||||
if src_pages.size == 0:
|
||||
empty = np.empty((0,), dtype=np.int64)
|
||||
return empty, empty.copy(), empty.copy(), empty.copy()
|
||||
|
||||
# CP may assign this sender only a contiguous subset of the chunk pages;
|
||||
# index_slice.start still carries that subset's suffix-relative offset.
|
||||
relative_pages = src_page_offset + np.arange(src_pages.size, dtype=np.int64)
|
||||
dst_positions = relative_pages // dcp_size
|
||||
if dst_positions[-1] >= dst_pages.size:
|
||||
raise ValueError(
|
||||
"Ascend PD DCP destination does not contain enough virtual pages: "
|
||||
f"required={dst_positions[-1] + 1}, available={dst_pages.size}"
|
||||
)
|
||||
dst_super_pages = dst_pages[dst_positions]
|
||||
owners = (decode_prefix_len // page_size + relative_pages) % dcp_size
|
||||
local = owners == dcp_rank
|
||||
|
||||
return (
|
||||
src_pages[local],
|
||||
dst_super_pages[local],
|
||||
src_pages,
|
||||
dst_super_pages * dcp_size + owners,
|
||||
)
|
||||
|
||||
|
||||
class AscendKVManager(MooncakeKVManager):
|
||||
def __init__(
|
||||
self,
|
||||
args,
|
||||
disaggregation_mode,
|
||||
server_args,
|
||||
is_mla_backend: Optional[bool] = False,
|
||||
dcp_remote_decode_layout=None,
|
||||
):
|
||||
self._dcp_remote_decode_layout = (
|
||||
None if dcp_remote_decode_layout is None else list(dcp_remote_decode_layout)
|
||||
)
|
||||
super().__init__(args, disaggregation_mode, server_args, is_mla_backend)
|
||||
|
||||
def _is_npu_dsa_layout(self) -> bool:
|
||||
return getattr(self.kv_args, "kv_buf_groups", 1) == 3
|
||||
|
||||
def _requires_exact_state_index_match(self, st: StateType) -> bool:
|
||||
return (
|
||||
super()._requires_exact_state_index_match(st)
|
||||
@@ -67,6 +141,47 @@ class AscendKVManager(MooncakeKVManager):
|
||||
if ptrs:
|
||||
self.engine.batch_register(ptrs, lens)
|
||||
|
||||
def requires_dcp_relayout(self, dst_dcp_size: int, dst_dcp_rank: int) -> bool:
|
||||
if self._is_npu_dsa_layout() and self.dcp_size != dst_dcp_size:
|
||||
if self.dcp_size == 1 and dst_dcp_size > 1:
|
||||
return True
|
||||
raise RuntimeError(
|
||||
"NPU DSA PD supports prefill DCP 1 -> decode DCP N only, got "
|
||||
f"prefill={self.dcp_size}, decode={dst_dcp_size}"
|
||||
)
|
||||
return super().requires_dcp_relayout(dst_dcp_size, dst_dcp_rank)
|
||||
|
||||
def _get_dcp_remote_decode_layout(self) -> List[bool]:
|
||||
layout = self._dcp_remote_decode_layout
|
||||
if layout is None or len(layout) != len(self.kv_args.kv_data_ptrs):
|
||||
raise RuntimeError(
|
||||
"Ascend PD DCP layout does not match its source KV entries"
|
||||
)
|
||||
return layout
|
||||
|
||||
def prepare_dcp_token_item_lens(
|
||||
self, dst_page_item_lens: List[Optional[int]], dst_dcp_size: int
|
||||
) -> List[int]:
|
||||
if not self._is_npu_dsa_layout():
|
||||
return super().prepare_dcp_token_item_lens(dst_page_item_lens, dst_dcp_size)
|
||||
self._get_dcp_remote_decode_layout()
|
||||
page_size = self.kv_args.page_size
|
||||
token_item_lens = []
|
||||
for entry, item_len in enumerate(self.kv_args.kv_item_lens):
|
||||
token_item_len, remainder = divmod(item_len, page_size)
|
||||
if remainder:
|
||||
raise RuntimeError(
|
||||
f"Ascend PD DCP source entry {entry} is not page aligned"
|
||||
)
|
||||
token_item_lens.append(token_item_len)
|
||||
return token_item_lens
|
||||
|
||||
def _init_dcp_pack_buffers_once(
|
||||
self, dcp_size: int, *, include_draft: bool = False
|
||||
) -> None:
|
||||
# Page-interleaved NPU DCP transfers whole pages directly.
|
||||
self._dcp_pack_buffers = []
|
||||
|
||||
def get_mla_kv_ptrs_with_pp(
|
||||
self, src_kv_ptrs: List[int], dst_kv_ptrs: List[int], state_type=None
|
||||
) -> Tuple[List[int], List[int], int]:
|
||||
@@ -289,6 +404,109 @@ class AscendKVManager(MooncakeKVManager):
|
||||
|
||||
return 0
|
||||
|
||||
def send_kvcache_dcp(
|
||||
self,
|
||||
mooncake_session_id: str,
|
||||
prefill_kv_indices: npt.NDArray[np.int32],
|
||||
dst_kv_ptrs: list[int],
|
||||
dst_kv_indices: npt.NDArray[np.int32],
|
||||
*,
|
||||
dcp_token_item_lens: List[int],
|
||||
dst_dcp_size: int,
|
||||
dst_dcp_rank: int,
|
||||
src_page_offset: int,
|
||||
decode_prefix_len: int,
|
||||
num_kv_tokens: int,
|
||||
executor: concurrent.futures.ThreadPoolExecutor,
|
||||
dst_layer_ids: List[int],
|
||||
pack_buffer=None,
|
||||
dst_kv_item_lens: Optional[List[int]] = None,
|
||||
dst_tp_rank: int = 0,
|
||||
dst_attn_tp_size: Optional[int] = None,
|
||||
) -> int:
|
||||
if not self._is_npu_dsa_layout():
|
||||
return super().send_kvcache_dcp(
|
||||
mooncake_session_id,
|
||||
prefill_kv_indices,
|
||||
dst_kv_ptrs,
|
||||
dst_kv_indices,
|
||||
dcp_token_item_lens=dcp_token_item_lens,
|
||||
dst_dcp_size=dst_dcp_size,
|
||||
dst_dcp_rank=dst_dcp_rank,
|
||||
src_page_offset=src_page_offset,
|
||||
decode_prefix_len=decode_prefix_len,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
executor=executor,
|
||||
dst_layer_ids=dst_layer_ids,
|
||||
pack_buffer=pack_buffer,
|
||||
dst_kv_item_lens=dst_kv_item_lens,
|
||||
dst_tp_rank=dst_tp_rank,
|
||||
dst_attn_tp_size=dst_attn_tp_size,
|
||||
)
|
||||
|
||||
src_kv_ptrs = self.kv_args.kv_data_ptrs
|
||||
_, dst_kv_ptrs, _ = self.get_mla_kv_ptrs_with_pp(src_kv_ptrs, dst_kv_ptrs)
|
||||
if dst_kv_item_lens:
|
||||
_, dst_kv_item_lens, _ = self.get_mla_kv_ptrs_with_pp(
|
||||
self.kv_args.kv_item_lens, dst_kv_item_lens
|
||||
)
|
||||
|
||||
layout = self._get_dcp_remote_decode_layout()
|
||||
num_entries = len(src_kv_ptrs)
|
||||
if not (
|
||||
len(dst_kv_ptrs)
|
||||
== len(self.kv_args.kv_item_lens)
|
||||
== len(dcp_token_item_lens)
|
||||
== len(layout)
|
||||
== num_entries
|
||||
):
|
||||
raise RuntimeError("Ascend PD DCP KV entry metadata is inconsistent")
|
||||
if dst_kv_item_lens and len(dst_kv_item_lens) != num_entries:
|
||||
raise RuntimeError("Ascend PD DCP destination KV metadata is inconsistent")
|
||||
|
||||
local_src, local_dst, global_src, global_dst = _build_page_interleaved_dcp_plan(
|
||||
prefill_kv_indices,
|
||||
dst_kv_indices,
|
||||
page_size=self.kv_args.page_size,
|
||||
dcp_size=dst_dcp_size,
|
||||
dcp_rank=dst_dcp_rank,
|
||||
src_page_offset=src_page_offset,
|
||||
decode_prefix_len=decode_prefix_len,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
)
|
||||
|
||||
transfer_blocks: List[Tuple[int, int, int]] = []
|
||||
for entry, uses_global_slots in enumerate(layout):
|
||||
page_bytes = self.kv_args.kv_item_lens[entry]
|
||||
if page_bytes != dcp_token_item_lens[entry] * self.kv_args.page_size:
|
||||
raise RuntimeError(
|
||||
f"Ascend PD DCP source geometry differs at entry {entry}"
|
||||
)
|
||||
if dst_kv_item_lens:
|
||||
expected_dst_bytes = page_bytes * (
|
||||
dst_dcp_size if uses_global_slots else 1
|
||||
)
|
||||
if dst_kv_item_lens[entry] != expected_dst_bytes:
|
||||
raise RuntimeError(
|
||||
"Ascend PD DCP destination geometry differs at entry "
|
||||
f"{entry}: expected={expected_dst_bytes}, "
|
||||
f"actual={dst_kv_item_lens[entry]}"
|
||||
)
|
||||
|
||||
src_pages = global_src if uses_global_slots else local_src
|
||||
dst_pages = global_dst if uses_global_slots else local_dst
|
||||
src_groups, dst_groups = group_concurrent_contiguous(src_pages, dst_pages)
|
||||
for src_group, dst_group in zip(src_groups, dst_groups):
|
||||
transfer_blocks.append(
|
||||
(
|
||||
src_kv_ptrs[entry] + int(src_group[0]) * page_bytes,
|
||||
dst_kv_ptrs[entry] + int(dst_group[0]) * page_bytes,
|
||||
len(src_group) * page_bytes,
|
||||
)
|
||||
)
|
||||
|
||||
return self._transfer_data(mooncake_session_id, transfer_blocks)
|
||||
|
||||
def _is_generic_kvcache_state_type(self, st) -> bool:
|
||||
# DSV4 per-pool components also use the page-indexed send path.
|
||||
return (
|
||||
|
||||
@@ -290,6 +290,13 @@ class PrefillBootstrapQueue:
|
||||
kv_item_lens += draft_kv_item_lens
|
||||
num_draft_entries = len(draft_kv_data_ptrs)
|
||||
|
||||
dcp_remote_decode_layout = []
|
||||
if self.transfer_backend == TransferBackend.ASCEND:
|
||||
for pool in (self.token_to_kv_pool, draft_kv_pool):
|
||||
get_layout = getattr(pool, "get_dcp_remote_decode_layout", None)
|
||||
if get_layout is not None:
|
||||
dcp_remote_decode_layout.extend(get_layout())
|
||||
|
||||
kv_args.kv_data_ptrs = kv_data_ptrs
|
||||
kv_args.kv_data_lens = kv_data_lens
|
||||
kv_args.kv_item_lens = kv_item_lens
|
||||
@@ -323,11 +330,17 @@ class PrefillBootstrapQueue:
|
||||
)
|
||||
|
||||
kv_manager_class = get_kv_class(self.transfer_backend, KVClassType.MANAGER)
|
||||
kv_manager_kwargs = (
|
||||
{"dcp_remote_decode_layout": dcp_remote_decode_layout}
|
||||
if self.transfer_backend == TransferBackend.ASCEND
|
||||
else {}
|
||||
)
|
||||
kv_manager = kv_manager_class(
|
||||
kv_args,
|
||||
DisaggregationMode.PREFILL,
|
||||
self.scheduler.server_args,
|
||||
self.is_mla_backend,
|
||||
**kv_manager_kwargs,
|
||||
)
|
||||
# Pass KV pool tensor refs to the manager for GPU gather (staging mode)
|
||||
if (
|
||||
|
||||
@@ -1558,7 +1558,7 @@ def setup_state_kv_args(
|
||||
tail_item_lens = tail_item_lens + draft_tail_item_lens
|
||||
if isinstance(token_to_kv_pool, NPUMLATokenToKVPool):
|
||||
kv_args.kv_buf_groups = (
|
||||
len(kv_args.kv_data_ptrs) // token_to_kv_pool.layer_num
|
||||
3 if token_to_kv_pool.index_head_dim is not None else 2
|
||||
)
|
||||
kv_args.hidden_kv_layers = total_kv_layers
|
||||
kv_args.draft_kv_layers = (
|
||||
|
||||
@@ -2639,12 +2639,12 @@ def initialize_model_parallel(
|
||||
raise RuntimeError(
|
||||
f"decode_context_parallel_size ({decode_context_parallel_size}) must be >= 1"
|
||||
)
|
||||
if decode_context_parallel_size > 1 and not (is_hip() or is_cuda()):
|
||||
if decode_context_parallel_size > 1 and not (is_hip() or is_cuda() or _is_npu):
|
||||
raise RuntimeError(
|
||||
"Decode context parallel (decode_context_parallel_size > 1) is "
|
||||
"currently only supported on the AMD HIP platform or CUDA platform, but got "
|
||||
f"decode_context_parallel_size ({decode_context_parallel_size}) "
|
||||
"on a non-HIP or non-CUDA platform."
|
||||
"currently only supported on the AMD HIP, CUDA, or NPU "
|
||||
"platform, but got decode_context_parallel_size "
|
||||
f"({decode_context_parallel_size}) on an unsupported platform."
|
||||
)
|
||||
if tensor_model_parallel_size % decode_context_parallel_size != 0:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -15,6 +15,9 @@ from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.hardware_backend.npu.attention.ascend_torch_native_backend import (
|
||||
AscendTorchNativeAttnBackend,
|
||||
)
|
||||
from sglang.srt.hardware_backend.npu.attention.dsa_dcp import (
|
||||
forward_dcp_sparse_attention,
|
||||
)
|
||||
from sglang.srt.hardware_backend.npu.attention.mla_cache import gather_mla_cache_pages
|
||||
from sglang.srt.hardware_backend.npu.attention.mla_preprocess import (
|
||||
is_fia_nz,
|
||||
@@ -26,6 +29,10 @@ from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import (
|
||||
)
|
||||
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.dcp.layout import (
|
||||
get_dcp_chain_spec_lens,
|
||||
get_dcp_lens,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.layers.utils.cp_utils import cp_all_gather_rerange_kv_cache
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
@@ -100,6 +107,16 @@ class ForwardMetadata:
|
||||
actual_seq_lengths_q_pa_cpu: Optional[torch.Tensor] = None
|
||||
actual_seq_lengths_kv: Optional[torch.Tensor] = None
|
||||
|
||||
# DCP decode inputs
|
||||
dcp_seq_lens_cpu_int: Optional[torch.Tensor] = None
|
||||
dcp_seq_lens: Optional[torch.Tensor] = None
|
||||
dcp_block_tables: Optional[torch.Tensor] = None
|
||||
dcp_origin_out_cache_loc: Optional[torch.Tensor] = None
|
||||
# DCP target-verify inputs, expanded to one row per speculative query.
|
||||
dcp_spec_seq_lens_cpu_int: Optional[torch.Tensor] = None
|
||||
dcp_spec_seq_lens: Optional[torch.Tensor] = None
|
||||
dcp_spec_block_tables: Optional[torch.Tensor] = None
|
||||
|
||||
# swa attention mask for graph mode decode
|
||||
swa_mask: Optional[torch.Tensor] = None
|
||||
|
||||
@@ -309,6 +326,7 @@ class AscendAttnBackend(AttentionBackend):
|
||||
super().__init__()
|
||||
self.forward_metadata = None
|
||||
self.device = model_runner.device
|
||||
self.is_draft_worker = model_runner.is_draft_worker
|
||||
self.speculative_step_id = speculative_step_id
|
||||
self.speculative_step_offset_npu = torch.tensor(
|
||||
speculative_step_id + 1, device="npu"
|
||||
@@ -444,6 +462,40 @@ class AscendAttnBackend(AttentionBackend):
|
||||
v = layer.v_head_dim
|
||||
return (d == v and d in (128, 192, 256)) or (d == 192 and v == 128)
|
||||
|
||||
def _get_kv_lens_and_block_tables(
|
||||
self,
|
||||
kv_lens_cpu: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
is_spec: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Build rank-local paged KV metadata for NPU DSA DCP."""
|
||||
parallel = get_parallel()
|
||||
if is_spec:
|
||||
local_kv_lens = get_dcp_chain_spec_lens(
|
||||
kv_lens_cpu,
|
||||
self.speculative_num_draft_tokens,
|
||||
parallel.attn_dcp_size,
|
||||
parallel.attn_dcp_rank,
|
||||
interleave_size=self.page_size,
|
||||
)
|
||||
else:
|
||||
local_kv_lens = get_dcp_lens(
|
||||
kv_lens_cpu,
|
||||
parallel.attn_dcp_size,
|
||||
parallel.attn_dcp_rank,
|
||||
interleave_size=self.page_size,
|
||||
).int()
|
||||
page_stride = self.page_size * parallel.attn_dcp_size
|
||||
max_len = int(kv_lens_cpu.max().item()) if kv_lens_cpu.numel() else 0
|
||||
block_tables = (
|
||||
self.req_to_token[req_pool_indices, :max_len:page_stride] // page_stride
|
||||
).to(torch.int32)
|
||||
if is_spec:
|
||||
block_tables = block_tables.repeat_interleave(
|
||||
self.speculative_num_draft_tokens, dim=0
|
||||
)
|
||||
return local_kv_lens, block_tables
|
||||
|
||||
def update_verify_buffers_to_fill_after_draft(
|
||||
self, spec_info: SpecInput, cuda_graph_bs: Optional[int]
|
||||
):
|
||||
@@ -474,6 +526,7 @@ class AscendAttnBackend(AttentionBackend):
|
||||
forward_mode=forward_batch.forward_mode,
|
||||
spec_info=forward_batch.spec_info,
|
||||
out_cache_loc=forward_batch.out_cache_loc,
|
||||
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
|
||||
)
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
@@ -591,6 +644,40 @@ class AscendAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if get_parallel().dcp_enabled and not self.is_draft_worker:
|
||||
# Draft workers retain allocator-global cache slots and use the non-DCP
|
||||
# attention path, so only target workers need DCP-specific metadata.
|
||||
self.forward_metadata.dcp_origin_out_cache_loc = (
|
||||
forward_batch.origin_out_cache_loc
|
||||
)
|
||||
if forward_batch.forward_mode.is_target_verify():
|
||||
(
|
||||
self.forward_metadata.dcp_spec_seq_lens_cpu_int,
|
||||
self.forward_metadata.dcp_spec_block_tables,
|
||||
) = self._get_kv_lens_and_block_tables(
|
||||
kv_lens_cpu=self.forward_metadata.seq_lens_cpu_int,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
is_spec=True,
|
||||
)
|
||||
self.forward_metadata.dcp_spec_seq_lens = (
|
||||
self.forward_metadata.dcp_spec_seq_lens_cpu_int.to(
|
||||
device=self.device, dtype=torch.int32
|
||||
)
|
||||
)
|
||||
else:
|
||||
(
|
||||
self.forward_metadata.dcp_seq_lens_cpu_int,
|
||||
self.forward_metadata.dcp_block_tables,
|
||||
) = self._get_kv_lens_and_block_tables(
|
||||
kv_lens_cpu=self.forward_metadata.seq_lens_cpu_int,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
)
|
||||
self.forward_metadata.dcp_seq_lens = (
|
||||
self.forward_metadata.dcp_seq_lens_cpu_int.to(
|
||||
device=self.device, dtype=torch.int32
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
self.use_mla
|
||||
and forward_batch.forward_mode.is_extend()
|
||||
@@ -639,6 +726,40 @@ class AscendAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
),
|
||||
}
|
||||
# DCP decode needs its own block_table (different stride: dcp_page_size).
|
||||
if get_parallel().dcp_enabled and not self.is_draft_worker:
|
||||
dcp_page_size = self.page_size * get_parallel().attn_dcp_size
|
||||
max_dcp_seq_pages = (total_context_len + dcp_page_size - 1) // dcp_page_size
|
||||
# The captured DSA indexer cache-write operation reads from this fixed
|
||||
# buffer. Replay refreshes its allocator-global slot ids before execution.
|
||||
self.graph_metadata["dcp_origin_out_cache_loc"] = torch.zeros(
|
||||
max_num_tokens,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
if self.speculative_num_draft_tokens is not None:
|
||||
self.graph_metadata["dcp_spec_block_tables"] = torch.zeros(
|
||||
(
|
||||
max_bs * self.speculative_num_draft_tokens,
|
||||
max_dcp_seq_pages,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.graph_metadata["dcp_spec_seq_lens"] = torch.zeros(
|
||||
max_bs * self.speculative_num_draft_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
self.graph_metadata["dcp_block_tables"] = torch.zeros(
|
||||
(max_bs, max_dcp_seq_pages),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.graph_metadata["dcp_seq_lens"] = torch.zeros(
|
||||
max_bs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
if self.is_hybrid_swa:
|
||||
self.graph_metadata["block_tables_swa"] = torch.empty(
|
||||
(max_bs, total_context_len // self.page_size),
|
||||
@@ -694,6 +815,26 @@ class AscendAttnBackend(AttentionBackend):
|
||||
torch.npu.current_stream(self.device), self.device
|
||||
)
|
||||
metadata.block_tables = self.graph_metadata["block_tables"][:bs, :]
|
||||
if get_parallel().dcp_enabled:
|
||||
if "dcp_origin_out_cache_loc" in self.graph_metadata:
|
||||
metadata.dcp_origin_out_cache_loc = self.graph_metadata[
|
||||
"dcp_origin_out_cache_loc"
|
||||
][: out_cache_loc.shape[0]]
|
||||
if not self.is_draft_worker and "dcp_block_tables" in self.graph_metadata:
|
||||
metadata.dcp_block_tables = self.graph_metadata["dcp_block_tables"][:bs]
|
||||
metadata.dcp_seq_lens = self.graph_metadata["dcp_seq_lens"][:bs]
|
||||
if (
|
||||
get_parallel().dcp_enabled
|
||||
and "dcp_spec_block_tables" in self.graph_metadata
|
||||
and forward_mode.is_target_verify()
|
||||
):
|
||||
num_spec_rows = bs * self.speculative_num_draft_tokens
|
||||
metadata.dcp_spec_block_tables = self.graph_metadata[
|
||||
"dcp_spec_block_tables"
|
||||
][:num_spec_rows]
|
||||
metadata.dcp_spec_seq_lens = self.graph_metadata["dcp_spec_seq_lens"][
|
||||
:num_spec_rows
|
||||
]
|
||||
if self.is_hybrid_swa:
|
||||
metadata.block_tables_swa = self.graph_metadata["block_tables_swa"][:bs, :]
|
||||
metadata.swa_mask = self.graph_metadata["swa_mask"][:bs, :, :]
|
||||
@@ -765,6 +906,7 @@ class AscendAttnBackend(AttentionBackend):
|
||||
forward_mode: ForwardMode,
|
||||
spec_info: Optional[SpecInput],
|
||||
out_cache_loc: Optional[torch.Tensor] = None,
|
||||
origin_out_cache_loc: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""Shared capture+replay body for the cuda-graph init path.
|
||||
|
||||
@@ -779,11 +921,17 @@ class AscendAttnBackend(AttentionBackend):
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc)
|
||||
)
|
||||
max_len = seq_lens_cpu[:bs].max().item()
|
||||
# Compute the host-side KV lengths visible to this attention step. Target
|
||||
# verify adds the draft block except for DFlash, whose seq_lens_cpu already
|
||||
# includes it; speculative decode adds tokens through the current draft step.
|
||||
attention_kv_lens_cpu = seq_lens_cpu[:bs].int()
|
||||
if forward_mode.is_target_verify() and not _is_dflash_verify(spec_info):
|
||||
max_len += self.speculative_num_draft_tokens
|
||||
attention_kv_lens_cpu = (
|
||||
attention_kv_lens_cpu + self.speculative_num_draft_tokens
|
||||
)
|
||||
elif forward_mode.is_decode_or_idle() and spec_info is not None:
|
||||
max_len += self.speculative_step_id + 1
|
||||
attention_kv_lens_cpu = attention_kv_lens_cpu + self.speculative_step_id + 1
|
||||
max_len = attention_kv_lens_cpu.max().item()
|
||||
max_seq_pages = (max_len + self.page_size - 1) // self.page_size
|
||||
|
||||
if self.is_hybrid_swa:
|
||||
@@ -826,6 +974,58 @@ class AscendAttnBackend(AttentionBackend):
|
||||
)
|
||||
|
||||
metadata.block_tables[:bs, max_seq_pages:].fill_(0)
|
||||
|
||||
if get_parallel().dcp_enabled and not self.is_draft_worker:
|
||||
if "dcp_origin_out_cache_loc" in self.graph_metadata:
|
||||
buffer = self.graph_metadata["dcp_origin_out_cache_loc"]
|
||||
if origin_out_cache_loc is None:
|
||||
buffer.zero_()
|
||||
else:
|
||||
num_tokens = origin_out_cache_loc.shape[0]
|
||||
assert num_tokens <= buffer.shape[0], (
|
||||
"NPU DSA+DCP origin_out_cache_loc exceeds its graph buffer: "
|
||||
f"{num_tokens} > {buffer.shape[0]}"
|
||||
)
|
||||
buffer[:num_tokens].copy_(origin_out_cache_loc)
|
||||
buffer[num_tokens:].zero_()
|
||||
# DCP decode/speculative paths use rank-local KV lengths and a block
|
||||
# table whose stride is page_size * dcp_world_size. Draft attention
|
||||
# uses the ordinary full-KV metadata above instead.
|
||||
if forward_mode.is_target_verify():
|
||||
(
|
||||
metadata.dcp_spec_seq_lens_cpu_int,
|
||||
dcp_spec_block_tables,
|
||||
) = self._get_kv_lens_and_block_tables(
|
||||
kv_lens_cpu=attention_kv_lens_cpu,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
is_spec=True,
|
||||
)
|
||||
metadata.dcp_spec_seq_lens.copy_(
|
||||
metadata.dcp_spec_seq_lens_cpu_int.to(
|
||||
device=metadata.dcp_spec_seq_lens.device
|
||||
)
|
||||
)
|
||||
dcp_pages = dcp_spec_block_tables.shape[1]
|
||||
metadata.dcp_spec_block_tables[:, :dcp_pages].copy_(
|
||||
dcp_spec_block_tables
|
||||
)
|
||||
metadata.dcp_spec_block_tables[:, dcp_pages:].fill_(0)
|
||||
else:
|
||||
(
|
||||
metadata.dcp_seq_lens_cpu_int,
|
||||
dcp_block_tables,
|
||||
) = self._get_kv_lens_and_block_tables(
|
||||
kv_lens_cpu=attention_kv_lens_cpu,
|
||||
req_pool_indices=req_pool_indices[:bs],
|
||||
)
|
||||
metadata.dcp_seq_lens.copy_(
|
||||
metadata.dcp_seq_lens_cpu_int.to(
|
||||
device=metadata.dcp_seq_lens.device
|
||||
)
|
||||
)
|
||||
dcp_pages = dcp_block_tables.shape[1]
|
||||
metadata.dcp_block_tables[:bs, :dcp_pages].copy_(dcp_block_tables)
|
||||
metadata.dcp_block_tables[:bs, dcp_pages:].fill_(0)
|
||||
metadata.block_tables[bs:, :].fill_(0)
|
||||
|
||||
if forward_mode.is_target_verify():
|
||||
@@ -1225,6 +1425,19 @@ class AscendAttnBackend(AttentionBackend):
|
||||
if topk_indices is not None:
|
||||
topk_indices = self._pad_topk_indices(topk_indices, q_nope.shape[0])
|
||||
topk_indices = _expand_dsa_sparse_indices(topk_indices)
|
||||
if get_parallel().dcp_enabled and not self.is_draft_worker:
|
||||
return forward_dcp_sparse_attention(
|
||||
q_nope=q_nope,
|
||||
q_rope=q_pe,
|
||||
k_nope=k_nope,
|
||||
k_rope=k_pe,
|
||||
topk_indices=topk_indices,
|
||||
actual_seq_lengths_query=actual_seq_qlen,
|
||||
forward_metadata=self.forward_metadata,
|
||||
forward_batch=forward_batch,
|
||||
speculative_num_draft_tokens=self.speculative_num_draft_tokens,
|
||||
scaling=layer.scaling,
|
||||
)
|
||||
if self.kv_cache_dtype == torch.float8_e4m3fn:
|
||||
assert q_nope.dtype == q_pe.dtype == torch.bfloat16
|
||||
packed = k_nope.view(torch.float8_e4m3fn)
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""NPU DSA adaptation for token-sharded decode context parallelism."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import sgl_kernel_npu # noqa: F401 Registers torch.ops.sgl_kernel_npu.
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
|
||||
def forward_dcp_sparse_attention(
|
||||
*,
|
||||
q_nope: torch.Tensor,
|
||||
q_rope: torch.Tensor,
|
||||
k_nope: torch.Tensor,
|
||||
k_rope: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
actual_seq_lengths_query: torch.Tensor,
|
||||
forward_metadata,
|
||||
forward_batch: ForwardBatch,
|
||||
speculative_num_draft_tokens: int | None,
|
||||
scaling: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute one rank's sparse partial attention and natural-log LSE."""
|
||||
|
||||
is_target_verify = forward_batch.forward_mode.is_target_verify()
|
||||
if is_target_verify:
|
||||
block_tables = forward_metadata.dcp_spec_block_tables
|
||||
local_lens = forward_metadata.dcp_spec_seq_lens
|
||||
else:
|
||||
block_tables = forward_metadata.dcp_block_tables
|
||||
local_lens = forward_metadata.dcp_seq_lens
|
||||
|
||||
assert block_tables is not None, (
|
||||
"NPU DSA+DCP sparse attention requires rank-local paged-KV metadata"
|
||||
)
|
||||
if is_target_verify:
|
||||
# DCP verify presents every speculative query as an independent
|
||||
# single-query sequence. Its KV frontier and page-table row therefore
|
||||
# stay expanded in request-major [bs * draft_token_num] order.
|
||||
num_query_rows = q_nope.shape[0]
|
||||
assert block_tables.shape[0] == num_query_rows, (
|
||||
"NPU DSA+DCP verify block-table rows must match query rows: "
|
||||
f"{block_tables.shape[0]} != {num_query_rows}"
|
||||
)
|
||||
assert local_lens.shape[0] == num_query_rows, (
|
||||
"NPU DSA+DCP verify KV-length rows must match query rows: "
|
||||
f"{local_lens.shape[0]} != {num_query_rows}"
|
||||
)
|
||||
actual_seq_lengths_query = torch.arange(
|
||||
1,
|
||||
num_query_rows + 1,
|
||||
device=q_nope.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
else:
|
||||
actual_seq_lengths_query = actual_seq_lengths_query.to(
|
||||
device=q_nope.device, dtype=torch.int32
|
||||
)
|
||||
attn_out, softmax_max, softmax_sum = torch.ops.npu.sgl_sparse_flash_attention(
|
||||
query=q_nope,
|
||||
key=k_nope,
|
||||
value=k_nope,
|
||||
query_rope=q_rope,
|
||||
key_rope=k_rope,
|
||||
sparse_indices=topk_indices,
|
||||
scale_value=scaling,
|
||||
actual_seq_lengths_query=actual_seq_lengths_query,
|
||||
actual_seq_lengths_kv=local_lens,
|
||||
block_table=block_tables,
|
||||
sparse_block_size=1,
|
||||
layout_query="TND",
|
||||
layout_kv="PA_BSND",
|
||||
# Global top-k already enforces causal visibility. Local indices no
|
||||
# longer carry enough information for the kernel's causal sparse mode.
|
||||
sparse_mode=0,
|
||||
attention_mode=2,
|
||||
return_softmax_lse=True,
|
||||
)
|
||||
lse = softmax_max.float() + torch.log(softmax_sum.float())
|
||||
return attn_out, lse.permute(1, 0, 2).reshape(lse.shape[1], -1)
|
||||
@@ -4,6 +4,7 @@ import torch
|
||||
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dcp.layout import localize_dcp_indices
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
MHATokenToKOnlyPool,
|
||||
MHATokenToKVPool,
|
||||
@@ -12,6 +13,7 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
get_tensor_size_bytes,
|
||||
unwrap_write_loc,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var
|
||||
from sglang.srt.utils.common import is_npu
|
||||
|
||||
@@ -585,10 +587,13 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
device: str,
|
||||
enable_memory_saver: bool,
|
||||
index_head_dim: Optional[int] = None,
|
||||
index_size: Optional[int] = None,
|
||||
start_layer: Optional[int] = None,
|
||||
end_layer: Optional[int] = None,
|
||||
index_page_size: Optional[int] = None,
|
||||
indexer_layer_ids: Optional[Sequence[int]] = None,
|
||||
kv_cache_dim: Optional[int] = None,
|
||||
is_draft_worker: bool = False,
|
||||
):
|
||||
# MLAPO historically owned NZ writes. Keep the allocation unchanged and
|
||||
# write into the NZ-addressed view below so ordinary MLA (including
|
||||
@@ -613,7 +618,6 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
)
|
||||
if self.enable_sparsity_driven_kv_offload and self.index_head_dim is None:
|
||||
raise ValueError("Sparsity-driven KV offload requires an index KV cache.")
|
||||
|
||||
if index_head_dim is None:
|
||||
self.indexer_layer_ids = ()
|
||||
elif indexer_layer_ids is None:
|
||||
@@ -649,6 +653,19 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
self.kr_cache_dim = 0 if self.dsa_kv_cache_store_fp8 else qk_rope_head_dim
|
||||
self.index_k_scale_buffer = None
|
||||
self.indexer_hadamard_128 = None
|
||||
self.index_page_size = page_size if index_page_size is None else index_page_size
|
||||
self.index_size = size if index_size is None else index_size
|
||||
parallel = get_parallel()
|
||||
self.dcp_size = parallel.attn_dcp_size
|
||||
self.dcp_rank = parallel.attn_dcp_rank
|
||||
global_page_padding = self.dcp_size if self.dcp_size > 1 else 1
|
||||
kv_page_padding = global_page_padding if is_draft_worker else 1
|
||||
index_page_padding = global_page_padding if index_head_dim is not None else 1
|
||||
if kv_page_padding < 1 or index_page_padding < 1:
|
||||
raise ValueError("NPU MLA page padding must be positive")
|
||||
self.kv_page_padding = kv_page_padding
|
||||
self.index_page_padding = index_page_padding
|
||||
self.is_draft_worker = is_draft_worker
|
||||
|
||||
self.custom_mem_pool = None
|
||||
|
||||
@@ -661,7 +678,7 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
self.k_buffer = torch.zeros(
|
||||
(
|
||||
layer_num,
|
||||
self.size // self.page_size + 1,
|
||||
self.size // self.page_size + self.kv_page_padding,
|
||||
self.page_size,
|
||||
1,
|
||||
self.kv_cache_dim,
|
||||
@@ -672,7 +689,7 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
self.v_buffer = torch.zeros(
|
||||
(
|
||||
layer_num,
|
||||
self.size // self.page_size + 1,
|
||||
self.size // self.page_size + self.kv_page_padding,
|
||||
self.page_size,
|
||||
1,
|
||||
self.kr_cache_dim,
|
||||
@@ -689,8 +706,9 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
self.index_k_buffer = torch.zeros(
|
||||
(
|
||||
self.num_indexer_layers,
|
||||
self.size // self.page_size + 1,
|
||||
self.page_size,
|
||||
self.index_size // self.index_page_size
|
||||
+ self.index_page_padding,
|
||||
self.index_page_size,
|
||||
1,
|
||||
self.index_head_dim,
|
||||
),
|
||||
@@ -713,6 +731,17 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
|
||||
self._finalize_allocation_log(size)
|
||||
|
||||
def _copy_indices_for_buffer(self, indices, uses_global_slots):
|
||||
if uses_global_slots or self.dcp_size <= 1:
|
||||
return indices
|
||||
local_indices = localize_dcp_indices(
|
||||
indices,
|
||||
self.dcp_size,
|
||||
self.dcp_rank,
|
||||
self.page_size,
|
||||
)
|
||||
return local_indices[local_indices >= 0]
|
||||
|
||||
def get_kv_size_bytes(self):
|
||||
kv_size_bytes = 0
|
||||
|
||||
@@ -805,25 +834,37 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
scale.view(-1, 1),
|
||||
)
|
||||
|
||||
def _get_disagg_buffer_entries(self):
|
||||
"""Return (buffer, uses_global_slots) entries in PD transfer order."""
|
||||
self._raise_if_native_kv_cache_disabled()
|
||||
global_kv = self.is_draft_worker
|
||||
entries = [(buffer, global_kv) for buffer in self.k_buffer]
|
||||
if not getattr(self, "dsa_kv_cache_store_fp8", False):
|
||||
entries += [(buffer, global_kv) for buffer in self.v_buffer]
|
||||
if self.index_head_dim is not None:
|
||||
entries += [(buffer, True) for buffer in self.index_k_buffer]
|
||||
if self.index_k_scale_buffer is not None:
|
||||
entries += [(buffer, True) for buffer in self.index_k_scale_buffer]
|
||||
return entries
|
||||
|
||||
# for disagg
|
||||
def get_contiguous_buf_infos(self):
|
||||
self._raise_if_native_kv_cache_disabled()
|
||||
# MLA has only one kv_buffer, so only the information of this buffer needs to be returned.
|
||||
kv_data_ptrs = [self.k_buffer[i].data_ptr() for i in range(self.layer_num)]
|
||||
kv_data_lens = [self.k_buffer[i].nbytes for i in range(self.layer_num)]
|
||||
kv_item_lens = [self.k_buffer[i][0].nbytes for i in range(self.layer_num)]
|
||||
# When DSA KV cache is packed into the FP8 k_buffer, the v_buffer is
|
||||
# intentionally empty (kr_cache_dim == 0). Its data_ptr() is null
|
||||
if not getattr(self, "dsa_kv_cache_store_fp8", False):
|
||||
kv_data_ptrs += [self.v_buffer[i].data_ptr() for i in range(self.layer_num)]
|
||||
kv_data_lens += [self.v_buffer[i].nbytes for i in range(self.layer_num)]
|
||||
kv_item_lens += [self.v_buffer[i][0].nbytes for i in range(self.layer_num)]
|
||||
if self.index_head_dim is not None:
|
||||
ptrs, lens, item_lens = self.get_state_buf_infos()
|
||||
kv_data_ptrs += ptrs
|
||||
kv_data_lens += lens
|
||||
kv_item_lens += item_lens
|
||||
return kv_data_ptrs, kv_data_lens, kv_item_lens
|
||||
entries = self._get_disagg_buffer_entries()
|
||||
return (
|
||||
[buffer.data_ptr() for buffer, _ in entries],
|
||||
[buffer.nbytes for buffer, _ in entries],
|
||||
[
|
||||
buffer[0].nbytes * (self.dcp_size if uses_global_slots else 1)
|
||||
for buffer, uses_global_slots in entries
|
||||
],
|
||||
)
|
||||
|
||||
def get_dcp_remote_decode_layout(self) -> list[bool]:
|
||||
"""Whether each PD entry uses allocator-global slots on decode."""
|
||||
return [
|
||||
uses_global_slots
|
||||
for _, uses_global_slots in self._get_disagg_buffer_entries()
|
||||
]
|
||||
|
||||
def get_kv_layer_ids(self):
|
||||
return (
|
||||
@@ -956,17 +997,25 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
index_k.view(-1, 1, self.index_head_dim),
|
||||
)
|
||||
|
||||
def _chunk_copy_npu_to_cpu(self, buf_of_layers, indices):
|
||||
def _chunk_copy_npu_to_cpu(
|
||||
self, buf_of_layers, indices, uses_global_slots_per_layer
|
||||
):
|
||||
chunk_size = self.cpu_offloading_chunk_size
|
||||
out = []
|
||||
for tensors_per_layer in buf_of_layers: # [k_buf, v_buf, ik_buf/None]
|
||||
for tensors_per_layer, uses_global_slots in zip(
|
||||
buf_of_layers, uses_global_slots_per_layer, strict=True
|
||||
): # [k_buf, v_buf, ik_buf/None]
|
||||
layer_chunks = []
|
||||
for i in range(0, len(indices), chunk_size):
|
||||
ci = indices[i : i + chunk_size]
|
||||
layer_chunks.append(
|
||||
[
|
||||
t[ci].to("cpu", non_blocking=True)
|
||||
for t in tensors_per_layer
|
||||
t[self._copy_indices_for_buffer(ci, uses_global)].to(
|
||||
"cpu", non_blocking=True
|
||||
)
|
||||
for t, uses_global in zip(
|
||||
tensors_per_layer, uses_global_slots, strict=True
|
||||
)
|
||||
if t is not None
|
||||
]
|
||||
)
|
||||
@@ -998,8 +1047,18 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
buf_of_layers = [
|
||||
self._get_cpu_offload_layer_buffers(i) for i in range(self.layer_num)
|
||||
]
|
||||
uses_global_slots_per_layer = []
|
||||
for buffers in buf_of_layers:
|
||||
# MLA K/V is rank-local under DCP. The replicated
|
||||
# indexer buffers retain allocator-global slot identities.
|
||||
uses_global_slots_per_layer.append(
|
||||
[self.is_draft_worker, self.is_draft_worker]
|
||||
+ [True] * (len(buffers) - 2)
|
||||
)
|
||||
|
||||
kv_cache_cpu = self._chunk_copy_npu_to_cpu(buf_of_layers, indices)
|
||||
kv_cache_cpu = self._chunk_copy_npu_to_cpu(
|
||||
buf_of_layers, indices, uses_global_slots_per_layer
|
||||
)
|
||||
torch.npu.synchronize()
|
||||
return kv_cache_cpu
|
||||
|
||||
@@ -1013,7 +1072,20 @@ class NPUMLATokenToKVPool(MLATokenToKVPool):
|
||||
for i in range(0, len(indices), chunk_size):
|
||||
chunk_indices = indices[i : i + chunk_size]
|
||||
chunk = kv_cache_cpu[local_layer_id][i // chunk_size]
|
||||
for buffer, cpu in zip(buffers, chunk, strict=True):
|
||||
assert cpu.shape[0] == len(chunk_indices)
|
||||
buffer[chunk_indices] = cpu.to(buffer.device, non_blocking=True)
|
||||
cpu_index = 0
|
||||
for buffer, uses_global_slots in zip(
|
||||
buffers,
|
||||
[self.is_draft_worker, self.is_draft_worker]
|
||||
+ [True] * (len(buffers) - 2),
|
||||
strict=True,
|
||||
):
|
||||
if buffer is None:
|
||||
continue
|
||||
cpu = chunk[cpu_index]
|
||||
cpu_index += 1
|
||||
target_indices = self._copy_indices_for_buffer(
|
||||
chunk_indices, uses_global_slots
|
||||
)
|
||||
assert cpu.shape[0] == len(target_indices)
|
||||
buffer[target_indices] = cpu.to(buffer.device, non_blocking=True)
|
||||
torch.npu.synchronize()
|
||||
|
||||
@@ -4,6 +4,10 @@ import torch
|
||||
import torch_npu
|
||||
from sgl_kernel_npu.norm.fused_split_qk_norm import fused_split_qk_norm
|
||||
|
||||
import sglang.srt.layers.dcp.comm as dcp_comm
|
||||
import sglang.srt.layers.dcp.layout as dcp_layout
|
||||
import sglang.srt.model_executor.forward_context as forward_context
|
||||
import sglang.srt.runtime_context as runtime_context
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.hardware_backend.npu.attention.mla_preprocess import (
|
||||
NPUFusedMLAPreprocess,
|
||||
@@ -28,6 +32,15 @@ _use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
|
||||
_is_npu_arch35 = is_npu_arch35()
|
||||
|
||||
|
||||
def _use_dsa_dcp_partial_attention(forward_batch: "ForwardBatch") -> bool:
|
||||
return (
|
||||
runtime_context.get_parallel().dcp_enabled
|
||||
and not forward_context.get_attn_backend().is_draft_worker
|
||||
and not dsa_use_prefill_cp(forward_batch)
|
||||
and not forward_batch.forward_mode.is_idle()
|
||||
)
|
||||
|
||||
|
||||
# region MHA
|
||||
def forward_mha_prepare_npu(
|
||||
m: "DeepseekV2AttentionMLA",
|
||||
@@ -488,9 +501,23 @@ def forward_dsa_prepare_npu(
|
||||
input_on_attn_tp_slices,
|
||||
dynamic_scale,
|
||||
)
|
||||
# DSA layers that skip the indexer reuse ``prev_topk_indices``. Remap
|
||||
# only when a fresh global top-k is produced so shared-index layers do
|
||||
# not repeat the same DCP partitioning work.
|
||||
if _use_dsa_dcp_partial_attention(forward_batch):
|
||||
parallel = runtime_context.get_parallel()
|
||||
topk_indices = dcp_layout.remap_dcp_sparse_indices(
|
||||
topk_indices,
|
||||
parallel.attn_dcp_size,
|
||||
parallel.attn_dcp_rank,
|
||||
interleave_size=forward_context.get_attn_backend().page_size,
|
||||
)
|
||||
else:
|
||||
topk_indices = prev_topk_indices
|
||||
|
||||
if _use_dsa_dcp_partial_attention(forward_batch):
|
||||
q_nope_out, q_pe = dcp_comm.all_gather_q_for_mla_decode(q_nope_out, q_pe)
|
||||
|
||||
topk_indices = maybe_capture_indexer_topk(m.layer_id, topk_indices)
|
||||
|
||||
return (
|
||||
@@ -522,7 +549,9 @@ def forward_dsa_core_npu(
|
||||
# a trailing arg. None everywhere else.
|
||||
gate: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
attn_output = m.attn_mqa(
|
||||
use_dcp = _use_dsa_dcp_partial_attention(forward_batch)
|
||||
attn = m.attn_mqa_for_dcp_decode if use_dcp else m.attn_mqa
|
||||
attn_output = attn(
|
||||
q_nope_out.contiguous(),
|
||||
k_nope.contiguous(),
|
||||
k_nope.contiguous(),
|
||||
@@ -532,6 +561,16 @@ def forward_dsa_core_npu(
|
||||
k_rope=k_pe.contiguous(),
|
||||
topk_indices=topk_indices,
|
||||
)
|
||||
if use_dcp:
|
||||
attn_output, lse = attn_output
|
||||
attn_output = attn_output.view(
|
||||
-1,
|
||||
m.num_local_heads * runtime_context.get_parallel().attn_dcp_size,
|
||||
m.kv_lora_rank,
|
||||
)
|
||||
attn_output = dcp_comm.cp_lse_ag_out_rs_mla_npu(
|
||||
attn_output, lse, runtime_context.get_parallel().dcp_group
|
||||
)
|
||||
attn_output = attn_output.view(-1, m.num_local_heads, m.kv_lora_rank)
|
||||
|
||||
if _is_npu_arch35 or (
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.model_executor.forward_context import (
|
||||
get_attn_backend,
|
||||
get_token_to_kv_pool,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
if is_npu():
|
||||
@@ -207,16 +208,31 @@ class DSANPUIndexerMixin:
|
||||
torch.npu.current_stream(),
|
||||
)
|
||||
|
||||
indexer_cache_loc = forward_batch.out_cache_loc
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
parallel.dcp_enabled
|
||||
and parallel.attn_dcp_size > 1
|
||||
and not get_attn_backend().is_draft_worker
|
||||
):
|
||||
indexer_cache_loc = (
|
||||
get_attn_backend().forward_metadata.dcp_origin_out_cache_loc
|
||||
)
|
||||
assert indexer_cache_loc is not None, (
|
||||
"NPU DSA+DCP requires allocator-global origin_out_cache_loc metadata"
|
||||
)
|
||||
assert indexer_cache_loc.shape[0] == positions.shape[0], (
|
||||
"NPU DSA+DCP origin_out_cache_loc metadata has an incompatible "
|
||||
f"length: {indexer_cache_loc.shape[0]} != {positions.shape[0]}"
|
||||
)
|
||||
pool = get_token_to_kv_pool()
|
||||
use_quant_indexer = pool.index_k_scale_buffer is not None
|
||||
if use_quant_indexer:
|
||||
k, k_scale = _quantize_npu_indexer_activation(
|
||||
k, pool.indexer_hadamard_128, pool.dtype
|
||||
)
|
||||
pool.set_index_k_scale_buffer(
|
||||
layer_id, forward_batch.out_cache_loc, k_scale
|
||||
)
|
||||
pool.set_index_k_buffer(layer_id, forward_batch.out_cache_loc, k)
|
||||
pool.set_index_k_scale_buffer(layer_id, indexer_cache_loc, k_scale)
|
||||
pool.set_index_k_buffer(layer_id, indexer_cache_loc, k)
|
||||
if is_prefill:
|
||||
if (
|
||||
self.dsa_enable_prefill_cp
|
||||
|
||||
@@ -123,6 +123,41 @@ def cp_lse_ag_out_rs_mla(
|
||||
return out.to(cp_attn_out.dtype)
|
||||
|
||||
|
||||
def cp_lse_ag_out_rs_mla_npu(
|
||||
cp_attn_out: torch.Tensor,
|
||||
cp_attn_lse: torch.Tensor,
|
||||
cp_group: GroupCoordinator,
|
||||
) -> torch.Tensor:
|
||||
"""Merge NPU DCP partial outputs and return the local head slice."""
|
||||
if cp_group.world_size == 1:
|
||||
return cp_attn_out
|
||||
|
||||
import torch_npu
|
||||
|
||||
batch_size, total_heads, head_dim = cp_attn_out.shape
|
||||
world_size = cp_group.world_size
|
||||
local_heads = total_heads // world_size
|
||||
packed = torch.cat([cp_attn_out.float(), cp_attn_lse.float().unsqueeze(-1)], dim=-1)
|
||||
packed = packed.permute(1, 2, 0).contiguous()
|
||||
gathered = torch.empty_like(packed)
|
||||
cp_group.all_to_all_single(gathered, packed)
|
||||
# all_to_all_single splits the leading head dimension. After the exchange,
|
||||
# the heads are grouped by source rank inside every token. Move that source
|
||||
# rank in front before flattening tokens and local heads for the update op.
|
||||
gathered = gathered.permute(2, 0, 1).contiguous()
|
||||
gathered = (
|
||||
gathered.view(batch_size, world_size, local_heads, head_dim + 1)
|
||||
.permute(1, 0, 2, 3)
|
||||
.contiguous()
|
||||
.view(world_size, batch_size * local_heads, head_dim + 1)
|
||||
)
|
||||
out_flat, lse_flat = torch.split(gathered, [head_dim, 1], dim=-1)
|
||||
merged, _ = torch_npu.npu_attention_update(
|
||||
lse_flat.squeeze(-1).unbind(0), out_flat.unbind(0), 0
|
||||
)
|
||||
return merged.view(batch_size, local_heads, head_dim).to(cp_attn_out.dtype)
|
||||
|
||||
|
||||
def _all_gather_dcp_kv_cache(kv_a: torch.Tensor):
|
||||
parallel = get_parallel()
|
||||
dcp_world_size = parallel.dcp_size
|
||||
|
||||
@@ -25,20 +25,53 @@ def get_dcp_lens(
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
start: torch.Tensor | None = None,
|
||||
interleave_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Per-rank visible KV length under the owner rule pos % dcp_size == dcp_rank.
|
||||
"""Return the KV length owned by one DCP rank.
|
||||
|
||||
Superset implementation (PR #25090): supports both start=None and a per-request
|
||||
`start` offset. update_local_kv_lens_for_dcp is the start=None special case.
|
||||
Consecutive runs of ``interleave_size`` positions rotate across ranks. The
|
||||
default value preserves the token-interleaved layout used by the generic
|
||||
CUDA/ROCm DCP paths; NPU DSA passes its physical KV page size.
|
||||
"""
|
||||
if dcp_size == 1:
|
||||
return lens
|
||||
if start is None:
|
||||
return lens // dcp_size + (dcp_rank < lens % dcp_size)
|
||||
if interleave_size < 1:
|
||||
raise ValueError(f"interleave_size must be positive, got {interleave_size}")
|
||||
|
||||
first = start + torch.remainder(dcp_rank - start, dcp_size)
|
||||
remaining = start + lens - first
|
||||
return torch.clamp((remaining + dcp_size - 1) // dcp_size, min=0)
|
||||
cycle_size = dcp_size * interleave_size
|
||||
|
||||
def _count_before(end: torch.Tensor) -> torch.Tensor:
|
||||
full_cycles = end // cycle_size
|
||||
remainder = end % cycle_size
|
||||
rank_remainder = torch.clamp(
|
||||
remainder - dcp_rank * interleave_size,
|
||||
min=0,
|
||||
max=interleave_size,
|
||||
)
|
||||
return full_cycles * interleave_size + rank_remainder
|
||||
|
||||
if start is None:
|
||||
return _count_before(lens)
|
||||
return _count_before(start + lens) - _count_before(start)
|
||||
|
||||
|
||||
def localize_dcp_indices(
|
||||
indices: torch.Tensor,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
interleave_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Map global DCP indices to one rank, using ``-1`` for non-local rows."""
|
||||
if dcp_size == 1:
|
||||
return indices
|
||||
if interleave_size < 1:
|
||||
raise ValueError(f"interleave_size must be positive, got {interleave_size}")
|
||||
|
||||
interleave_block = torch.div(indices, interleave_size, rounding_mode="floor")
|
||||
is_local = (indices >= 0) & (interleave_block % dcp_size == dcp_rank)
|
||||
local_block = torch.div(interleave_block, dcp_size, rounding_mode="floor")
|
||||
local_indices = local_block * interleave_size + indices % interleave_size
|
||||
return torch.where(is_local, local_indices, torch.full_like(indices, -1))
|
||||
|
||||
|
||||
def maybe_dcp_kernel_indices(
|
||||
@@ -84,6 +117,77 @@ def filter_dcp_local_chunk_kv_indices(
|
||||
return torch.cat(parts)
|
||||
|
||||
|
||||
def remap_dcp_sparse_indices(
|
||||
topk_indices: torch.Tensor,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
interleave_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Map global sparse token indices to one rank's compact DCP KV layout.
|
||||
|
||||
Keep indices owned by this rank, convert them to local KV positions, and
|
||||
stably move them before ``-1`` padding while preserving their score order.
|
||||
"""
|
||||
if dcp_size == 1:
|
||||
return topk_indices
|
||||
if interleave_size < 1:
|
||||
raise ValueError(f"interleave_size must be positive, got {interleave_size}")
|
||||
|
||||
# Float32 is faster than integer division/remainder on Ascend for this hot
|
||||
# path. Keep the math equivalent to localize_dcp_indices above.
|
||||
topk_indices_fp32 = topk_indices.to(torch.float32)
|
||||
interleave_blocks = torch.floor(topk_indices_fp32 / interleave_size)
|
||||
local_owner_mask = (topk_indices_fp32 >= 0) & (
|
||||
torch.remainder(interleave_blocks, dcp_size) == dcp_rank
|
||||
)
|
||||
local_offsets = torch.remainder(topk_indices_fp32, interleave_size)
|
||||
local_indices = (
|
||||
torch.floor(interleave_blocks / dcp_size) * interleave_size + local_offsets
|
||||
)
|
||||
remapped_indices = torch.where(
|
||||
local_owner_mask,
|
||||
local_indices,
|
||||
torch.full_like(topk_indices_fp32, -1.0),
|
||||
).to(topk_indices.dtype)
|
||||
|
||||
# Move valid entries before padding without changing their top-k order.
|
||||
topk_count = topk_indices.shape[-1]
|
||||
original_order = torch.arange(
|
||||
topk_count, dtype=torch.float32, device=topk_indices.device
|
||||
).expand_as(topk_indices_fp32)
|
||||
pack_keys = original_order + (~local_owner_mask).to(torch.float32) * topk_count
|
||||
pack_order = torch.argsort(pack_keys, dim=-1).to(torch.int64)
|
||||
return torch.gather(remapped_indices, dim=-1, index=pack_order)
|
||||
|
||||
|
||||
def get_dcp_chain_spec_lens(
|
||||
total_kv_lens: torch.Tensor,
|
||||
tokens_per_req: int,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
interleave_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Return request-major local KV frontiers for a speculative chain."""
|
||||
if tokens_per_req < 1:
|
||||
raise ValueError(f"tokens_per_req must be >= 1, got {tokens_per_req}")
|
||||
total_kv_lens = total_kv_lens.int()
|
||||
steps = torch.arange(
|
||||
1, tokens_per_req + 1, dtype=total_kv_lens.dtype, device=total_kv_lens.device
|
||||
)
|
||||
global_query_lens = total_kv_lens.unsqueeze(1) - tokens_per_req + steps
|
||||
global_query_lens = torch.where(
|
||||
total_kv_lens.unsqueeze(1) >= tokens_per_req,
|
||||
global_query_lens,
|
||||
torch.zeros_like(global_query_lens),
|
||||
)
|
||||
return get_dcp_lens(
|
||||
global_query_lens.reshape(-1),
|
||||
dcp_size,
|
||||
dcp_rank,
|
||||
interleave_size=interleave_size,
|
||||
).int()
|
||||
|
||||
|
||||
def update_local_kv_lens_for_dcp(kv_len_arr):
|
||||
"""In-place per-rank KV length: the start=0 case of get_dcp_lens.
|
||||
|
||||
|
||||
@@ -1555,6 +1555,12 @@ class KVCacheConfigurator:
|
||||
)
|
||||
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
|
||||
|
||||
# Indexer uses the allocator-global slot space on DCP target workers.
|
||||
dcp_size = get_parallel().attn_dcp_size
|
||||
if self.is_draft_worker:
|
||||
index_size = max_total_num_tokens
|
||||
else:
|
||||
index_size = max_total_num_tokens * dcp_size
|
||||
is_arch35 = is_npu_arch35()
|
||||
use_compact_indexer_layout = (
|
||||
is_dsa_model
|
||||
@@ -1576,11 +1582,13 @@ class KVCacheConfigurator:
|
||||
)
|
||||
token_to_kv_pool = NPUMLATokenToKVPool(
|
||||
max_total_num_tokens,
|
||||
page_size=self.pool_page_size,
|
||||
page_size=get_schedule().page_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
kv_lora_rank=self.model_config.kv_lora_rank,
|
||||
qk_rope_head_dim=self.model_config.qk_rope_head_dim,
|
||||
index_head_dim=(self.model_config.index_head_dim if is_dsa_model else None),
|
||||
index_size=index_size,
|
||||
index_page_size=get_schedule().page_size,
|
||||
indexer_layer_ids=indexer_layer_ids,
|
||||
kv_cache_dim=(
|
||||
calculate_mla_kv_cache_dim(
|
||||
@@ -1589,6 +1597,7 @@ class KVCacheConfigurator:
|
||||
if use_dsa_fp8_kv_cache_storage
|
||||
else None
|
||||
),
|
||||
is_draft_worker=self.is_draft_worker,
|
||||
layer_num=self.layer_info.num_effective_layers,
|
||||
device=self.device,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
@@ -2057,8 +2066,13 @@ class KVCacheConfigurator:
|
||||
)
|
||||
|
||||
token_to_kv_pool_allocator = NPUPagedTokenToKVPoolAllocator(
|
||||
sizes.max_total_num_tokens,
|
||||
page_size=get_schedule().page_size,
|
||||
# DCP allocation is in the global virtual loc space.
|
||||
# The target attention path localizes these locs when
|
||||
# building rank-local metadata; draft/indexer paths
|
||||
# consume the allocator locs directly.
|
||||
sizes.max_total_num_tokens * get_parallel().attn_dcp_size,
|
||||
page_size=get_schedule().page_size
|
||||
* get_parallel().attn_dcp_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
device=self.device,
|
||||
kvcache=token_to_kv_pool,
|
||||
|
||||
@@ -43,6 +43,7 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
|
||||
compute_req_all_ids_info,
|
||||
)
|
||||
from sglang.srt.layers.dcp.layout import localize_dcp_indices
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
dp_slot_in,
|
||||
@@ -191,6 +192,21 @@ def _elastic_should_preserve_local_token_counts(
|
||||
return uneven_token_count
|
||||
|
||||
|
||||
def _localize_npu_dcp_out_cache_loc(
|
||||
out_cache_loc: torch.Tensor,
|
||||
*,
|
||||
interleave_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Map allocator-global NPU DCP slots to this target rank."""
|
||||
parallel = get_parallel()
|
||||
return localize_dcp_indices(
|
||||
out_cache_loc,
|
||||
parallel.dcp_size,
|
||||
parallel.dcp_rank,
|
||||
interleave_size,
|
||||
)
|
||||
|
||||
|
||||
class ForwardMode(IntEnum):
|
||||
# Extend a sequence. The KV cache of the beginning part of the sequence is already computed (e.g., system prompt).
|
||||
# It is also called "prefill" in common terminology.
|
||||
@@ -505,6 +521,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# The sum of all sequence lengths
|
||||
seq_lens_sum: int
|
||||
|
||||
# Allocator-global output slots before NPU DCP localizes ``out_cache_loc``.
|
||||
# DSA's replicated indexer cache uses the global slot identity.
|
||||
origin_out_cache_loc: Optional[torch.Tensor] = None
|
||||
|
||||
# === Borrowed from ScheduleBatch: GPU tensors (cross-stream; clone targets for stream isolation) ===
|
||||
# FIXME(lsyin): these are currently aliased by reference from ScheduleBatch. Once
|
||||
# they are cloned/relayed into FB-owned copies at the boundary, move them out of
|
||||
@@ -1025,6 +1045,15 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
spec_info=batch.spec_info,
|
||||
)
|
||||
|
||||
# ScheduleBatch and req_to_token keep allocator-global slot identities.
|
||||
# Preserve that view before exposing rank-local NPU DCP write slots.
|
||||
if _is_npu and get_parallel().dcp_enabled and not model_runner.is_draft_worker:
|
||||
ret.origin_out_cache_loc = ret.out_cache_loc
|
||||
if ret.out_cache_loc is not None:
|
||||
ret.out_cache_loc = _localize_npu_dcp_out_cache_loc(
|
||||
ret.out_cache_loc,
|
||||
interleave_size=model_runner.page_size,
|
||||
)
|
||||
ret._maybe_init_non_generation_fields(batch)
|
||||
|
||||
device = model_runner.device
|
||||
@@ -1820,6 +1849,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
)
|
||||
|
||||
self.out_cache_loc = self._pad_tensor_to_size(self.out_cache_loc, num_tokens)
|
||||
if self.origin_out_cache_loc is not None:
|
||||
self.origin_out_cache_loc = self._pad_tensor_to_size(
|
||||
self.origin_out_cache_loc, num_tokens
|
||||
)
|
||||
if self.encoder_lens is not None:
|
||||
self.encoder_lens = self._pad_tensor_to_size(self.encoder_lens, bs)
|
||||
self.positions = self._pad_tensor_to_size(self.positions, num_tokens)
|
||||
@@ -2086,6 +2119,7 @@ def build_inner_fb_view(
|
||||
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
|
||||
# A caller may hand in another view that does not carry this field.
|
||||
out_cache_loc_virtual=getattr(forward_batch, "out_cache_loc_virtual", None),
|
||||
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
|
||||
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
|
||||
spec_info=forward_batch.spec_info,
|
||||
)
|
||||
|
||||
@@ -192,6 +192,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
|
||||
def __init__(self, kvc: KVCacheConfigurator):
|
||||
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
|
||||
dcp_size = get_parallel().attn_dcp_size
|
||||
# Determine effective number of layers for KV cache
|
||||
if mambaish := mambaish_config(kvc.model_config):
|
||||
effective_layer_ids = [
|
||||
@@ -233,6 +234,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
kvc=kvc,
|
||||
num_layers=num_layers,
|
||||
)
|
||||
if _is_npu and dcp_size > 1:
|
||||
target_indexer_size *= dcp_size
|
||||
target_kv_size = self._cell_size - target_indexer_size
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
get_glm_dsa_layer_split_effective_num_layers,
|
||||
@@ -249,6 +252,11 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
num_layers=draft_num_layers,
|
||||
allocate_all_layers=True,
|
||||
)
|
||||
# The draft pool is replicated and consumes the widened
|
||||
# allocator-global slot space on NPU DCP.
|
||||
if _is_npu and dcp_size > 1:
|
||||
draft_kv_size *= dcp_size
|
||||
draft_indexer_size *= dcp_size
|
||||
self._cell_size += draft_kv_size + draft_indexer_size
|
||||
else:
|
||||
self._cell_size = int(
|
||||
@@ -337,10 +345,13 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
|
||||
# Add indexer KV cache overhead for DSA models (DeepSeek V3.2)
|
||||
if is_deepseek_dsa(model_config.hf_config):
|
||||
cell_size += self._compute_dsa_indexer_cell_size(
|
||||
indexer_cell_size = self._compute_dsa_indexer_cell_size(
|
||||
kvc=kvc,
|
||||
num_layers=num_layers,
|
||||
)
|
||||
if _is_npu and not kvc.is_draft_worker and dcp_size > 1:
|
||||
indexer_cell_size *= dcp_size
|
||||
cell_size += indexer_cell_size
|
||||
elif is_minimax_sparse(model_config.hf_config):
|
||||
from sglang.srt.server_args import m3_fp8_attn_gemm_enabled
|
||||
|
||||
|
||||
@@ -198,6 +198,7 @@ def build_replay_fb_view(
|
||||
encoder_lens=buffers.encoder_lens[:bs] if is_encoder_decoder else None,
|
||||
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
|
||||
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
|
||||
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
|
||||
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
|
||||
max_seq_len_override=forward_batch.max_seq_len_override,
|
||||
# The mamba-track registry slot (VIRTUAL ids) is the v2p translate SOURCE
|
||||
|
||||
@@ -3420,6 +3420,8 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
kv_cache_device,
|
||||
create_chunked_prefix_cache_kv_indices_fn,
|
||||
):
|
||||
if _is_npu:
|
||||
return None
|
||||
return prepare_decode_context_parallel_metadata(
|
||||
seq_lens=seq_lens,
|
||||
extend_prefix_lens=extend_prefix_lens,
|
||||
|
||||
Reference in New Issue
Block a user