[DCP] Decode context parallelism for DSA (GLM-5.2 / GLM-5.3) with MTP (#43735)
Co-authored-by: vincentzed <207368749+vincentzed@users.noreply.github.com>
This commit is contained in:
@@ -181,6 +181,7 @@ for _mod, _fn in [
|
||||
("dsa.transform_index", "transform_index_page_table_prefill"),
|
||||
("dsa.transform_index", "transform_index_page_table_decode"),
|
||||
("dsa.transform_index", "prepare_trtllm_nope_sparse_metadata"),
|
||||
("dsa.transform_index", "transform_index_page_table_dcp"),
|
||||
("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"),
|
||||
("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"),
|
||||
("dsv4.fp4_indexer", "store_fp4_index_k_cache"),
|
||||
|
||||
@@ -605,7 +605,7 @@ def _dcp_lse_combine_kernel(
|
||||
partial_out = tl.load(recv_output_ptr + o_offsets).to(tl.float32)
|
||||
acc += partial_out * w
|
||||
|
||||
acc = acc / weight_sum
|
||||
acc = acc / tl.maximum(weight_sum, 1.0)
|
||||
|
||||
out_offsets = (
|
||||
batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D
|
||||
@@ -710,7 +710,7 @@ def _lse_weighted_combine_cpu(
|
||||
weights = torch.pow(2.0, centered)
|
||||
|
||||
weight_sum = weights.sum(dim=0, keepdim=True)
|
||||
weights = weights / weight_sum
|
||||
weights = weights / weight_sum.clamp_min(1.0)
|
||||
|
||||
combined = (partial_outputs * weights.unsqueeze(-1)).sum(dim=0)
|
||||
return combined
|
||||
|
||||
@@ -14,6 +14,55 @@ def transform_index_page_table_decode(**kwargs):
|
||||
return transform_index_page_table_decode_fast(**kwargs)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def transform_index_page_table_dcp_kernel(
|
||||
page_table_ptr,
|
||||
result_ptr,
|
||||
counts_ptr,
|
||||
row_stride: tl.constexpr,
|
||||
DCP_SIZE: tl.constexpr,
|
||||
DCP_RANK: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
offsets = tl.arange(0, BLOCK_TOPK)
|
||||
slots = tl.load(
|
||||
page_table_ptr + row * row_stride + offsets, mask=offsets < TOPK, other=-1
|
||||
)
|
||||
owned = (slots >= 0) & (slots % DCP_SIZE == DCP_RANK)
|
||||
packed_offsets = tl.cumsum(owned.to(tl.int32), axis=0) - 1
|
||||
count = tl.sum(owned.to(tl.int32), axis=0)
|
||||
tl.store(result_ptr + row * TOPK + packed_offsets, slots // DCP_SIZE, mask=owned)
|
||||
tl.store(result_ptr + row * TOPK + count + offsets, -1, mask=offsets < TOPK - count)
|
||||
tl.store(counts_ptr + row, count)
|
||||
|
||||
|
||||
def transform_index_page_table_dcp(
|
||||
page_table: torch.Tensor,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_rows, topk = page_table.shape
|
||||
result = torch.empty_like(page_table)
|
||||
counts = torch.empty(num_rows, dtype=torch.int32, device=page_table.device)
|
||||
if num_rows == 0 or topk == 0:
|
||||
return result, counts.zero_()
|
||||
assert page_table.stride(1) == 1
|
||||
transform_index_page_table_dcp_kernel[(num_rows,)](
|
||||
page_table,
|
||||
result,
|
||||
counts,
|
||||
page_table.stride(0),
|
||||
DCP_SIZE=dcp_size,
|
||||
DCP_RANK=dcp_rank,
|
||||
TOPK=topk,
|
||||
BLOCK_TOPK=triton.next_power_of_2(topk),
|
||||
num_warps=4,
|
||||
)
|
||||
return result, counts
|
||||
|
||||
|
||||
@triton.jit
|
||||
def prepare_trtllm_nope_sparse_metadata_kernel(
|
||||
page_table_ptr: torch.Tensor,
|
||||
|
||||
@@ -15,6 +15,25 @@ from sglang.srt.runtime_context import get_platform
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _dsa_dcp_overrides(cfg: Any) -> dict:
|
||||
if cfg.dcp_size <= 1:
|
||||
return {}
|
||||
if cfg.enable_hisparse or cfg.enable_hierarchical_cache:
|
||||
raise ValueError("DSA DCP does not support HiSparse or HiCache.")
|
||||
if cfg.disaggregation_mode != "null":
|
||||
raise ValueError("DSA DCP does not support PD disaggregation.")
|
||||
if cfg.speculative_eagle_topk not in (None, 1):
|
||||
raise ValueError("DSA DCP requires --speculative-eagle-topk 1.")
|
||||
overrides = {
|
||||
name: "trtllm"
|
||||
for name in ("dsa_prefill_backend", "dsa_decode_backend")
|
||||
if getattr(cfg, name) is None
|
||||
}
|
||||
if cfg.dcp_replicate_q_proj is None and cfg.dcp_comm_backend in ("a2a", "fi_a2a"):
|
||||
overrides["dcp_replicate_q_proj"] = True
|
||||
return overrides
|
||||
|
||||
|
||||
@_register_for(
|
||||
"DeepseekV3ForCausalLM",
|
||||
"DeepseekV32ForCausalLM",
|
||||
@@ -78,6 +97,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
|
||||
if is_deepseek_dsa(hf_config): # DeepSeek 3.2/GLM 5
|
||||
overrides.update(_dsa_dcp_overrides(cfg))
|
||||
# Set attention backend for DeepSeek
|
||||
if is_attention_backend_not_set(cfg):
|
||||
overrides["attention_backend"] = "dsa"
|
||||
|
||||
@@ -240,6 +240,8 @@ def handle_decode_context_parallelism(server_args: Any):
|
||||
"--decode-context-parallel-size) must be >= 1, but got "
|
||||
f"dcp_size={cfg.dcp_size}."
|
||||
)
|
||||
if cfg.dcp_size > 1 and cfg.enable_prefill_cp:
|
||||
raise ValueError("--dcp-size > 1 does not support --enable-prefill-cp.")
|
||||
if cfg.dcp_comm_backend in ("a2a", "fi_a2a") and cfg.dcp_size <= 1:
|
||||
raise ValueError(
|
||||
f"--dcp-comm-backend {cfg.dcp_comm_backend} only affects the "
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsa.transform_index import (
|
||||
transform_index_page_table_dcp,
|
||||
)
|
||||
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
|
||||
|
||||
TRTLLM_MAX_BATCH_ROWS = 65535
|
||||
# FlashInfer carves heads * rows * 256 float2 softmax slots out of the workspace per call;
|
||||
# the rest is kernel scratch.
|
||||
_SOFTMAX_BYTES_PER_ROW_HEAD = 256 * 8
|
||||
_SCRATCH_RESERVE_BYTES = 128 << 20
|
||||
|
||||
|
||||
def trtllm_rows_per_call(*, workspace_bytes: int, heads: int) -> int:
|
||||
rows = (workspace_bytes - _SCRATCH_RESERVE_BYTES) // (
|
||||
heads * _SOFTMAX_BYTES_PER_ROW_HEAD
|
||||
)
|
||||
return max(1, min(TRTLLM_MAX_BATCH_ROWS, rows))
|
||||
|
||||
|
||||
class DcpPageTableCache(msgspec.Struct):
|
||||
"""NOTE(vincentzed): GLM-5.2/5.3 reuse the previous layer's top-k tensor in
|
||||
57 of 78 layers; remapping once per fresh top-k cuts 78 launches/step to 21.
|
||||
"""
|
||||
|
||||
metadata: object = None
|
||||
layer_id: int = -1
|
||||
topk_indices: Optional[torch.Tensor] = None
|
||||
page_table: Optional[torch.Tensor] = None
|
||||
counts: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def localize_dcp_page_table(
|
||||
cache: DcpPageTableCache,
|
||||
*,
|
||||
metadata: object,
|
||||
layer_id: int,
|
||||
topk_indices: Optional[torch.Tensor],
|
||||
page_table: torch.Tensor,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
reuse = (
|
||||
topk_indices is not None
|
||||
and cache.metadata is metadata
|
||||
and cache.layer_id == layer_id - 1
|
||||
and cache.topk_indices is topk_indices
|
||||
)
|
||||
if not reuse:
|
||||
cache.metadata = metadata
|
||||
cache.topk_indices = topk_indices
|
||||
cache.page_table, cache.counts = transform_index_page_table_dcp(
|
||||
page_table, dcp_size, dcp_rank
|
||||
)
|
||||
cache.layer_id = layer_id
|
||||
return cache.page_table, cache.counts
|
||||
|
||||
|
||||
def dcp_trtllm_sparse_attention(
|
||||
*,
|
||||
query: torch.Tensor,
|
||||
block_tables: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
cu_seqlens_q: torch.Tensor,
|
||||
**attention_kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import flashinfer.decode
|
||||
|
||||
rows, _, heads, _ = query.shape
|
||||
out_dtype = torch.bfloat16
|
||||
if rows == 0:
|
||||
return (
|
||||
torch.empty(
|
||||
(0, 1, heads, attention_kwargs["kv_lora_rank"]),
|
||||
dtype=out_dtype,
|
||||
device=query.device,
|
||||
),
|
||||
torch.empty((0, heads), dtype=torch.float32, device=query.device),
|
||||
)
|
||||
lse = torch.empty((rows, heads), dtype=torch.float32, device=query.device)
|
||||
workspace = attention_kwargs["workspace_buffer"]
|
||||
step = trtllm_rows_per_call(
|
||||
workspace_bytes=workspace.numel() * workspace.element_size(), heads=heads
|
||||
)
|
||||
if rows > step:
|
||||
attention_kwargs["enable_pdl"] = False
|
||||
top_k_lens = attention_kwargs.pop("sparse_mla_top_k_lens", None)
|
||||
outputs = []
|
||||
for start in range(0, rows, step):
|
||||
end = min(start + step, rows)
|
||||
output, _ = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=query[start:end],
|
||||
block_tables=block_tables[start:end],
|
||||
seq_lens=seq_lens[start:end],
|
||||
return_lse=True,
|
||||
lse=lse[start:end],
|
||||
sparse_mla_top_k_lens=(
|
||||
None if top_k_lens is None else top_k_lens[start:end]
|
||||
),
|
||||
**attention_kwargs,
|
||||
)
|
||||
output = output.view(end - start, heads, -1)
|
||||
fixup_zero_kv_rows(
|
||||
output,
|
||||
lse[start:end],
|
||||
seq_lens[start:end],
|
||||
cu_seqlens_q[: end - start + 1],
|
||||
1,
|
||||
)
|
||||
outputs.append(output)
|
||||
output = outputs[0] if len(outputs) == 1 else torch.cat(outputs)
|
||||
return output.view(rows, 1, heads, -1), lse
|
||||
@@ -638,7 +638,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
pool.invalidate_index_buffer_for_layer(layer_id)
|
||||
kv_cache = (
|
||||
pool.get_index_k_with_scale_buffer(layer_id=layer_id)
|
||||
.view(-1, pool.page_size, self.head_dim + 4)
|
||||
.view(-1, pool.index_page_size, self.head_dim + 4)
|
||||
.view(fp8_dtype)
|
||||
)
|
||||
|
||||
@@ -739,8 +739,8 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
return None
|
||||
|
||||
pool = get_token_to_kv_pool()
|
||||
if pool.page_size != PAGE_SIZE:
|
||||
_decline_fused(f"pool page_size is {pool.page_size}, not {PAGE_SIZE}")
|
||||
if pool.index_page_size != PAGE_SIZE:
|
||||
_decline_fused(f"pool page_size is {pool.index_page_size}, not {PAGE_SIZE}")
|
||||
return None
|
||||
# DSA cache layer split: the standard path stores index-K per layer, and the
|
||||
# fused path addresses the same buffer directly.
|
||||
@@ -1000,7 +1000,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
if out_cache_loc is None:
|
||||
out_cache_loc = forward_batch.out_cache_loc
|
||||
pool = get_token_to_kv_pool()
|
||||
page_size = pool.page_size
|
||||
page_size = pool.index_page_size
|
||||
if hasattr(pool, "invalidate_index_buffer_for_layer"):
|
||||
pool.invalidate_index_buffer_for_layer(layer_id)
|
||||
if hasattr(pool, "_is_layer_owned") and not pool._is_layer_owned(layer_id):
|
||||
@@ -1199,7 +1199,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool)
|
||||
|
||||
page_size = get_token_to_kv_pool().page_size
|
||||
page_size = get_token_to_kv_pool().index_page_size
|
||||
# NOTE(dark): blocksize = 64 is hardcoded in deep_gemm
|
||||
if _is_hip:
|
||||
if _use_aiter_preshuffle:
|
||||
@@ -1463,7 +1463,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
|
||||
assert forward_batch.forward_mode.is_extend_without_speculative()
|
||||
|
||||
page_size = get_token_to_kv_pool().page_size
|
||||
page_size = get_token_to_kv_pool().index_page_size
|
||||
if _is_hip:
|
||||
if _use_aiter_preshuffle:
|
||||
assert page_size % 16 == 0, (
|
||||
@@ -1808,7 +1808,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
and can_use_dsa_fused_store(
|
||||
key.dtype,
|
||||
out_cache_loc.dtype,
|
||||
pool.page_size,
|
||||
pool.index_page_size,
|
||||
)
|
||||
):
|
||||
# NOTE: wrapper already normalizes shape/contiguity and asserts dtypes.
|
||||
@@ -1817,7 +1817,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
key,
|
||||
buf,
|
||||
out_cache_loc,
|
||||
pool.page_size,
|
||||
pool.index_page_size,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1827,7 +1827,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
# layout with page_size=1; the same kv_cache.view works for both cases
|
||||
# because page_size is 1 there.
|
||||
if _use_aiter:
|
||||
page_size = pool.page_size
|
||||
page_size = pool.index_page_size
|
||||
buf = pool.get_index_k_with_scale_buffer(layer_id=layer_id)
|
||||
kv_cache = buf.view(-1, page_size, 132).view(fp8_dtype)
|
||||
out_loc = forward_batch.out_cache_loc
|
||||
|
||||
@@ -60,6 +60,11 @@ from sglang.srt.layers.attention.dsa.dsa_backend_mtp_precompute import (
|
||||
PrecomputedMetadata,
|
||||
compute_cu_seqlens,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.dsa_dcp import (
|
||||
DcpPageTableCache,
|
||||
dcp_trtllm_sparse_attention,
|
||||
localize_dcp_page_table,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer_metadata import DSAIndexerMetadata
|
||||
from sglang.srt.layers.attention.dsa.dsa_metadata_manager import (
|
||||
DSAMetadataManagementMixin,
|
||||
@@ -529,6 +534,13 @@ class DeepseekSparseAttnBackend(
|
||||
self.num_q_heads = (
|
||||
model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size
|
||||
)
|
||||
if model_runner.is_draft_worker:
|
||||
self.dcp_size, self.dcp_rank = 1, 0
|
||||
else:
|
||||
self.dcp_size = get_parallel().attn_dcp_size
|
||||
self.dcp_rank = get_parallel().attn_dcp_rank
|
||||
self.num_dcp_q_heads = self.num_q_heads * self.dcp_size
|
||||
self._dcp_page_table_cache = DcpPageTableCache()
|
||||
self.kv_cache_dim = model_runner.token_to_kv_pool.kv_cache_dim
|
||||
self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim
|
||||
self.kv_lora_rank = model_runner.model_config.kv_lora_rank
|
||||
@@ -543,7 +555,7 @@ class DeepseekSparseAttnBackend(
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
|
||||
self.use_mha: bool = False
|
||||
self.supports_mha_one_shot: bool = True
|
||||
self.supports_mha_one_shot: bool = self.dcp_size == 1
|
||||
self.dsa_prefill_impl: _DSA_IMPL_T = get_exec().kernel.dsa_prefill_backend
|
||||
self.dsa_decode_impl: _DSA_IMPL_T = get_exec().kernel.dsa_decode_backend
|
||||
if (
|
||||
@@ -742,7 +754,7 @@ class DeepseekSparseAttnBackend(
|
||||
self.workspace_buffer = get_buffer(
|
||||
"dsa_trtllm_workspace",
|
||||
lambda: torch.empty(
|
||||
envs.SGLANG_FLASHINFER_WORKSPACE_SIZE.get(),
|
||||
envs.SGLANG_FLASHINFER_WORKSPACE_SIZE.get() * self.dcp_size,
|
||||
dtype=torch.uint8,
|
||||
device=model_runner.device,
|
||||
),
|
||||
@@ -750,7 +762,7 @@ class DeepseekSparseAttnBackend(
|
||||
self._multi_ctas_kv_counter_buffer = (
|
||||
make_persistent_multi_ctas_kv_counter_buffer(
|
||||
torch.device(self.device),
|
||||
self.num_q_heads,
|
||||
self.num_dcp_q_heads,
|
||||
max_batch_size=model_runner.max_running_requests,
|
||||
)
|
||||
)
|
||||
@@ -1525,7 +1537,7 @@ class DeepseekSparseAttnBackend(
|
||||
counter = grow_multi_ctas_kv_counter_buffer_if_needed(
|
||||
buffer=self._multi_ctas_kv_counter_buffer,
|
||||
device=torch.device(self.device),
|
||||
num_q_heads=self.num_q_heads,
|
||||
num_q_heads=self.num_dcp_q_heads,
|
||||
batch_size=num_query_rows,
|
||||
)
|
||||
# Capacity is set before capture, so a grow here is a broken invariant.
|
||||
@@ -1543,7 +1555,7 @@ class DeepseekSparseAttnBackend(
|
||||
grow_multi_ctas_kv_counter_buffer_if_needed(
|
||||
buffer=self._multi_ctas_kv_counter_buffer,
|
||||
device=torch.device(self.device),
|
||||
num_q_heads=self.num_q_heads,
|
||||
num_q_heads=self.num_dcp_q_heads,
|
||||
batch_size=num_query_rows,
|
||||
)
|
||||
)
|
||||
@@ -3774,6 +3786,8 @@ class DeepseekSparseAttnBackend(
|
||||
else:
|
||||
q_all = q.view(-1, layer.tp_q_head_num, layer.head_dim)
|
||||
|
||||
shared_topk_indices = topk_indices
|
||||
|
||||
# Eager DP attention can pad q beyond metadata that was deliberately
|
||||
# planned on the real draft batch. Pad top-k to the physical q shape,
|
||||
# then run decode attention only on metadata-backed rows. The output is
|
||||
@@ -3815,6 +3829,16 @@ class DeepseekSparseAttnBackend(
|
||||
page_table_1, sparse_mla_top_k = self._pad_trtllm_sparse_page_table(
|
||||
page_table_1
|
||||
)
|
||||
if self.dcp_size > 1:
|
||||
page_table_1, seq_lens = localize_dcp_page_table(
|
||||
self._dcp_page_table_cache,
|
||||
metadata=metadata,
|
||||
layer_id=layer.layer_id,
|
||||
topk_indices=shared_topk_indices,
|
||||
page_table=page_table_1,
|
||||
dcp_size=self.dcp_size,
|
||||
dcp_rank=self.dcp_rank,
|
||||
)
|
||||
sparse_mla_top_k_lens = None
|
||||
if self.qk_rope_head_dim == 0:
|
||||
sparse_mla_top_k_lens = prepare_trtllm_nope_sparse_metadata(page_table_1)
|
||||
@@ -3827,8 +3851,7 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
bmm1_scale = q_scale * k_scale * layer.scaling
|
||||
|
||||
batch_size = page_table_1.shape[0]
|
||||
_, num_heads, head_dim = q_all.shape
|
||||
batch_size, num_heads, head_dim = q_all.shape
|
||||
|
||||
multi_ctas_kv_counter_buffer = self._multi_ctas_kv_counter_for(batch_size)
|
||||
|
||||
@@ -3837,15 +3860,12 @@ class DeepseekSparseAttnBackend(
|
||||
block_tables = page_table_1.unsqueeze(1)
|
||||
seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens
|
||||
|
||||
out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q,
|
||||
attention_kwargs = dict(
|
||||
kv_cache=kv,
|
||||
workspace_buffer=self.workspace_buffer,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=metadata.max_seq_len_k,
|
||||
sparse_mla_top_k=sparse_mla_top_k,
|
||||
bmm1_scale=bmm1_scale,
|
||||
@@ -3855,6 +3875,30 @@ class DeepseekSparseAttnBackend(
|
||||
multi_ctas_kv_counter_buffer=multi_ctas_kv_counter_buffer,
|
||||
)
|
||||
|
||||
if self.dcp_size > 1:
|
||||
out, lse = dcp_trtllm_sparse_attention(
|
||||
query=q,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens,
|
||||
cu_seqlens_q=self.get_device_int32_arange(batch_size + 1),
|
||||
**attention_kwargs,
|
||||
)
|
||||
if num_decode_padding_rows:
|
||||
lse = torch.cat(
|
||||
[
|
||||
lse,
|
||||
lse.new_full((num_decode_padding_rows, num_heads), -torch.inf),
|
||||
]
|
||||
)
|
||||
return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows), lse
|
||||
|
||||
out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens,
|
||||
**attention_kwargs,
|
||||
)
|
||||
|
||||
return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows)
|
||||
|
||||
def _pad_topk_indices(
|
||||
|
||||
@@ -35,10 +35,18 @@ class IndexKeyCache:
|
||||
pool = self.pool
|
||||
return (
|
||||
num_pages,
|
||||
pool.index_page_size
|
||||
pool.page_size
|
||||
// pool.index_kpool
|
||||
* (pool.index_head_dim + pool.index_head_dim // pool.quant_block_size * 4),
|
||||
)
|
||||
|
||||
def _kernel_view(self, buffer: torch.Tensor) -> torch.Tensor:
|
||||
pool = self.pool
|
||||
kernel_pages_per_row = (
|
||||
pool.page_size // pool.index_kpool // pool.index_page_size
|
||||
)
|
||||
return buffer.view(-1, buffer.shape[1] // kernel_pages_per_row)
|
||||
|
||||
def _layer_num_pages(self, layer_idx: int, num_pages: int) -> int:
|
||||
# Layers that reuse the previous layer's top-k never write index-K, so
|
||||
# they get a 0-row placeholder that keeps ``buffer`` layer-aligned.
|
||||
@@ -62,7 +70,7 @@ class IndexKeyCache:
|
||||
self.pool.layer_transfer_counter.wait_until(
|
||||
layer_id - self.pool.start_layer
|
||||
)
|
||||
return self.buffer[layer_id - self.pool.start_layer]
|
||||
return self._kernel_view(self.buffer[layer_id - self.pool.start_layer])
|
||||
|
||||
def get_buffer(self, layer_id: int) -> torch.Tensor:
|
||||
return self.get_local_buffer(layer_id)
|
||||
@@ -92,7 +100,7 @@ class IndexKeyCache:
|
||||
index_k: torch.Tensor,
|
||||
index_k_scale: torch.Tensor,
|
||||
) -> None:
|
||||
buf = self.buffer[layer_id - self.pool.start_layer]
|
||||
buf = self._kernel_view(self.buffer[layer_id - self.pool.start_layer])
|
||||
index_buf_accessor.SetKAndS.execute(
|
||||
pool=self.pool,
|
||||
buf=buf,
|
||||
|
||||
@@ -2042,6 +2042,8 @@ class KVCacheConfigurator:
|
||||
pool_kwargs["layer_shard_size"] = dsa_cp_layer_shard_size
|
||||
else:
|
||||
PoolCls = dsa_pool_class
|
||||
if self.loc_space_scale > 1:
|
||||
pool_kwargs["dcp_replicated"] = True
|
||||
if _should_elide_dsa_index_k(is_draft_worker=self.is_draft_worker):
|
||||
pool_kwargs["skip_topk_layers"] = [
|
||||
dsa_layer_skips_topk(self.model_config.hf_config, layer_id)
|
||||
|
||||
@@ -4684,6 +4684,7 @@ class MLATokenToKVPool(KVCache):
|
||||
end_layer: Optional[int] = None,
|
||||
use_dsa: bool = False,
|
||||
override_kv_cache_dim: Optional[int] = None,
|
||||
dcp_replicated: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
size,
|
||||
@@ -4699,6 +4700,7 @@ class MLATokenToKVPool(KVCache):
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.use_dsa = use_dsa
|
||||
self.dcp_replicated = dcp_replicated
|
||||
self.dsa_kv_cache_store_fp8 = (
|
||||
use_dsa
|
||||
and dtype == torch.float8_e4m3fn
|
||||
@@ -4786,11 +4788,14 @@ class MLATokenToKVPool(KVCache):
|
||||
# False: this pool takes a WIDENED loc. The unified pool resolves it in
|
||||
# the plan's write translation (`translate_write_loc`) and flips this.
|
||||
write_loc_is_dcp_resolved = False
|
||||
dcp_replicated = False
|
||||
|
||||
@property
|
||||
def _write_loc_dcp_span(self) -> int:
|
||||
"""How many logical ids one stored row spans in the write-loc space."""
|
||||
return 1 if self.write_loc_is_dcp_resolved else get_parallel().attn_dcp_size
|
||||
if self.write_loc_is_dcp_resolved or self.dcp_replicated:
|
||||
return 1
|
||||
return get_parallel().attn_dcp_size
|
||||
|
||||
def _scatter_mla_rows(
|
||||
self,
|
||||
@@ -4799,7 +4804,7 @@ class MLATokenToKVPool(KVCache):
|
||||
cache_k_nope: torch.Tensor,
|
||||
cache_k_rope: torch.Tensor,
|
||||
) -> None:
|
||||
if self.write_loc_is_dcp_resolved:
|
||||
if self.write_loc_is_dcp_resolved or self.dcp_replicated:
|
||||
set_mla_kv_buffer_triton(dst_buffer, loc, cache_k_nope, cache_k_rope)
|
||||
else:
|
||||
set_mla_kv_buffer_dcp_sharded_triton(
|
||||
@@ -4823,7 +4828,11 @@ class MLATokenToKVPool(KVCache):
|
||||
assert not self.dsa_kv_cache_store_fp8
|
||||
# No DCP-aware variant is possible: the two backends reaching this door
|
||||
# disagree on the loc space (flashinfer-MLA widened, Triton collapsed).
|
||||
assert self.write_loc_is_dcp_resolved or not get_parallel().dcp_enabled, (
|
||||
assert (
|
||||
self.write_loc_is_dcp_resolved
|
||||
or self.dcp_replicated
|
||||
or not get_parallel().dcp_enabled
|
||||
), (
|
||||
"MLATokenToKVPool.set_kv_buffer has no DCP-aware write path. Under "
|
||||
"--dcp-size > 1 the MLA write must go through set_mla_kv_buffer, "
|
||||
"whose kernel resolves the owner rule; reaching the combined-row "
|
||||
@@ -5164,6 +5173,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
tail_extra_slots: int = 0,
|
||||
max_running_requests: Optional[int] = None,
|
||||
skip_topk_layers: Optional[List[bool]] = None,
|
||||
dcp_replicated: bool = False,
|
||||
):
|
||||
override_dim = (
|
||||
kv_cache_dim if kv_cache_dim != kv_lora_rank + qk_rope_head_dim else None
|
||||
@@ -5182,6 +5192,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
end_layer,
|
||||
use_dsa=True,
|
||||
override_kv_cache_dim=override_dim,
|
||||
dcp_replicated=dcp_replicated,
|
||||
)
|
||||
# self.index_k_dtype = torch.float8_e4m3fn
|
||||
# self.index_k_scale_dtype = torch.float32
|
||||
@@ -5193,8 +5204,15 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
f"page_size {self.page_size} must be a multiple of index_kpool {index_kpool}"
|
||||
)
|
||||
self.index_page_size = self.page_size // index_kpool
|
||||
if dcp_replicated:
|
||||
self.index_page_size //= get_parallel().attn_dcp_size
|
||||
if index_buf_size is None:
|
||||
index_buf_size = size
|
||||
dcp_size = get_parallel().attn_dcp_size
|
||||
index_buf_size = (
|
||||
size
|
||||
if dcp_replicated
|
||||
else (size + self.page_size) * dcp_size - self.page_size
|
||||
)
|
||||
self.index_buf_size = index_buf_size
|
||||
# num head == 1 and head dim == 128 for index_k in DSA
|
||||
assert index_head_dim == 128
|
||||
@@ -5206,7 +5224,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
)
|
||||
assert len(self.skip_topk_layers) == layer_num
|
||||
|
||||
physical_page_size = self.page_size // index_kpool
|
||||
physical_page_size = self.index_page_size
|
||||
if _is_hip:
|
||||
if aiter_can_use_preshuffle_paged_mqa():
|
||||
assert physical_page_size % 16 == 0, (
|
||||
|
||||
@@ -270,8 +270,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
kvc=kvc,
|
||||
num_layers=num_layers,
|
||||
)
|
||||
if _is_npu and dcp_size > 1:
|
||||
target_indexer_size *= dcp_size
|
||||
target_indexer_size *= dcp_size
|
||||
target_kv_size = self._cell_size - target_indexer_size
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
get_glm_dsa_layer_split_effective_num_layers,
|
||||
@@ -288,11 +287,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
num_layers=draft_num_layers,
|
||||
allocate_all_layers=True,
|
||||
)
|
||||
# The draft pool is replicated and consumes the widened
|
||||
# allocator-global slot space on NPU DCP.
|
||||
if _is_npu and dcp_size > 1:
|
||||
draft_kv_size *= dcp_size
|
||||
draft_indexer_size *= dcp_size
|
||||
draft_kv_size *= dcp_size
|
||||
draft_indexer_size *= dcp_size
|
||||
self._cell_size += draft_kv_size + draft_indexer_size
|
||||
else:
|
||||
self._cell_size = int(
|
||||
@@ -388,7 +384,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
kvc=kvc,
|
||||
num_layers=num_layers,
|
||||
)
|
||||
if _is_npu and not kvc.is_draft_worker and dcp_size > 1:
|
||||
if not kvc.is_draft_worker:
|
||||
indexer_cell_size *= dcp_size
|
||||
cell_size += indexer_cell_size
|
||||
elif is_minimax_sparse(model_config.hf_config):
|
||||
|
||||
@@ -266,7 +266,7 @@ class DeepseekMHAForwardMixin:
|
||||
kv_a, k_pe = self._get_mla_kv_buffer_from_fp8_for_dsa(forward_batch)
|
||||
else:
|
||||
# BF16/FP16 path: directly fetch from cache
|
||||
if get_parallel().dcp_enabled:
|
||||
if self.dcp_size > 1:
|
||||
kv_a, k_pe = all_gather_kv_cache_for_mha_extend(
|
||||
get_token_to_kv_pool(),
|
||||
self.attn_mha,
|
||||
|
||||
+2
-2
@@ -34,7 +34,7 @@ from sglang.srt.models.deepseek_common.utils import (
|
||||
_use_aiter_bpreshuffle_gfx95,
|
||||
_use_aiter_gfx95,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import BumpAllocator, get_bool_env_var
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -232,7 +232,7 @@ class DeepseekMHARocmForwardMixin:
|
||||
kv_a, k_pe = self._get_mla_kv_buffer_from_fp8_for_dsa(forward_batch)
|
||||
else:
|
||||
# BF16/FP16 path: directly fetch from cache
|
||||
if get_parallel().dcp_enabled:
|
||||
if self.dcp_size > 1:
|
||||
kv_a, k_pe = all_gather_kv_cache_for_mha_extend(
|
||||
get_token_to_kv_pool(),
|
||||
self.attn_mha,
|
||||
|
||||
@@ -92,15 +92,6 @@ def _select_local_dcp_heads_for_autotune(
|
||||
return attn_output.narrow(1, rank * num_local_heads, num_local_heads)
|
||||
|
||||
|
||||
def is_dcp_mla_decode_phase(forward_batch: ForwardBatch) -> bool:
|
||||
if not get_parallel().dcp_enabled:
|
||||
return False
|
||||
return (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
)
|
||||
|
||||
|
||||
def is_mla_dcp_lse_base_on_e(attention_backend: Optional[str]) -> bool:
|
||||
return attention_backend in {"flashmla", "cutedsl_mla", "aiter"}
|
||||
|
||||
@@ -131,6 +122,26 @@ def _apply_attention_output_gate(module, attn_output, gate):
|
||||
|
||||
|
||||
class DeepseekMLAForwardMixin:
|
||||
def is_dcp_mla_decode_phase(
|
||||
self: DeepseekV2AttentionMLA, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
if self.dcp_size == 1:
|
||||
return False
|
||||
mode = forward_batch.forward_mode
|
||||
return (
|
||||
mode.is_decode()
|
||||
or mode.is_target_verify()
|
||||
or (self.use_dsa and mode.is_extend())
|
||||
)
|
||||
|
||||
def is_dcp_q_replicate_phase(
|
||||
self: DeepseekV2AttentionMLA, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
if not get_parallel().dcp_replicate_q_proj or self.dcp_size == 1:
|
||||
return False
|
||||
mode = forward_batch.forward_mode
|
||||
return mode.is_decode() or mode.is_target_verify()
|
||||
|
||||
def init_mla_forward(self: DeepseekV2AttentionMLA):
|
||||
self.flashinfer_mla_disable_ragged = (
|
||||
get_exec().kernel.flashinfer_mla_disable_ragged
|
||||
@@ -162,6 +173,8 @@ class DeepseekMLAForwardMixin:
|
||||
) -> bool:
|
||||
if getattr(self, "_kimi_split_gguf_kv_b", False):
|
||||
return False
|
||||
if self.dcp_size > 1:
|
||||
return False
|
||||
# Shared activation surface with the DSA indexer graph dispatch
|
||||
# (in piecewise/breakable graph + non-speculative extend). Like the indexer
|
||||
# dispatch, this fusion is on by default on that surface.
|
||||
@@ -264,7 +277,7 @@ class DeepseekMLAForwardMixin:
|
||||
return None
|
||||
if get_is_capture_mode():
|
||||
return None
|
||||
if get_parallel().dcp_enabled:
|
||||
if self.dcp_size > 1:
|
||||
return None
|
||||
# Kernel shape constraints (tl.arange / tl.dot / block tiling). K
|
||||
# (qk_nope_head_dim) needs only K % 16 == 0 and K <= 256: power-of-2
|
||||
@@ -314,8 +327,7 @@ class DeepseekMLAForwardMixin:
|
||||
# --dcp-replicate-q-proj: project full-head Q locally from pre-gathered
|
||||
# weights and skip the per-layer Q all-gather (bf16 decode absorb only).
|
||||
q_replicate_active = (
|
||||
get_parallel().dcp_replicate_q_proj
|
||||
and is_dcp_mla_decode_phase(forward_batch)
|
||||
self.is_dcp_q_replicate_phase(forward_batch)
|
||||
and not self.use_deep_gemm_bmm
|
||||
and self.w_kc_qrep is not None
|
||||
and self.q_b_proj_qrep_weight is not None
|
||||
@@ -391,7 +403,7 @@ class DeepseekMLAForwardMixin:
|
||||
# full-head Q from the gathered weight (skips Q all-gather)
|
||||
q = torch.nn.functional.linear(q, self.q_b_proj_qrep_weight).view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.qk_head_dim,
|
||||
)
|
||||
else:
|
||||
@@ -463,7 +475,7 @@ class DeepseekMLAForwardMixin:
|
||||
hidden_states, self.q_b_proj_qrep_weight
|
||||
).view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.qk_head_dim,
|
||||
)
|
||||
else:
|
||||
@@ -633,8 +645,8 @@ class DeepseekMLAForwardMixin:
|
||||
)
|
||||
|
||||
# all_gather q_pe, q_nope_out,take tp8 as an example, q_pe [B, H, ROPE_DIM], q_nope_out [B, H, NOPE_DIM] gathered to [B, H * dcp_world_size, ROPE_DIM] [B, H * dcp_world_size, NOPE_DIM] for decode batch, and all gather k_pe, k_nope for extend batch.
|
||||
if get_parallel().dcp_enabled:
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.dcp_size > 1:
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
if not q_replicate_active:
|
||||
q_nope_out, q_pe = all_gather_q_for_mla_decode(
|
||||
q_nope_out=q_nope_out,
|
||||
@@ -727,7 +739,7 @@ class DeepseekMLAForwardMixin:
|
||||
topk_indices=topk_indices,
|
||||
)
|
||||
attn_output = fusion_plan.attn_output_buf
|
||||
elif is_dcp_mla_decode_phase(forward_batch):
|
||||
elif self.is_dcp_mla_decode_phase(forward_batch):
|
||||
# set return_lse=True to correct attn_output
|
||||
attn_output, lse = self.attn_mqa_for_dcp_decode(
|
||||
q_nope_out,
|
||||
@@ -776,10 +788,10 @@ class DeepseekMLAForwardMixin:
|
||||
)
|
||||
|
||||
# correct attn_output with respect to lse from other ranks
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
attn_output = attn_output.view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.kv_lora_rank,
|
||||
)
|
||||
if get_in_autotune_dummy_run():
|
||||
|
||||
+11
-13
@@ -51,7 +51,6 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mla import (
|
||||
_select_local_dcp_heads_for_autotune,
|
||||
is_dcp_mla_decode_phase,
|
||||
is_mla_dcp_lse_base_on_e,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.utils import (
|
||||
@@ -441,7 +440,7 @@ def _can_fuse_bmm_rope_cat_and_cache(attn: DeepseekV2AttentionMLA) -> bool:
|
||||
"""
|
||||
return (
|
||||
_use_aiter_gfx95
|
||||
and not get_parallel().dcp_enabled
|
||||
and attn.dcp_size == 1
|
||||
and attn.w_kc.dtype == torch.float8_e4m3fn
|
||||
and attn.rotary_emb is not None
|
||||
and attn._skip_rope_for_dsa_tilelang_fused()
|
||||
@@ -503,8 +502,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
|
||||
q_replicate_active = (
|
||||
get_parallel().dcp_replicate_q_proj
|
||||
and is_dcp_mla_decode_phase(forward_batch)
|
||||
self.is_dcp_q_replicate_phase(forward_batch)
|
||||
and not self.use_deep_gemm_bmm
|
||||
and self.w_kc_qrep is not None
|
||||
and self.q_b_proj_qrep_weight is not None
|
||||
@@ -634,7 +632,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
if q_replicate_active:
|
||||
q = torch.nn.functional.linear(q, self.q_b_proj_qrep_weight).view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.qk_head_dim,
|
||||
)
|
||||
else:
|
||||
@@ -659,7 +657,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
hidden_states, self.q_b_proj_qrep_weight
|
||||
).view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.qk_head_dim,
|
||||
)
|
||||
else:
|
||||
@@ -740,7 +738,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
fuse_rope_for_trtllm_mla = self._fuse_rope_for_trtllm_mla(forward_batch)
|
||||
|
||||
force_rope_for_aiter_dcp_decode = (
|
||||
get_parallel().dcp_enabled
|
||||
self.dcp_size > 1
|
||||
and (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
@@ -771,8 +769,8 @@ class DeepseekMLARocmForwardMixin:
|
||||
q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
|
||||
|
||||
# all_gather q_pe, q_nope_out,take tp8 as an example, q_pe [B, H, ROPE_DIM], q_nope_out [B, H, NOPE_DIM] gathered to [B, H * dcp_world_size, ROPE_DIM] [B, H * dcp_world_size, NOPE_DIM] for decode batch, and all gather k_pe, k_nope for extend batch.
|
||||
if get_parallel().dcp_enabled:
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.dcp_size > 1:
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
if not q_replicate_active:
|
||||
q_nope_out, q_pe = all_gather_q_for_mla_decode(
|
||||
q_nope_out=q_nope_out,
|
||||
@@ -910,7 +908,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
"is_neox": self.rotary_emb.is_neox_style,
|
||||
"llama_4_scaling": llama_4_scaling,
|
||||
}
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
# set return_lse=True to correct attn_output
|
||||
attn_output, lse = self.attn_mqa_for_dcp_decode(
|
||||
q_nope_out,
|
||||
@@ -948,7 +946,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
or forward_batch.forward_mode.is_draft_extend_v2()
|
||||
)
|
||||
and get_parallel().dcp_enabled
|
||||
and self.dcp_size > 1
|
||||
):
|
||||
q = torch.cat([q_nope_out, q_pe], dim=-1)
|
||||
if llama_4_scaling is not None:
|
||||
@@ -1003,10 +1001,10 @@ class DeepseekMLARocmForwardMixin:
|
||||
)
|
||||
|
||||
# correct attn_output with respect to lse from other ranks
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
attn_output = attn_output.view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.kv_lora_rank,
|
||||
)
|
||||
if get_in_autotune_dummy_run():
|
||||
|
||||
@@ -2018,6 +2018,9 @@ class DeepseekV2AttentionMLA(
|
||||
self.is_nextn = is_nextn
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
self.use_dsa = is_deepseek_dsa(config)
|
||||
self.dcp_size = (
|
||||
1 if (is_nextn and self.use_dsa) else get_parallel().attn_dcp_size
|
||||
)
|
||||
self.num_heads = num_heads
|
||||
assert num_heads % attn_tp_size == 0
|
||||
self.num_local_heads = num_heads // attn_tp_size
|
||||
@@ -2156,9 +2159,9 @@ class DeepseekV2AttentionMLA(
|
||||
prefix=add_prefix("attn_mqa", prefix),
|
||||
)
|
||||
# use num_local_heads * dcp_world_size because q_nope, q_rope is all gathered from dcp ranks
|
||||
if get_parallel().dcp_enabled:
|
||||
if self.dcp_size > 1:
|
||||
self.attn_mqa_for_dcp_decode = RadixAttention(
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=1,
|
||||
@@ -3441,7 +3444,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
kv_cache_device,
|
||||
create_chunked_prefix_cache_kv_indices_fn,
|
||||
):
|
||||
if _is_npu:
|
||||
if _is_npu or is_deepseek_dsa(self.config):
|
||||
return None
|
||||
return prepare_decode_context_parallel_metadata(
|
||||
seq_lens=seq_lens,
|
||||
|
||||
Reference in New Issue
Block a user