[DCP] Decode context parallelism for DSA (GLM-5.2 / GLM-5.3) with MTP (#43735)

Co-authored-by: vincentzed <207368749+vincentzed@users.noreply.github.com>
This commit is contained in:
Khoa Pham
2026-10-13 11:35:34 -07:00
committed by GitHub
co-authored by vincentzed
parent 6ade23696d
commit 8bf145d119
26 changed files with 565 additions and 88 deletions
@@ -181,6 +181,7 @@ for _mod, _fn in [
("dsa.transform_index", "transform_index_page_table_prefill"),
("dsa.transform_index", "transform_index_page_table_decode"),
("dsa.transform_index", "prepare_trtllm_nope_sparse_metadata"),
("dsa.transform_index", "transform_index_page_table_dcp"),
("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"),
("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"),
("dsv4.fp4_indexer", "store_fp4_index_k_cache"),
@@ -605,7 +605,7 @@ def _dcp_lse_combine_kernel(
partial_out = tl.load(recv_output_ptr + o_offsets).to(tl.float32)
acc += partial_out * w
acc = acc / weight_sum
acc = acc / tl.maximum(weight_sum, 1.0)
out_offsets = (
batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D
@@ -710,7 +710,7 @@ def _lse_weighted_combine_cpu(
weights = torch.pow(2.0, centered)
weight_sum = weights.sum(dim=0, keepdim=True)
weights = weights / weight_sum
weights = weights / weight_sum.clamp_min(1.0)
combined = (partial_outputs * weights.unsqueeze(-1)).sum(dim=0)
return combined
@@ -14,6 +14,55 @@ def transform_index_page_table_decode(**kwargs):
return transform_index_page_table_decode_fast(**kwargs)
@triton.jit
def transform_index_page_table_dcp_kernel(
page_table_ptr,
result_ptr,
counts_ptr,
row_stride: tl.constexpr,
DCP_SIZE: tl.constexpr,
DCP_RANK: tl.constexpr,
TOPK: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
):
row = tl.program_id(0)
offsets = tl.arange(0, BLOCK_TOPK)
slots = tl.load(
page_table_ptr + row * row_stride + offsets, mask=offsets < TOPK, other=-1
)
owned = (slots >= 0) & (slots % DCP_SIZE == DCP_RANK)
packed_offsets = tl.cumsum(owned.to(tl.int32), axis=0) - 1
count = tl.sum(owned.to(tl.int32), axis=0)
tl.store(result_ptr + row * TOPK + packed_offsets, slots // DCP_SIZE, mask=owned)
tl.store(result_ptr + row * TOPK + count + offsets, -1, mask=offsets < TOPK - count)
tl.store(counts_ptr + row, count)
def transform_index_page_table_dcp(
page_table: torch.Tensor,
dcp_size: int,
dcp_rank: int,
) -> tuple[torch.Tensor, torch.Tensor]:
num_rows, topk = page_table.shape
result = torch.empty_like(page_table)
counts = torch.empty(num_rows, dtype=torch.int32, device=page_table.device)
if num_rows == 0 or topk == 0:
return result, counts.zero_()
assert page_table.stride(1) == 1
transform_index_page_table_dcp_kernel[(num_rows,)](
page_table,
result,
counts,
page_table.stride(0),
DCP_SIZE=dcp_size,
DCP_RANK=dcp_rank,
TOPK=topk,
BLOCK_TOPK=triton.next_power_of_2(topk),
num_warps=4,
)
return result, counts
@triton.jit
def prepare_trtllm_nope_sparse_metadata_kernel(
page_table_ptr: torch.Tensor,
@@ -15,6 +15,25 @@ from sglang.srt.runtime_context import get_platform
logger = logging.getLogger(__name__)
def _dsa_dcp_overrides(cfg: Any) -> dict:
if cfg.dcp_size <= 1:
return {}
if cfg.enable_hisparse or cfg.enable_hierarchical_cache:
raise ValueError("DSA DCP does not support HiSparse or HiCache.")
if cfg.disaggregation_mode != "null":
raise ValueError("DSA DCP does not support PD disaggregation.")
if cfg.speculative_eagle_topk not in (None, 1):
raise ValueError("DSA DCP requires --speculative-eagle-topk 1.")
overrides = {
name: "trtllm"
for name in ("dsa_prefill_backend", "dsa_decode_backend")
if getattr(cfg, name) is None
}
if cfg.dcp_replicate_q_proj is None and cfg.dcp_comm_backend in ("a2a", "fi_a2a"):
overrides["dcp_replicate_q_proj"] = True
return overrides
@_register_for(
"DeepseekV3ForCausalLM",
"DeepseekV32ForCausalLM",
@@ -78,6 +97,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
)
if is_deepseek_dsa(hf_config): # DeepSeek 3.2/GLM 5
overrides.update(_dsa_dcp_overrides(cfg))
# Set attention backend for DeepSeek
if is_attention_backend_not_set(cfg):
overrides["attention_backend"] = "dsa"
@@ -240,6 +240,8 @@ def handle_decode_context_parallelism(server_args: Any):
"--decode-context-parallel-size) must be >= 1, but got "
f"dcp_size={cfg.dcp_size}."
)
if cfg.dcp_size > 1 and cfg.enable_prefill_cp:
raise ValueError("--dcp-size > 1 does not support --enable-prefill-cp.")
if cfg.dcp_comm_backend in ("a2a", "fi_a2a") and cfg.dcp_size <= 1:
raise ValueError(
f"--dcp-comm-backend {cfg.dcp_comm_backend} only affects the "
@@ -0,0 +1,118 @@
from __future__ import annotations
from typing import Optional
import msgspec
import torch
from sglang.kernels.ops.attention.dsa.transform_index import (
transform_index_page_table_dcp,
)
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
TRTLLM_MAX_BATCH_ROWS = 65535
# FlashInfer carves heads * rows * 256 float2 softmax slots out of the workspace per call;
# the rest is kernel scratch.
_SOFTMAX_BYTES_PER_ROW_HEAD = 256 * 8
_SCRATCH_RESERVE_BYTES = 128 << 20
def trtllm_rows_per_call(*, workspace_bytes: int, heads: int) -> int:
rows = (workspace_bytes - _SCRATCH_RESERVE_BYTES) // (
heads * _SOFTMAX_BYTES_PER_ROW_HEAD
)
return max(1, min(TRTLLM_MAX_BATCH_ROWS, rows))
class DcpPageTableCache(msgspec.Struct):
"""NOTE(vincentzed): GLM-5.2/5.3 reuse the previous layer's top-k tensor in
57 of 78 layers; remapping once per fresh top-k cuts 78 launches/step to 21.
"""
metadata: object = None
layer_id: int = -1
topk_indices: Optional[torch.Tensor] = None
page_table: Optional[torch.Tensor] = None
counts: Optional[torch.Tensor] = None
def localize_dcp_page_table(
cache: DcpPageTableCache,
*,
metadata: object,
layer_id: int,
topk_indices: Optional[torch.Tensor],
page_table: torch.Tensor,
dcp_size: int,
dcp_rank: int,
) -> tuple[torch.Tensor, torch.Tensor]:
reuse = (
topk_indices is not None
and cache.metadata is metadata
and cache.layer_id == layer_id - 1
and cache.topk_indices is topk_indices
)
if not reuse:
cache.metadata = metadata
cache.topk_indices = topk_indices
cache.page_table, cache.counts = transform_index_page_table_dcp(
page_table, dcp_size, dcp_rank
)
cache.layer_id = layer_id
return cache.page_table, cache.counts
def dcp_trtllm_sparse_attention(
*,
query: torch.Tensor,
block_tables: torch.Tensor,
seq_lens: torch.Tensor,
cu_seqlens_q: torch.Tensor,
**attention_kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
import flashinfer.decode
rows, _, heads, _ = query.shape
out_dtype = torch.bfloat16
if rows == 0:
return (
torch.empty(
(0, 1, heads, attention_kwargs["kv_lora_rank"]),
dtype=out_dtype,
device=query.device,
),
torch.empty((0, heads), dtype=torch.float32, device=query.device),
)
lse = torch.empty((rows, heads), dtype=torch.float32, device=query.device)
workspace = attention_kwargs["workspace_buffer"]
step = trtllm_rows_per_call(
workspace_bytes=workspace.numel() * workspace.element_size(), heads=heads
)
if rows > step:
attention_kwargs["enable_pdl"] = False
top_k_lens = attention_kwargs.pop("sparse_mla_top_k_lens", None)
outputs = []
for start in range(0, rows, step):
end = min(start + step, rows)
output, _ = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla(
query=query[start:end],
block_tables=block_tables[start:end],
seq_lens=seq_lens[start:end],
return_lse=True,
lse=lse[start:end],
sparse_mla_top_k_lens=(
None if top_k_lens is None else top_k_lens[start:end]
),
**attention_kwargs,
)
output = output.view(end - start, heads, -1)
fixup_zero_kv_rows(
output,
lse[start:end],
seq_lens[start:end],
cu_seqlens_q[: end - start + 1],
1,
)
outputs.append(output)
output = outputs[0] if len(outputs) == 1 else torch.cat(outputs)
return output.view(rows, 1, heads, -1), lse
@@ -638,7 +638,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
pool.invalidate_index_buffer_for_layer(layer_id)
kv_cache = (
pool.get_index_k_with_scale_buffer(layer_id=layer_id)
.view(-1, pool.page_size, self.head_dim + 4)
.view(-1, pool.index_page_size, self.head_dim + 4)
.view(fp8_dtype)
)
@@ -739,8 +739,8 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
return None
pool = get_token_to_kv_pool()
if pool.page_size != PAGE_SIZE:
_decline_fused(f"pool page_size is {pool.page_size}, not {PAGE_SIZE}")
if pool.index_page_size != PAGE_SIZE:
_decline_fused(f"pool page_size is {pool.index_page_size}, not {PAGE_SIZE}")
return None
# DSA cache layer split: the standard path stores index-K per layer, and the
# fused path addresses the same buffer directly.
@@ -1000,7 +1000,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
if out_cache_loc is None:
out_cache_loc = forward_batch.out_cache_loc
pool = get_token_to_kv_pool()
page_size = pool.page_size
page_size = pool.index_page_size
if hasattr(pool, "invalidate_index_buffer_for_layer"):
pool.invalidate_index_buffer_for_layer(layer_id)
if hasattr(pool, "_is_layer_owned") and not pool._is_layer_owned(layer_id):
@@ -1199,7 +1199,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
if TYPE_CHECKING:
assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool)
page_size = get_token_to_kv_pool().page_size
page_size = get_token_to_kv_pool().index_page_size
# NOTE(dark): blocksize = 64 is hardcoded in deep_gemm
if _is_hip:
if _use_aiter_preshuffle:
@@ -1463,7 +1463,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
assert forward_batch.forward_mode.is_extend_without_speculative()
page_size = get_token_to_kv_pool().page_size
page_size = get_token_to_kv_pool().index_page_size
if _is_hip:
if _use_aiter_preshuffle:
assert page_size % 16 == 0, (
@@ -1808,7 +1808,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
and can_use_dsa_fused_store(
key.dtype,
out_cache_loc.dtype,
pool.page_size,
pool.index_page_size,
)
):
# NOTE: wrapper already normalizes shape/contiguity and asserts dtypes.
@@ -1817,7 +1817,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
key,
buf,
out_cache_loc,
pool.page_size,
pool.index_page_size,
)
return
@@ -1827,7 +1827,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
# layout with page_size=1; the same kv_cache.view works for both cases
# because page_size is 1 there.
if _use_aiter:
page_size = pool.page_size
page_size = pool.index_page_size
buf = pool.get_index_k_with_scale_buffer(layer_id=layer_id)
kv_cache = buf.view(-1, page_size, 132).view(fp8_dtype)
out_loc = forward_batch.out_cache_loc
@@ -60,6 +60,11 @@ from sglang.srt.layers.attention.dsa.dsa_backend_mtp_precompute import (
PrecomputedMetadata,
compute_cu_seqlens,
)
from sglang.srt.layers.attention.dsa.dsa_dcp import (
DcpPageTableCache,
dcp_trtllm_sparse_attention,
localize_dcp_page_table,
)
from sglang.srt.layers.attention.dsa.dsa_indexer_metadata import DSAIndexerMetadata
from sglang.srt.layers.attention.dsa.dsa_metadata_manager import (
DSAMetadataManagementMixin,
@@ -529,6 +534,13 @@ class DeepseekSparseAttnBackend(
self.num_q_heads = (
model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size
)
if model_runner.is_draft_worker:
self.dcp_size, self.dcp_rank = 1, 0
else:
self.dcp_size = get_parallel().attn_dcp_size
self.dcp_rank = get_parallel().attn_dcp_rank
self.num_dcp_q_heads = self.num_q_heads * self.dcp_size
self._dcp_page_table_cache = DcpPageTableCache()
self.kv_cache_dim = model_runner.token_to_kv_pool.kv_cache_dim
self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim
self.kv_lora_rank = model_runner.model_config.kv_lora_rank
@@ -543,7 +555,7 @@ class DeepseekSparseAttnBackend(
self.req_to_token = model_runner.req_to_token_pool.req_to_token
self.use_mha: bool = False
self.supports_mha_one_shot: bool = True
self.supports_mha_one_shot: bool = self.dcp_size == 1
self.dsa_prefill_impl: _DSA_IMPL_T = get_exec().kernel.dsa_prefill_backend
self.dsa_decode_impl: _DSA_IMPL_T = get_exec().kernel.dsa_decode_backend
if (
@@ -742,7 +754,7 @@ class DeepseekSparseAttnBackend(
self.workspace_buffer = get_buffer(
"dsa_trtllm_workspace",
lambda: torch.empty(
envs.SGLANG_FLASHINFER_WORKSPACE_SIZE.get(),
envs.SGLANG_FLASHINFER_WORKSPACE_SIZE.get() * self.dcp_size,
dtype=torch.uint8,
device=model_runner.device,
),
@@ -750,7 +762,7 @@ class DeepseekSparseAttnBackend(
self._multi_ctas_kv_counter_buffer = (
make_persistent_multi_ctas_kv_counter_buffer(
torch.device(self.device),
self.num_q_heads,
self.num_dcp_q_heads,
max_batch_size=model_runner.max_running_requests,
)
)
@@ -1525,7 +1537,7 @@ class DeepseekSparseAttnBackend(
counter = grow_multi_ctas_kv_counter_buffer_if_needed(
buffer=self._multi_ctas_kv_counter_buffer,
device=torch.device(self.device),
num_q_heads=self.num_q_heads,
num_q_heads=self.num_dcp_q_heads,
batch_size=num_query_rows,
)
# Capacity is set before capture, so a grow here is a broken invariant.
@@ -1543,7 +1555,7 @@ class DeepseekSparseAttnBackend(
grow_multi_ctas_kv_counter_buffer_if_needed(
buffer=self._multi_ctas_kv_counter_buffer,
device=torch.device(self.device),
num_q_heads=self.num_q_heads,
num_q_heads=self.num_dcp_q_heads,
batch_size=num_query_rows,
)
)
@@ -3774,6 +3786,8 @@ class DeepseekSparseAttnBackend(
else:
q_all = q.view(-1, layer.tp_q_head_num, layer.head_dim)
shared_topk_indices = topk_indices
# Eager DP attention can pad q beyond metadata that was deliberately
# planned on the real draft batch. Pad top-k to the physical q shape,
# then run decode attention only on metadata-backed rows. The output is
@@ -3815,6 +3829,16 @@ class DeepseekSparseAttnBackend(
page_table_1, sparse_mla_top_k = self._pad_trtllm_sparse_page_table(
page_table_1
)
if self.dcp_size > 1:
page_table_1, seq_lens = localize_dcp_page_table(
self._dcp_page_table_cache,
metadata=metadata,
layer_id=layer.layer_id,
topk_indices=shared_topk_indices,
page_table=page_table_1,
dcp_size=self.dcp_size,
dcp_rank=self.dcp_rank,
)
sparse_mla_top_k_lens = None
if self.qk_rope_head_dim == 0:
sparse_mla_top_k_lens = prepare_trtllm_nope_sparse_metadata(page_table_1)
@@ -3827,8 +3851,7 @@ class DeepseekSparseAttnBackend(
)
bmm1_scale = q_scale * k_scale * layer.scaling
batch_size = page_table_1.shape[0]
_, num_heads, head_dim = q_all.shape
batch_size, num_heads, head_dim = q_all.shape
multi_ctas_kv_counter_buffer = self._multi_ctas_kv_counter_for(batch_size)
@@ -3837,15 +3860,12 @@ class DeepseekSparseAttnBackend(
block_tables = page_table_1.unsqueeze(1)
seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens
out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla(
query=q,
attention_kwargs = dict(
kv_cache=kv,
workspace_buffer=self.workspace_buffer,
qk_nope_head_dim=self.qk_nope_head_dim,
kv_lora_rank=self.kv_lora_rank,
qk_rope_head_dim=self.qk_rope_head_dim,
block_tables=block_tables,
seq_lens=seq_lens,
max_seq_len=metadata.max_seq_len_k,
sparse_mla_top_k=sparse_mla_top_k,
bmm1_scale=bmm1_scale,
@@ -3855,6 +3875,30 @@ class DeepseekSparseAttnBackend(
multi_ctas_kv_counter_buffer=multi_ctas_kv_counter_buffer,
)
if self.dcp_size > 1:
out, lse = dcp_trtllm_sparse_attention(
query=q,
block_tables=block_tables,
seq_lens=seq_lens,
cu_seqlens_q=self.get_device_int32_arange(batch_size + 1),
**attention_kwargs,
)
if num_decode_padding_rows:
lse = torch.cat(
[
lse,
lse.new_full((num_decode_padding_rows, num_heads), -torch.inf),
]
)
return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows), lse
out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla(
query=q,
block_tables=block_tables,
seq_lens=seq_lens,
**attention_kwargs,
)
return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows)
def _pad_topk_indices(
+11 -3
View File
@@ -35,10 +35,18 @@ class IndexKeyCache:
pool = self.pool
return (
num_pages,
pool.index_page_size
pool.page_size
// pool.index_kpool
* (pool.index_head_dim + pool.index_head_dim // pool.quant_block_size * 4),
)
def _kernel_view(self, buffer: torch.Tensor) -> torch.Tensor:
pool = self.pool
kernel_pages_per_row = (
pool.page_size // pool.index_kpool // pool.index_page_size
)
return buffer.view(-1, buffer.shape[1] // kernel_pages_per_row)
def _layer_num_pages(self, layer_idx: int, num_pages: int) -> int:
# Layers that reuse the previous layer's top-k never write index-K, so
# they get a 0-row placeholder that keeps ``buffer`` layer-aligned.
@@ -62,7 +70,7 @@ class IndexKeyCache:
self.pool.layer_transfer_counter.wait_until(
layer_id - self.pool.start_layer
)
return self.buffer[layer_id - self.pool.start_layer]
return self._kernel_view(self.buffer[layer_id - self.pool.start_layer])
def get_buffer(self, layer_id: int) -> torch.Tensor:
return self.get_local_buffer(layer_id)
@@ -92,7 +100,7 @@ class IndexKeyCache:
index_k: torch.Tensor,
index_k_scale: torch.Tensor,
) -> None:
buf = self.buffer[layer_id - self.pool.start_layer]
buf = self._kernel_view(self.buffer[layer_id - self.pool.start_layer])
index_buf_accessor.SetKAndS.execute(
pool=self.pool,
buf=buf,
@@ -2042,6 +2042,8 @@ class KVCacheConfigurator:
pool_kwargs["layer_shard_size"] = dsa_cp_layer_shard_size
else:
PoolCls = dsa_pool_class
if self.loc_space_scale > 1:
pool_kwargs["dcp_replicated"] = True
if _should_elide_dsa_index_k(is_draft_worker=self.is_draft_worker):
pool_kwargs["skip_topk_layers"] = [
dsa_layer_skips_topk(self.model_config.hf_config, layer_id)
+23 -5
View File
@@ -4684,6 +4684,7 @@ class MLATokenToKVPool(KVCache):
end_layer: Optional[int] = None,
use_dsa: bool = False,
override_kv_cache_dim: Optional[int] = None,
dcp_replicated: bool = False,
):
super().__init__(
size,
@@ -4699,6 +4700,7 @@ class MLATokenToKVPool(KVCache):
self.kv_lora_rank = kv_lora_rank
self.qk_rope_head_dim = qk_rope_head_dim
self.use_dsa = use_dsa
self.dcp_replicated = dcp_replicated
self.dsa_kv_cache_store_fp8 = (
use_dsa
and dtype == torch.float8_e4m3fn
@@ -4786,11 +4788,14 @@ class MLATokenToKVPool(KVCache):
# False: this pool takes a WIDENED loc. The unified pool resolves it in
# the plan's write translation (`translate_write_loc`) and flips this.
write_loc_is_dcp_resolved = False
dcp_replicated = False
@property
def _write_loc_dcp_span(self) -> int:
"""How many logical ids one stored row spans in the write-loc space."""
return 1 if self.write_loc_is_dcp_resolved else get_parallel().attn_dcp_size
if self.write_loc_is_dcp_resolved or self.dcp_replicated:
return 1
return get_parallel().attn_dcp_size
def _scatter_mla_rows(
self,
@@ -4799,7 +4804,7 @@ class MLATokenToKVPool(KVCache):
cache_k_nope: torch.Tensor,
cache_k_rope: torch.Tensor,
) -> None:
if self.write_loc_is_dcp_resolved:
if self.write_loc_is_dcp_resolved or self.dcp_replicated:
set_mla_kv_buffer_triton(dst_buffer, loc, cache_k_nope, cache_k_rope)
else:
set_mla_kv_buffer_dcp_sharded_triton(
@@ -4823,7 +4828,11 @@ class MLATokenToKVPool(KVCache):
assert not self.dsa_kv_cache_store_fp8
# No DCP-aware variant is possible: the two backends reaching this door
# disagree on the loc space (flashinfer-MLA widened, Triton collapsed).
assert self.write_loc_is_dcp_resolved or not get_parallel().dcp_enabled, (
assert (
self.write_loc_is_dcp_resolved
or self.dcp_replicated
or not get_parallel().dcp_enabled
), (
"MLATokenToKVPool.set_kv_buffer has no DCP-aware write path. Under "
"--dcp-size > 1 the MLA write must go through set_mla_kv_buffer, "
"whose kernel resolves the owner rule; reaching the combined-row "
@@ -5164,6 +5173,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
tail_extra_slots: int = 0,
max_running_requests: Optional[int] = None,
skip_topk_layers: Optional[List[bool]] = None,
dcp_replicated: bool = False,
):
override_dim = (
kv_cache_dim if kv_cache_dim != kv_lora_rank + qk_rope_head_dim else None
@@ -5182,6 +5192,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
end_layer,
use_dsa=True,
override_kv_cache_dim=override_dim,
dcp_replicated=dcp_replicated,
)
# self.index_k_dtype = torch.float8_e4m3fn
# self.index_k_scale_dtype = torch.float32
@@ -5193,8 +5204,15 @@ class DSATokenToKVPool(MLATokenToKVPool):
f"page_size {self.page_size} must be a multiple of index_kpool {index_kpool}"
)
self.index_page_size = self.page_size // index_kpool
if dcp_replicated:
self.index_page_size //= get_parallel().attn_dcp_size
if index_buf_size is None:
index_buf_size = size
dcp_size = get_parallel().attn_dcp_size
index_buf_size = (
size
if dcp_replicated
else (size + self.page_size) * dcp_size - self.page_size
)
self.index_buf_size = index_buf_size
# num head == 1 and head dim == 128 for index_k in DSA
assert index_head_dim == 128
@@ -5206,7 +5224,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
)
assert len(self.skip_topk_layers) == layer_num
physical_page_size = self.page_size // index_kpool
physical_page_size = self.index_page_size
if _is_hip:
if aiter_can_use_preshuffle_paged_mqa():
assert physical_page_size % 16 == 0, (
@@ -270,8 +270,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
kvc=kvc,
num_layers=num_layers,
)
if _is_npu and dcp_size > 1:
target_indexer_size *= dcp_size
target_indexer_size *= dcp_size
target_kv_size = self._cell_size - target_indexer_size
from sglang.srt.layers.cp.utils import (
get_glm_dsa_layer_split_effective_num_layers,
@@ -288,11 +287,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
num_layers=draft_num_layers,
allocate_all_layers=True,
)
# The draft pool is replicated and consumes the widened
# allocator-global slot space on NPU DCP.
if _is_npu and dcp_size > 1:
draft_kv_size *= dcp_size
draft_indexer_size *= dcp_size
draft_kv_size *= dcp_size
draft_indexer_size *= dcp_size
self._cell_size += draft_kv_size + draft_indexer_size
else:
self._cell_size = int(
@@ -388,7 +384,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
kvc=kvc,
num_layers=num_layers,
)
if _is_npu and not kvc.is_draft_worker and dcp_size > 1:
if not kvc.is_draft_worker:
indexer_cell_size *= dcp_size
cell_size += indexer_cell_size
elif is_minimax_sparse(model_config.hf_config):
@@ -266,7 +266,7 @@ class DeepseekMHAForwardMixin:
kv_a, k_pe = self._get_mla_kv_buffer_from_fp8_for_dsa(forward_batch)
else:
# BF16/FP16 path: directly fetch from cache
if get_parallel().dcp_enabled:
if self.dcp_size > 1:
kv_a, k_pe = all_gather_kv_cache_for_mha_extend(
get_token_to_kv_pool(),
self.attn_mha,
@@ -34,7 +34,7 @@ from sglang.srt.models.deepseek_common.utils import (
_use_aiter_bpreshuffle_gfx95,
_use_aiter_gfx95,
)
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import BumpAllocator, get_bool_env_var
if TYPE_CHECKING:
@@ -232,7 +232,7 @@ class DeepseekMHARocmForwardMixin:
kv_a, k_pe = self._get_mla_kv_buffer_from_fp8_for_dsa(forward_batch)
else:
# BF16/FP16 path: directly fetch from cache
if get_parallel().dcp_enabled:
if self.dcp_size > 1:
kv_a, k_pe = all_gather_kv_cache_for_mha_extend(
get_token_to_kv_pool(),
self.attn_mha,
@@ -92,15 +92,6 @@ def _select_local_dcp_heads_for_autotune(
return attn_output.narrow(1, rank * num_local_heads, num_local_heads)
def is_dcp_mla_decode_phase(forward_batch: ForwardBatch) -> bool:
if not get_parallel().dcp_enabled:
return False
return (
forward_batch.forward_mode.is_decode()
or forward_batch.forward_mode.is_target_verify()
)
def is_mla_dcp_lse_base_on_e(attention_backend: Optional[str]) -> bool:
return attention_backend in {"flashmla", "cutedsl_mla", "aiter"}
@@ -131,6 +122,26 @@ def _apply_attention_output_gate(module, attn_output, gate):
class DeepseekMLAForwardMixin:
def is_dcp_mla_decode_phase(
self: DeepseekV2AttentionMLA, forward_batch: ForwardBatch
) -> bool:
if self.dcp_size == 1:
return False
mode = forward_batch.forward_mode
return (
mode.is_decode()
or mode.is_target_verify()
or (self.use_dsa and mode.is_extend())
)
def is_dcp_q_replicate_phase(
self: DeepseekV2AttentionMLA, forward_batch: ForwardBatch
) -> bool:
if not get_parallel().dcp_replicate_q_proj or self.dcp_size == 1:
return False
mode = forward_batch.forward_mode
return mode.is_decode() or mode.is_target_verify()
def init_mla_forward(self: DeepseekV2AttentionMLA):
self.flashinfer_mla_disable_ragged = (
get_exec().kernel.flashinfer_mla_disable_ragged
@@ -162,6 +173,8 @@ class DeepseekMLAForwardMixin:
) -> bool:
if getattr(self, "_kimi_split_gguf_kv_b", False):
return False
if self.dcp_size > 1:
return False
# Shared activation surface with the DSA indexer graph dispatch
# (in piecewise/breakable graph + non-speculative extend). Like the indexer
# dispatch, this fusion is on by default on that surface.
@@ -264,7 +277,7 @@ class DeepseekMLAForwardMixin:
return None
if get_is_capture_mode():
return None
if get_parallel().dcp_enabled:
if self.dcp_size > 1:
return None
# Kernel shape constraints (tl.arange / tl.dot / block tiling). K
# (qk_nope_head_dim) needs only K % 16 == 0 and K <= 256: power-of-2
@@ -314,8 +327,7 @@ class DeepseekMLAForwardMixin:
# --dcp-replicate-q-proj: project full-head Q locally from pre-gathered
# weights and skip the per-layer Q all-gather (bf16 decode absorb only).
q_replicate_active = (
get_parallel().dcp_replicate_q_proj
and is_dcp_mla_decode_phase(forward_batch)
self.is_dcp_q_replicate_phase(forward_batch)
and not self.use_deep_gemm_bmm
and self.w_kc_qrep is not None
and self.q_b_proj_qrep_weight is not None
@@ -391,7 +403,7 @@ class DeepseekMLAForwardMixin:
# full-head Q from the gathered weight (skips Q all-gather)
q = torch.nn.functional.linear(q, self.q_b_proj_qrep_weight).view(
-1,
self.num_local_heads * get_parallel().attn_dcp_size,
self.num_local_heads * self.dcp_size,
self.qk_head_dim,
)
else:
@@ -463,7 +475,7 @@ class DeepseekMLAForwardMixin:
hidden_states, self.q_b_proj_qrep_weight
).view(
-1,
self.num_local_heads * get_parallel().attn_dcp_size,
self.num_local_heads * self.dcp_size,
self.qk_head_dim,
)
else:
@@ -633,8 +645,8 @@ class DeepseekMLAForwardMixin:
)
# all_gather q_pe, q_nope_out,take tp8 as an example, q_pe [B, H, ROPE_DIM], q_nope_out [B, H, NOPE_DIM] gathered to [B, H * dcp_world_size, ROPE_DIM] [B, H * dcp_world_size, NOPE_DIM] for decode batch, and all gather k_pe, k_nope for extend batch.
if get_parallel().dcp_enabled:
if is_dcp_mla_decode_phase(forward_batch):
if self.dcp_size > 1:
if self.is_dcp_mla_decode_phase(forward_batch):
if not q_replicate_active:
q_nope_out, q_pe = all_gather_q_for_mla_decode(
q_nope_out=q_nope_out,
@@ -727,7 +739,7 @@ class DeepseekMLAForwardMixin:
topk_indices=topk_indices,
)
attn_output = fusion_plan.attn_output_buf
elif is_dcp_mla_decode_phase(forward_batch):
elif self.is_dcp_mla_decode_phase(forward_batch):
# set return_lse=True to correct attn_output
attn_output, lse = self.attn_mqa_for_dcp_decode(
q_nope_out,
@@ -776,10 +788,10 @@ class DeepseekMLAForwardMixin:
)
# correct attn_output with respect to lse from other ranks
if is_dcp_mla_decode_phase(forward_batch):
if self.is_dcp_mla_decode_phase(forward_batch):
attn_output = attn_output.view(
-1,
self.num_local_heads * get_parallel().attn_dcp_size,
self.num_local_heads * self.dcp_size,
self.kv_lora_rank,
)
if get_in_autotune_dummy_run():
@@ -51,7 +51,6 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
)
from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mla import (
_select_local_dcp_heads_for_autotune,
is_dcp_mla_decode_phase,
is_mla_dcp_lse_base_on_e,
)
from sglang.srt.models.deepseek_common.utils import (
@@ -441,7 +440,7 @@ def _can_fuse_bmm_rope_cat_and_cache(attn: DeepseekV2AttentionMLA) -> bool:
"""
return (
_use_aiter_gfx95
and not get_parallel().dcp_enabled
and attn.dcp_size == 1
and attn.w_kc.dtype == torch.float8_e4m3fn
and attn.rotary_emb is not None
and attn._skip_rope_for_dsa_tilelang_fused()
@@ -503,8 +502,7 @@ class DeepseekMLARocmForwardMixin:
from sglang.srt.model_executor.runner import get_is_capture_mode
q_replicate_active = (
get_parallel().dcp_replicate_q_proj
and is_dcp_mla_decode_phase(forward_batch)
self.is_dcp_q_replicate_phase(forward_batch)
and not self.use_deep_gemm_bmm
and self.w_kc_qrep is not None
and self.q_b_proj_qrep_weight is not None
@@ -634,7 +632,7 @@ class DeepseekMLARocmForwardMixin:
if q_replicate_active:
q = torch.nn.functional.linear(q, self.q_b_proj_qrep_weight).view(
-1,
self.num_local_heads * get_parallel().attn_dcp_size,
self.num_local_heads * self.dcp_size,
self.qk_head_dim,
)
else:
@@ -659,7 +657,7 @@ class DeepseekMLARocmForwardMixin:
hidden_states, self.q_b_proj_qrep_weight
).view(
-1,
self.num_local_heads * get_parallel().attn_dcp_size,
self.num_local_heads * self.dcp_size,
self.qk_head_dim,
)
else:
@@ -740,7 +738,7 @@ class DeepseekMLARocmForwardMixin:
fuse_rope_for_trtllm_mla = self._fuse_rope_for_trtllm_mla(forward_batch)
force_rope_for_aiter_dcp_decode = (
get_parallel().dcp_enabled
self.dcp_size > 1
and (
forward_batch.forward_mode.is_decode()
or forward_batch.forward_mode.is_target_verify()
@@ -771,8 +769,8 @@ class DeepseekMLARocmForwardMixin:
q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
# all_gather q_pe, q_nope_out,take tp8 as an example, q_pe [B, H, ROPE_DIM], q_nope_out [B, H, NOPE_DIM] gathered to [B, H * dcp_world_size, ROPE_DIM] [B, H * dcp_world_size, NOPE_DIM] for decode batch, and all gather k_pe, k_nope for extend batch.
if get_parallel().dcp_enabled:
if is_dcp_mla_decode_phase(forward_batch):
if self.dcp_size > 1:
if self.is_dcp_mla_decode_phase(forward_batch):
if not q_replicate_active:
q_nope_out, q_pe = all_gather_q_for_mla_decode(
q_nope_out=q_nope_out,
@@ -910,7 +908,7 @@ class DeepseekMLARocmForwardMixin:
"is_neox": self.rotary_emb.is_neox_style,
"llama_4_scaling": llama_4_scaling,
}
if is_dcp_mla_decode_phase(forward_batch):
if self.is_dcp_mla_decode_phase(forward_batch):
# set return_lse=True to correct attn_output
attn_output, lse = self.attn_mqa_for_dcp_decode(
q_nope_out,
@@ -948,7 +946,7 @@ class DeepseekMLARocmForwardMixin:
or forward_batch.forward_mode.is_target_verify()
or forward_batch.forward_mode.is_draft_extend_v2()
)
and get_parallel().dcp_enabled
and self.dcp_size > 1
):
q = torch.cat([q_nope_out, q_pe], dim=-1)
if llama_4_scaling is not None:
@@ -1003,10 +1001,10 @@ class DeepseekMLARocmForwardMixin:
)
# correct attn_output with respect to lse from other ranks
if is_dcp_mla_decode_phase(forward_batch):
if self.is_dcp_mla_decode_phase(forward_batch):
attn_output = attn_output.view(
-1,
self.num_local_heads * get_parallel().attn_dcp_size,
self.num_local_heads * self.dcp_size,
self.kv_lora_rank,
)
if get_in_autotune_dummy_run():
+6 -3
View File
@@ -2018,6 +2018,9 @@ class DeepseekV2AttentionMLA(
self.is_nextn = is_nextn
attn_tp_size = get_parallel().attn_tp_size
self.use_dsa = is_deepseek_dsa(config)
self.dcp_size = (
1 if (is_nextn and self.use_dsa) else get_parallel().attn_dcp_size
)
self.num_heads = num_heads
assert num_heads % attn_tp_size == 0
self.num_local_heads = num_heads // attn_tp_size
@@ -2156,9 +2159,9 @@ class DeepseekV2AttentionMLA(
prefix=add_prefix("attn_mqa", prefix),
)
# use num_local_heads * dcp_world_size because q_nope, q_rope is all gathered from dcp ranks
if get_parallel().dcp_enabled:
if self.dcp_size > 1:
self.attn_mqa_for_dcp_decode = RadixAttention(
self.num_local_heads * get_parallel().attn_dcp_size,
self.num_local_heads * self.dcp_size,
self.kv_lora_rank + self.qk_rope_head_dim,
self.scaling,
num_kv_heads=1,
@@ -3441,7 +3444,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
kv_cache_device,
create_chunked_prefix_cache_kv_indices_fn,
):
if _is_npu:
if _is_npu or is_deepseek_dsa(self.config):
return None
return prepare_decode_context_parallel_metadata(
seq_lens=seq_lens,