[DCP] Decode context parallelism for DSA (GLM-5.2 / GLM-5.3) with MTP (#43735)
Co-authored-by: vincentzed <207368749+vincentzed@users.noreply.github.com>
This commit is contained in:
@@ -172,6 +172,29 @@ Before H2D or D2H, both index lists keep positions owned by this rank and map `i
|
||||
|
||||
The current scope is MLA L1/L2. L3, LMCache, HiSparse, non-MLA KV host pools, speculative decoding, and PD decode are not supported in this combination.
|
||||
|
||||
### DCP × DSA (GLM-5.2, GLM-5.3)
|
||||
|
||||
DeepSeek Sparse Attention attends over a top-k slice of the context, so DCP keeps the target's latent KV striped by position while the **index-K cache stays replicated** on every rank: each rank scores the whole sequence, takes the global top-k, and attends over the selected slots it owns. Decode, target verify and extend all follow the same path, so the prefix KV is never gathered.
|
||||
|
||||
1. The index-K cache is addressed by the widened virtual slot and costs `c` times its usual share of each rank's KV budget (about 6% at DCP2 and 17% at DCP4 on GLM-5.3).
|
||||
2. Each layer maps the global top-k to rank-local rows with one Triton kernel; layers that share the previous layer's top-k reuse the mapping.
|
||||
3. The TRT-LLM sparse kernel returns the per-head LSE and the usual DCP merge combines the partial outputs.
|
||||
4. The MTP draft (nextn) layer keeps a replicated KV cache over the widened slots and attends without the DCP group, so MTP index share feeds the draft directly.
|
||||
|
||||
Requirements: CUDA SM100/SM103 with the `trtllm` DSA backends (selected automatically), an `fp8_e4m3` or `bf16` KV cache, `attn_tp_size % dcp_size == 0`, and, with speculative decoding, the checkpoint's own MTP draft as a single EAGLE chain (`--speculative-eagle-topk 1`, draft tokens equal to steps + 1). HiSparse, prefill CP, HiCache, PD disaggregation, unified memory and `index_kpool > 1` are not supported in this combination.
|
||||
|
||||
```bash Command
|
||||
sglang serve \
|
||||
--model-path zai-org/GLM-5.3 \
|
||||
--trust-remote-code \
|
||||
--tp-size 8 \
|
||||
--dcp-size 2 \
|
||||
--speculative-algorithm EAGLE \
|
||||
--speculative-num-steps 5 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 6
|
||||
```
|
||||
|
||||
## References
|
||||
|
||||
- [DCP and Helix Parallelism roadmap](https://github.com/sgl-project/sglang/issues/29736)
|
||||
|
||||
@@ -181,6 +181,7 @@ for _mod, _fn in [
|
||||
("dsa.transform_index", "transform_index_page_table_prefill"),
|
||||
("dsa.transform_index", "transform_index_page_table_decode"),
|
||||
("dsa.transform_index", "prepare_trtllm_nope_sparse_metadata"),
|
||||
("dsa.transform_index", "transform_index_page_table_dcp"),
|
||||
("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"),
|
||||
("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"),
|
||||
("dsv4.fp4_indexer", "store_fp4_index_k_cache"),
|
||||
|
||||
@@ -605,7 +605,7 @@ def _dcp_lse_combine_kernel(
|
||||
partial_out = tl.load(recv_output_ptr + o_offsets).to(tl.float32)
|
||||
acc += partial_out * w
|
||||
|
||||
acc = acc / weight_sum
|
||||
acc = acc / tl.maximum(weight_sum, 1.0)
|
||||
|
||||
out_offsets = (
|
||||
batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D
|
||||
@@ -710,7 +710,7 @@ def _lse_weighted_combine_cpu(
|
||||
weights = torch.pow(2.0, centered)
|
||||
|
||||
weight_sum = weights.sum(dim=0, keepdim=True)
|
||||
weights = weights / weight_sum
|
||||
weights = weights / weight_sum.clamp_min(1.0)
|
||||
|
||||
combined = (partial_outputs * weights.unsqueeze(-1)).sum(dim=0)
|
||||
return combined
|
||||
|
||||
@@ -14,6 +14,55 @@ def transform_index_page_table_decode(**kwargs):
|
||||
return transform_index_page_table_decode_fast(**kwargs)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def transform_index_page_table_dcp_kernel(
|
||||
page_table_ptr,
|
||||
result_ptr,
|
||||
counts_ptr,
|
||||
row_stride: tl.constexpr,
|
||||
DCP_SIZE: tl.constexpr,
|
||||
DCP_RANK: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
offsets = tl.arange(0, BLOCK_TOPK)
|
||||
slots = tl.load(
|
||||
page_table_ptr + row * row_stride + offsets, mask=offsets < TOPK, other=-1
|
||||
)
|
||||
owned = (slots >= 0) & (slots % DCP_SIZE == DCP_RANK)
|
||||
packed_offsets = tl.cumsum(owned.to(tl.int32), axis=0) - 1
|
||||
count = tl.sum(owned.to(tl.int32), axis=0)
|
||||
tl.store(result_ptr + row * TOPK + packed_offsets, slots // DCP_SIZE, mask=owned)
|
||||
tl.store(result_ptr + row * TOPK + count + offsets, -1, mask=offsets < TOPK - count)
|
||||
tl.store(counts_ptr + row, count)
|
||||
|
||||
|
||||
def transform_index_page_table_dcp(
|
||||
page_table: torch.Tensor,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_rows, topk = page_table.shape
|
||||
result = torch.empty_like(page_table)
|
||||
counts = torch.empty(num_rows, dtype=torch.int32, device=page_table.device)
|
||||
if num_rows == 0 or topk == 0:
|
||||
return result, counts.zero_()
|
||||
assert page_table.stride(1) == 1
|
||||
transform_index_page_table_dcp_kernel[(num_rows,)](
|
||||
page_table,
|
||||
result,
|
||||
counts,
|
||||
page_table.stride(0),
|
||||
DCP_SIZE=dcp_size,
|
||||
DCP_RANK=dcp_rank,
|
||||
TOPK=topk,
|
||||
BLOCK_TOPK=triton.next_power_of_2(topk),
|
||||
num_warps=4,
|
||||
)
|
||||
return result, counts
|
||||
|
||||
|
||||
@triton.jit
|
||||
def prepare_trtllm_nope_sparse_metadata_kernel(
|
||||
page_table_ptr: torch.Tensor,
|
||||
|
||||
@@ -15,6 +15,25 @@ from sglang.srt.runtime_context import get_platform
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _dsa_dcp_overrides(cfg: Any) -> dict:
|
||||
if cfg.dcp_size <= 1:
|
||||
return {}
|
||||
if cfg.enable_hisparse or cfg.enable_hierarchical_cache:
|
||||
raise ValueError("DSA DCP does not support HiSparse or HiCache.")
|
||||
if cfg.disaggregation_mode != "null":
|
||||
raise ValueError("DSA DCP does not support PD disaggregation.")
|
||||
if cfg.speculative_eagle_topk not in (None, 1):
|
||||
raise ValueError("DSA DCP requires --speculative-eagle-topk 1.")
|
||||
overrides = {
|
||||
name: "trtllm"
|
||||
for name in ("dsa_prefill_backend", "dsa_decode_backend")
|
||||
if getattr(cfg, name) is None
|
||||
}
|
||||
if cfg.dcp_replicate_q_proj is None and cfg.dcp_comm_backend in ("a2a", "fi_a2a"):
|
||||
overrides["dcp_replicate_q_proj"] = True
|
||||
return overrides
|
||||
|
||||
|
||||
@_register_for(
|
||||
"DeepseekV3ForCausalLM",
|
||||
"DeepseekV32ForCausalLM",
|
||||
@@ -78,6 +97,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
|
||||
if is_deepseek_dsa(hf_config): # DeepSeek 3.2/GLM 5
|
||||
overrides.update(_dsa_dcp_overrides(cfg))
|
||||
# Set attention backend for DeepSeek
|
||||
if is_attention_backend_not_set(cfg):
|
||||
overrides["attention_backend"] = "dsa"
|
||||
|
||||
@@ -240,6 +240,8 @@ def handle_decode_context_parallelism(server_args: Any):
|
||||
"--decode-context-parallel-size) must be >= 1, but got "
|
||||
f"dcp_size={cfg.dcp_size}."
|
||||
)
|
||||
if cfg.dcp_size > 1 and cfg.enable_prefill_cp:
|
||||
raise ValueError("--dcp-size > 1 does not support --enable-prefill-cp.")
|
||||
if cfg.dcp_comm_backend in ("a2a", "fi_a2a") and cfg.dcp_size <= 1:
|
||||
raise ValueError(
|
||||
f"--dcp-comm-backend {cfg.dcp_comm_backend} only affects the "
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsa.transform_index import (
|
||||
transform_index_page_table_dcp,
|
||||
)
|
||||
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
|
||||
|
||||
TRTLLM_MAX_BATCH_ROWS = 65535
|
||||
# FlashInfer carves heads * rows * 256 float2 softmax slots out of the workspace per call;
|
||||
# the rest is kernel scratch.
|
||||
_SOFTMAX_BYTES_PER_ROW_HEAD = 256 * 8
|
||||
_SCRATCH_RESERVE_BYTES = 128 << 20
|
||||
|
||||
|
||||
def trtllm_rows_per_call(*, workspace_bytes: int, heads: int) -> int:
|
||||
rows = (workspace_bytes - _SCRATCH_RESERVE_BYTES) // (
|
||||
heads * _SOFTMAX_BYTES_PER_ROW_HEAD
|
||||
)
|
||||
return max(1, min(TRTLLM_MAX_BATCH_ROWS, rows))
|
||||
|
||||
|
||||
class DcpPageTableCache(msgspec.Struct):
|
||||
"""NOTE(vincentzed): GLM-5.2/5.3 reuse the previous layer's top-k tensor in
|
||||
57 of 78 layers; remapping once per fresh top-k cuts 78 launches/step to 21.
|
||||
"""
|
||||
|
||||
metadata: object = None
|
||||
layer_id: int = -1
|
||||
topk_indices: Optional[torch.Tensor] = None
|
||||
page_table: Optional[torch.Tensor] = None
|
||||
counts: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def localize_dcp_page_table(
|
||||
cache: DcpPageTableCache,
|
||||
*,
|
||||
metadata: object,
|
||||
layer_id: int,
|
||||
topk_indices: Optional[torch.Tensor],
|
||||
page_table: torch.Tensor,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
reuse = (
|
||||
topk_indices is not None
|
||||
and cache.metadata is metadata
|
||||
and cache.layer_id == layer_id - 1
|
||||
and cache.topk_indices is topk_indices
|
||||
)
|
||||
if not reuse:
|
||||
cache.metadata = metadata
|
||||
cache.topk_indices = topk_indices
|
||||
cache.page_table, cache.counts = transform_index_page_table_dcp(
|
||||
page_table, dcp_size, dcp_rank
|
||||
)
|
||||
cache.layer_id = layer_id
|
||||
return cache.page_table, cache.counts
|
||||
|
||||
|
||||
def dcp_trtllm_sparse_attention(
|
||||
*,
|
||||
query: torch.Tensor,
|
||||
block_tables: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
cu_seqlens_q: torch.Tensor,
|
||||
**attention_kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import flashinfer.decode
|
||||
|
||||
rows, _, heads, _ = query.shape
|
||||
out_dtype = torch.bfloat16
|
||||
if rows == 0:
|
||||
return (
|
||||
torch.empty(
|
||||
(0, 1, heads, attention_kwargs["kv_lora_rank"]),
|
||||
dtype=out_dtype,
|
||||
device=query.device,
|
||||
),
|
||||
torch.empty((0, heads), dtype=torch.float32, device=query.device),
|
||||
)
|
||||
lse = torch.empty((rows, heads), dtype=torch.float32, device=query.device)
|
||||
workspace = attention_kwargs["workspace_buffer"]
|
||||
step = trtllm_rows_per_call(
|
||||
workspace_bytes=workspace.numel() * workspace.element_size(), heads=heads
|
||||
)
|
||||
if rows > step:
|
||||
attention_kwargs["enable_pdl"] = False
|
||||
top_k_lens = attention_kwargs.pop("sparse_mla_top_k_lens", None)
|
||||
outputs = []
|
||||
for start in range(0, rows, step):
|
||||
end = min(start + step, rows)
|
||||
output, _ = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=query[start:end],
|
||||
block_tables=block_tables[start:end],
|
||||
seq_lens=seq_lens[start:end],
|
||||
return_lse=True,
|
||||
lse=lse[start:end],
|
||||
sparse_mla_top_k_lens=(
|
||||
None if top_k_lens is None else top_k_lens[start:end]
|
||||
),
|
||||
**attention_kwargs,
|
||||
)
|
||||
output = output.view(end - start, heads, -1)
|
||||
fixup_zero_kv_rows(
|
||||
output,
|
||||
lse[start:end],
|
||||
seq_lens[start:end],
|
||||
cu_seqlens_q[: end - start + 1],
|
||||
1,
|
||||
)
|
||||
outputs.append(output)
|
||||
output = outputs[0] if len(outputs) == 1 else torch.cat(outputs)
|
||||
return output.view(rows, 1, heads, -1), lse
|
||||
@@ -638,7 +638,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
pool.invalidate_index_buffer_for_layer(layer_id)
|
||||
kv_cache = (
|
||||
pool.get_index_k_with_scale_buffer(layer_id=layer_id)
|
||||
.view(-1, pool.page_size, self.head_dim + 4)
|
||||
.view(-1, pool.index_page_size, self.head_dim + 4)
|
||||
.view(fp8_dtype)
|
||||
)
|
||||
|
||||
@@ -739,8 +739,8 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
return None
|
||||
|
||||
pool = get_token_to_kv_pool()
|
||||
if pool.page_size != PAGE_SIZE:
|
||||
_decline_fused(f"pool page_size is {pool.page_size}, not {PAGE_SIZE}")
|
||||
if pool.index_page_size != PAGE_SIZE:
|
||||
_decline_fused(f"pool page_size is {pool.index_page_size}, not {PAGE_SIZE}")
|
||||
return None
|
||||
# DSA cache layer split: the standard path stores index-K per layer, and the
|
||||
# fused path addresses the same buffer directly.
|
||||
@@ -1000,7 +1000,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
if out_cache_loc is None:
|
||||
out_cache_loc = forward_batch.out_cache_loc
|
||||
pool = get_token_to_kv_pool()
|
||||
page_size = pool.page_size
|
||||
page_size = pool.index_page_size
|
||||
if hasattr(pool, "invalidate_index_buffer_for_layer"):
|
||||
pool.invalidate_index_buffer_for_layer(layer_id)
|
||||
if hasattr(pool, "_is_layer_owned") and not pool._is_layer_owned(layer_id):
|
||||
@@ -1199,7 +1199,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool)
|
||||
|
||||
page_size = get_token_to_kv_pool().page_size
|
||||
page_size = get_token_to_kv_pool().index_page_size
|
||||
# NOTE(dark): blocksize = 64 is hardcoded in deep_gemm
|
||||
if _is_hip:
|
||||
if _use_aiter_preshuffle:
|
||||
@@ -1463,7 +1463,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
|
||||
assert forward_batch.forward_mode.is_extend_without_speculative()
|
||||
|
||||
page_size = get_token_to_kv_pool().page_size
|
||||
page_size = get_token_to_kv_pool().index_page_size
|
||||
if _is_hip:
|
||||
if _use_aiter_preshuffle:
|
||||
assert page_size % 16 == 0, (
|
||||
@@ -1808,7 +1808,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
and can_use_dsa_fused_store(
|
||||
key.dtype,
|
||||
out_cache_loc.dtype,
|
||||
pool.page_size,
|
||||
pool.index_page_size,
|
||||
)
|
||||
):
|
||||
# NOTE: wrapper already normalizes shape/contiguity and asserts dtypes.
|
||||
@@ -1817,7 +1817,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
key,
|
||||
buf,
|
||||
out_cache_loc,
|
||||
pool.page_size,
|
||||
pool.index_page_size,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1827,7 +1827,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
# layout with page_size=1; the same kv_cache.view works for both cases
|
||||
# because page_size is 1 there.
|
||||
if _use_aiter:
|
||||
page_size = pool.page_size
|
||||
page_size = pool.index_page_size
|
||||
buf = pool.get_index_k_with_scale_buffer(layer_id=layer_id)
|
||||
kv_cache = buf.view(-1, page_size, 132).view(fp8_dtype)
|
||||
out_loc = forward_batch.out_cache_loc
|
||||
|
||||
@@ -60,6 +60,11 @@ from sglang.srt.layers.attention.dsa.dsa_backend_mtp_precompute import (
|
||||
PrecomputedMetadata,
|
||||
compute_cu_seqlens,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.dsa_dcp import (
|
||||
DcpPageTableCache,
|
||||
dcp_trtllm_sparse_attention,
|
||||
localize_dcp_page_table,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer_metadata import DSAIndexerMetadata
|
||||
from sglang.srt.layers.attention.dsa.dsa_metadata_manager import (
|
||||
DSAMetadataManagementMixin,
|
||||
@@ -529,6 +534,13 @@ class DeepseekSparseAttnBackend(
|
||||
self.num_q_heads = (
|
||||
model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size
|
||||
)
|
||||
if model_runner.is_draft_worker:
|
||||
self.dcp_size, self.dcp_rank = 1, 0
|
||||
else:
|
||||
self.dcp_size = get_parallel().attn_dcp_size
|
||||
self.dcp_rank = get_parallel().attn_dcp_rank
|
||||
self.num_dcp_q_heads = self.num_q_heads * self.dcp_size
|
||||
self._dcp_page_table_cache = DcpPageTableCache()
|
||||
self.kv_cache_dim = model_runner.token_to_kv_pool.kv_cache_dim
|
||||
self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim
|
||||
self.kv_lora_rank = model_runner.model_config.kv_lora_rank
|
||||
@@ -543,7 +555,7 @@ class DeepseekSparseAttnBackend(
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
|
||||
self.use_mha: bool = False
|
||||
self.supports_mha_one_shot: bool = True
|
||||
self.supports_mha_one_shot: bool = self.dcp_size == 1
|
||||
self.dsa_prefill_impl: _DSA_IMPL_T = get_exec().kernel.dsa_prefill_backend
|
||||
self.dsa_decode_impl: _DSA_IMPL_T = get_exec().kernel.dsa_decode_backend
|
||||
if (
|
||||
@@ -742,7 +754,7 @@ class DeepseekSparseAttnBackend(
|
||||
self.workspace_buffer = get_buffer(
|
||||
"dsa_trtllm_workspace",
|
||||
lambda: torch.empty(
|
||||
envs.SGLANG_FLASHINFER_WORKSPACE_SIZE.get(),
|
||||
envs.SGLANG_FLASHINFER_WORKSPACE_SIZE.get() * self.dcp_size,
|
||||
dtype=torch.uint8,
|
||||
device=model_runner.device,
|
||||
),
|
||||
@@ -750,7 +762,7 @@ class DeepseekSparseAttnBackend(
|
||||
self._multi_ctas_kv_counter_buffer = (
|
||||
make_persistent_multi_ctas_kv_counter_buffer(
|
||||
torch.device(self.device),
|
||||
self.num_q_heads,
|
||||
self.num_dcp_q_heads,
|
||||
max_batch_size=model_runner.max_running_requests,
|
||||
)
|
||||
)
|
||||
@@ -1525,7 +1537,7 @@ class DeepseekSparseAttnBackend(
|
||||
counter = grow_multi_ctas_kv_counter_buffer_if_needed(
|
||||
buffer=self._multi_ctas_kv_counter_buffer,
|
||||
device=torch.device(self.device),
|
||||
num_q_heads=self.num_q_heads,
|
||||
num_q_heads=self.num_dcp_q_heads,
|
||||
batch_size=num_query_rows,
|
||||
)
|
||||
# Capacity is set before capture, so a grow here is a broken invariant.
|
||||
@@ -1543,7 +1555,7 @@ class DeepseekSparseAttnBackend(
|
||||
grow_multi_ctas_kv_counter_buffer_if_needed(
|
||||
buffer=self._multi_ctas_kv_counter_buffer,
|
||||
device=torch.device(self.device),
|
||||
num_q_heads=self.num_q_heads,
|
||||
num_q_heads=self.num_dcp_q_heads,
|
||||
batch_size=num_query_rows,
|
||||
)
|
||||
)
|
||||
@@ -3774,6 +3786,8 @@ class DeepseekSparseAttnBackend(
|
||||
else:
|
||||
q_all = q.view(-1, layer.tp_q_head_num, layer.head_dim)
|
||||
|
||||
shared_topk_indices = topk_indices
|
||||
|
||||
# Eager DP attention can pad q beyond metadata that was deliberately
|
||||
# planned on the real draft batch. Pad top-k to the physical q shape,
|
||||
# then run decode attention only on metadata-backed rows. The output is
|
||||
@@ -3815,6 +3829,16 @@ class DeepseekSparseAttnBackend(
|
||||
page_table_1, sparse_mla_top_k = self._pad_trtllm_sparse_page_table(
|
||||
page_table_1
|
||||
)
|
||||
if self.dcp_size > 1:
|
||||
page_table_1, seq_lens = localize_dcp_page_table(
|
||||
self._dcp_page_table_cache,
|
||||
metadata=metadata,
|
||||
layer_id=layer.layer_id,
|
||||
topk_indices=shared_topk_indices,
|
||||
page_table=page_table_1,
|
||||
dcp_size=self.dcp_size,
|
||||
dcp_rank=self.dcp_rank,
|
||||
)
|
||||
sparse_mla_top_k_lens = None
|
||||
if self.qk_rope_head_dim == 0:
|
||||
sparse_mla_top_k_lens = prepare_trtllm_nope_sparse_metadata(page_table_1)
|
||||
@@ -3827,8 +3851,7 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
bmm1_scale = q_scale * k_scale * layer.scaling
|
||||
|
||||
batch_size = page_table_1.shape[0]
|
||||
_, num_heads, head_dim = q_all.shape
|
||||
batch_size, num_heads, head_dim = q_all.shape
|
||||
|
||||
multi_ctas_kv_counter_buffer = self._multi_ctas_kv_counter_for(batch_size)
|
||||
|
||||
@@ -3837,15 +3860,12 @@ class DeepseekSparseAttnBackend(
|
||||
block_tables = page_table_1.unsqueeze(1)
|
||||
seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens
|
||||
|
||||
out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q,
|
||||
attention_kwargs = dict(
|
||||
kv_cache=kv,
|
||||
workspace_buffer=self.workspace_buffer,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=metadata.max_seq_len_k,
|
||||
sparse_mla_top_k=sparse_mla_top_k,
|
||||
bmm1_scale=bmm1_scale,
|
||||
@@ -3855,6 +3875,30 @@ class DeepseekSparseAttnBackend(
|
||||
multi_ctas_kv_counter_buffer=multi_ctas_kv_counter_buffer,
|
||||
)
|
||||
|
||||
if self.dcp_size > 1:
|
||||
out, lse = dcp_trtllm_sparse_attention(
|
||||
query=q,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens,
|
||||
cu_seqlens_q=self.get_device_int32_arange(batch_size + 1),
|
||||
**attention_kwargs,
|
||||
)
|
||||
if num_decode_padding_rows:
|
||||
lse = torch.cat(
|
||||
[
|
||||
lse,
|
||||
lse.new_full((num_decode_padding_rows, num_heads), -torch.inf),
|
||||
]
|
||||
)
|
||||
return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows), lse
|
||||
|
||||
out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens,
|
||||
**attention_kwargs,
|
||||
)
|
||||
|
||||
return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows)
|
||||
|
||||
def _pad_topk_indices(
|
||||
|
||||
@@ -35,10 +35,18 @@ class IndexKeyCache:
|
||||
pool = self.pool
|
||||
return (
|
||||
num_pages,
|
||||
pool.index_page_size
|
||||
pool.page_size
|
||||
// pool.index_kpool
|
||||
* (pool.index_head_dim + pool.index_head_dim // pool.quant_block_size * 4),
|
||||
)
|
||||
|
||||
def _kernel_view(self, buffer: torch.Tensor) -> torch.Tensor:
|
||||
pool = self.pool
|
||||
kernel_pages_per_row = (
|
||||
pool.page_size // pool.index_kpool // pool.index_page_size
|
||||
)
|
||||
return buffer.view(-1, buffer.shape[1] // kernel_pages_per_row)
|
||||
|
||||
def _layer_num_pages(self, layer_idx: int, num_pages: int) -> int:
|
||||
# Layers that reuse the previous layer's top-k never write index-K, so
|
||||
# they get a 0-row placeholder that keeps ``buffer`` layer-aligned.
|
||||
@@ -62,7 +70,7 @@ class IndexKeyCache:
|
||||
self.pool.layer_transfer_counter.wait_until(
|
||||
layer_id - self.pool.start_layer
|
||||
)
|
||||
return self.buffer[layer_id - self.pool.start_layer]
|
||||
return self._kernel_view(self.buffer[layer_id - self.pool.start_layer])
|
||||
|
||||
def get_buffer(self, layer_id: int) -> torch.Tensor:
|
||||
return self.get_local_buffer(layer_id)
|
||||
@@ -92,7 +100,7 @@ class IndexKeyCache:
|
||||
index_k: torch.Tensor,
|
||||
index_k_scale: torch.Tensor,
|
||||
) -> None:
|
||||
buf = self.buffer[layer_id - self.pool.start_layer]
|
||||
buf = self._kernel_view(self.buffer[layer_id - self.pool.start_layer])
|
||||
index_buf_accessor.SetKAndS.execute(
|
||||
pool=self.pool,
|
||||
buf=buf,
|
||||
|
||||
@@ -2042,6 +2042,8 @@ class KVCacheConfigurator:
|
||||
pool_kwargs["layer_shard_size"] = dsa_cp_layer_shard_size
|
||||
else:
|
||||
PoolCls = dsa_pool_class
|
||||
if self.loc_space_scale > 1:
|
||||
pool_kwargs["dcp_replicated"] = True
|
||||
if _should_elide_dsa_index_k(is_draft_worker=self.is_draft_worker):
|
||||
pool_kwargs["skip_topk_layers"] = [
|
||||
dsa_layer_skips_topk(self.model_config.hf_config, layer_id)
|
||||
|
||||
@@ -4684,6 +4684,7 @@ class MLATokenToKVPool(KVCache):
|
||||
end_layer: Optional[int] = None,
|
||||
use_dsa: bool = False,
|
||||
override_kv_cache_dim: Optional[int] = None,
|
||||
dcp_replicated: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
size,
|
||||
@@ -4699,6 +4700,7 @@ class MLATokenToKVPool(KVCache):
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.use_dsa = use_dsa
|
||||
self.dcp_replicated = dcp_replicated
|
||||
self.dsa_kv_cache_store_fp8 = (
|
||||
use_dsa
|
||||
and dtype == torch.float8_e4m3fn
|
||||
@@ -4786,11 +4788,14 @@ class MLATokenToKVPool(KVCache):
|
||||
# False: this pool takes a WIDENED loc. The unified pool resolves it in
|
||||
# the plan's write translation (`translate_write_loc`) and flips this.
|
||||
write_loc_is_dcp_resolved = False
|
||||
dcp_replicated = False
|
||||
|
||||
@property
|
||||
def _write_loc_dcp_span(self) -> int:
|
||||
"""How many logical ids one stored row spans in the write-loc space."""
|
||||
return 1 if self.write_loc_is_dcp_resolved else get_parallel().attn_dcp_size
|
||||
if self.write_loc_is_dcp_resolved or self.dcp_replicated:
|
||||
return 1
|
||||
return get_parallel().attn_dcp_size
|
||||
|
||||
def _scatter_mla_rows(
|
||||
self,
|
||||
@@ -4799,7 +4804,7 @@ class MLATokenToKVPool(KVCache):
|
||||
cache_k_nope: torch.Tensor,
|
||||
cache_k_rope: torch.Tensor,
|
||||
) -> None:
|
||||
if self.write_loc_is_dcp_resolved:
|
||||
if self.write_loc_is_dcp_resolved or self.dcp_replicated:
|
||||
set_mla_kv_buffer_triton(dst_buffer, loc, cache_k_nope, cache_k_rope)
|
||||
else:
|
||||
set_mla_kv_buffer_dcp_sharded_triton(
|
||||
@@ -4823,7 +4828,11 @@ class MLATokenToKVPool(KVCache):
|
||||
assert not self.dsa_kv_cache_store_fp8
|
||||
# No DCP-aware variant is possible: the two backends reaching this door
|
||||
# disagree on the loc space (flashinfer-MLA widened, Triton collapsed).
|
||||
assert self.write_loc_is_dcp_resolved or not get_parallel().dcp_enabled, (
|
||||
assert (
|
||||
self.write_loc_is_dcp_resolved
|
||||
or self.dcp_replicated
|
||||
or not get_parallel().dcp_enabled
|
||||
), (
|
||||
"MLATokenToKVPool.set_kv_buffer has no DCP-aware write path. Under "
|
||||
"--dcp-size > 1 the MLA write must go through set_mla_kv_buffer, "
|
||||
"whose kernel resolves the owner rule; reaching the combined-row "
|
||||
@@ -5164,6 +5173,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
tail_extra_slots: int = 0,
|
||||
max_running_requests: Optional[int] = None,
|
||||
skip_topk_layers: Optional[List[bool]] = None,
|
||||
dcp_replicated: bool = False,
|
||||
):
|
||||
override_dim = (
|
||||
kv_cache_dim if kv_cache_dim != kv_lora_rank + qk_rope_head_dim else None
|
||||
@@ -5182,6 +5192,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
end_layer,
|
||||
use_dsa=True,
|
||||
override_kv_cache_dim=override_dim,
|
||||
dcp_replicated=dcp_replicated,
|
||||
)
|
||||
# self.index_k_dtype = torch.float8_e4m3fn
|
||||
# self.index_k_scale_dtype = torch.float32
|
||||
@@ -5193,8 +5204,15 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
f"page_size {self.page_size} must be a multiple of index_kpool {index_kpool}"
|
||||
)
|
||||
self.index_page_size = self.page_size // index_kpool
|
||||
if dcp_replicated:
|
||||
self.index_page_size //= get_parallel().attn_dcp_size
|
||||
if index_buf_size is None:
|
||||
index_buf_size = size
|
||||
dcp_size = get_parallel().attn_dcp_size
|
||||
index_buf_size = (
|
||||
size
|
||||
if dcp_replicated
|
||||
else (size + self.page_size) * dcp_size - self.page_size
|
||||
)
|
||||
self.index_buf_size = index_buf_size
|
||||
# num head == 1 and head dim == 128 for index_k in DSA
|
||||
assert index_head_dim == 128
|
||||
@@ -5206,7 +5224,7 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
)
|
||||
assert len(self.skip_topk_layers) == layer_num
|
||||
|
||||
physical_page_size = self.page_size // index_kpool
|
||||
physical_page_size = self.index_page_size
|
||||
if _is_hip:
|
||||
if aiter_can_use_preshuffle_paged_mqa():
|
||||
assert physical_page_size % 16 == 0, (
|
||||
|
||||
@@ -270,8 +270,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
kvc=kvc,
|
||||
num_layers=num_layers,
|
||||
)
|
||||
if _is_npu and dcp_size > 1:
|
||||
target_indexer_size *= dcp_size
|
||||
target_indexer_size *= dcp_size
|
||||
target_kv_size = self._cell_size - target_indexer_size
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
get_glm_dsa_layer_split_effective_num_layers,
|
||||
@@ -288,11 +287,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
num_layers=draft_num_layers,
|
||||
allocate_all_layers=True,
|
||||
)
|
||||
# The draft pool is replicated and consumes the widened
|
||||
# allocator-global slot space on NPU DCP.
|
||||
if _is_npu and dcp_size > 1:
|
||||
draft_kv_size *= dcp_size
|
||||
draft_indexer_size *= dcp_size
|
||||
draft_kv_size *= dcp_size
|
||||
draft_indexer_size *= dcp_size
|
||||
self._cell_size += draft_kv_size + draft_indexer_size
|
||||
else:
|
||||
self._cell_size = int(
|
||||
@@ -388,7 +384,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
kvc=kvc,
|
||||
num_layers=num_layers,
|
||||
)
|
||||
if _is_npu and not kvc.is_draft_worker and dcp_size > 1:
|
||||
if not kvc.is_draft_worker:
|
||||
indexer_cell_size *= dcp_size
|
||||
cell_size += indexer_cell_size
|
||||
elif is_minimax_sparse(model_config.hf_config):
|
||||
|
||||
@@ -266,7 +266,7 @@ class DeepseekMHAForwardMixin:
|
||||
kv_a, k_pe = self._get_mla_kv_buffer_from_fp8_for_dsa(forward_batch)
|
||||
else:
|
||||
# BF16/FP16 path: directly fetch from cache
|
||||
if get_parallel().dcp_enabled:
|
||||
if self.dcp_size > 1:
|
||||
kv_a, k_pe = all_gather_kv_cache_for_mha_extend(
|
||||
get_token_to_kv_pool(),
|
||||
self.attn_mha,
|
||||
|
||||
+2
-2
@@ -34,7 +34,7 @@ from sglang.srt.models.deepseek_common.utils import (
|
||||
_use_aiter_bpreshuffle_gfx95,
|
||||
_use_aiter_gfx95,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import BumpAllocator, get_bool_env_var
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -232,7 +232,7 @@ class DeepseekMHARocmForwardMixin:
|
||||
kv_a, k_pe = self._get_mla_kv_buffer_from_fp8_for_dsa(forward_batch)
|
||||
else:
|
||||
# BF16/FP16 path: directly fetch from cache
|
||||
if get_parallel().dcp_enabled:
|
||||
if self.dcp_size > 1:
|
||||
kv_a, k_pe = all_gather_kv_cache_for_mha_extend(
|
||||
get_token_to_kv_pool(),
|
||||
self.attn_mha,
|
||||
|
||||
@@ -92,15 +92,6 @@ def _select_local_dcp_heads_for_autotune(
|
||||
return attn_output.narrow(1, rank * num_local_heads, num_local_heads)
|
||||
|
||||
|
||||
def is_dcp_mla_decode_phase(forward_batch: ForwardBatch) -> bool:
|
||||
if not get_parallel().dcp_enabled:
|
||||
return False
|
||||
return (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
)
|
||||
|
||||
|
||||
def is_mla_dcp_lse_base_on_e(attention_backend: Optional[str]) -> bool:
|
||||
return attention_backend in {"flashmla", "cutedsl_mla", "aiter"}
|
||||
|
||||
@@ -131,6 +122,26 @@ def _apply_attention_output_gate(module, attn_output, gate):
|
||||
|
||||
|
||||
class DeepseekMLAForwardMixin:
|
||||
def is_dcp_mla_decode_phase(
|
||||
self: DeepseekV2AttentionMLA, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
if self.dcp_size == 1:
|
||||
return False
|
||||
mode = forward_batch.forward_mode
|
||||
return (
|
||||
mode.is_decode()
|
||||
or mode.is_target_verify()
|
||||
or (self.use_dsa and mode.is_extend())
|
||||
)
|
||||
|
||||
def is_dcp_q_replicate_phase(
|
||||
self: DeepseekV2AttentionMLA, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
if not get_parallel().dcp_replicate_q_proj or self.dcp_size == 1:
|
||||
return False
|
||||
mode = forward_batch.forward_mode
|
||||
return mode.is_decode() or mode.is_target_verify()
|
||||
|
||||
def init_mla_forward(self: DeepseekV2AttentionMLA):
|
||||
self.flashinfer_mla_disable_ragged = (
|
||||
get_exec().kernel.flashinfer_mla_disable_ragged
|
||||
@@ -162,6 +173,8 @@ class DeepseekMLAForwardMixin:
|
||||
) -> bool:
|
||||
if getattr(self, "_kimi_split_gguf_kv_b", False):
|
||||
return False
|
||||
if self.dcp_size > 1:
|
||||
return False
|
||||
# Shared activation surface with the DSA indexer graph dispatch
|
||||
# (in piecewise/breakable graph + non-speculative extend). Like the indexer
|
||||
# dispatch, this fusion is on by default on that surface.
|
||||
@@ -264,7 +277,7 @@ class DeepseekMLAForwardMixin:
|
||||
return None
|
||||
if get_is_capture_mode():
|
||||
return None
|
||||
if get_parallel().dcp_enabled:
|
||||
if self.dcp_size > 1:
|
||||
return None
|
||||
# Kernel shape constraints (tl.arange / tl.dot / block tiling). K
|
||||
# (qk_nope_head_dim) needs only K % 16 == 0 and K <= 256: power-of-2
|
||||
@@ -314,8 +327,7 @@ class DeepseekMLAForwardMixin:
|
||||
# --dcp-replicate-q-proj: project full-head Q locally from pre-gathered
|
||||
# weights and skip the per-layer Q all-gather (bf16 decode absorb only).
|
||||
q_replicate_active = (
|
||||
get_parallel().dcp_replicate_q_proj
|
||||
and is_dcp_mla_decode_phase(forward_batch)
|
||||
self.is_dcp_q_replicate_phase(forward_batch)
|
||||
and not self.use_deep_gemm_bmm
|
||||
and self.w_kc_qrep is not None
|
||||
and self.q_b_proj_qrep_weight is not None
|
||||
@@ -391,7 +403,7 @@ class DeepseekMLAForwardMixin:
|
||||
# full-head Q from the gathered weight (skips Q all-gather)
|
||||
q = torch.nn.functional.linear(q, self.q_b_proj_qrep_weight).view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.qk_head_dim,
|
||||
)
|
||||
else:
|
||||
@@ -463,7 +475,7 @@ class DeepseekMLAForwardMixin:
|
||||
hidden_states, self.q_b_proj_qrep_weight
|
||||
).view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.qk_head_dim,
|
||||
)
|
||||
else:
|
||||
@@ -633,8 +645,8 @@ class DeepseekMLAForwardMixin:
|
||||
)
|
||||
|
||||
# all_gather q_pe, q_nope_out,take tp8 as an example, q_pe [B, H, ROPE_DIM], q_nope_out [B, H, NOPE_DIM] gathered to [B, H * dcp_world_size, ROPE_DIM] [B, H * dcp_world_size, NOPE_DIM] for decode batch, and all gather k_pe, k_nope for extend batch.
|
||||
if get_parallel().dcp_enabled:
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.dcp_size > 1:
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
if not q_replicate_active:
|
||||
q_nope_out, q_pe = all_gather_q_for_mla_decode(
|
||||
q_nope_out=q_nope_out,
|
||||
@@ -727,7 +739,7 @@ class DeepseekMLAForwardMixin:
|
||||
topk_indices=topk_indices,
|
||||
)
|
||||
attn_output = fusion_plan.attn_output_buf
|
||||
elif is_dcp_mla_decode_phase(forward_batch):
|
||||
elif self.is_dcp_mla_decode_phase(forward_batch):
|
||||
# set return_lse=True to correct attn_output
|
||||
attn_output, lse = self.attn_mqa_for_dcp_decode(
|
||||
q_nope_out,
|
||||
@@ -776,10 +788,10 @@ class DeepseekMLAForwardMixin:
|
||||
)
|
||||
|
||||
# correct attn_output with respect to lse from other ranks
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
attn_output = attn_output.view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.kv_lora_rank,
|
||||
)
|
||||
if get_in_autotune_dummy_run():
|
||||
|
||||
+11
-13
@@ -51,7 +51,6 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mla import (
|
||||
_select_local_dcp_heads_for_autotune,
|
||||
is_dcp_mla_decode_phase,
|
||||
is_mla_dcp_lse_base_on_e,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.utils import (
|
||||
@@ -441,7 +440,7 @@ def _can_fuse_bmm_rope_cat_and_cache(attn: DeepseekV2AttentionMLA) -> bool:
|
||||
"""
|
||||
return (
|
||||
_use_aiter_gfx95
|
||||
and not get_parallel().dcp_enabled
|
||||
and attn.dcp_size == 1
|
||||
and attn.w_kc.dtype == torch.float8_e4m3fn
|
||||
and attn.rotary_emb is not None
|
||||
and attn._skip_rope_for_dsa_tilelang_fused()
|
||||
@@ -503,8 +502,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
|
||||
q_replicate_active = (
|
||||
get_parallel().dcp_replicate_q_proj
|
||||
and is_dcp_mla_decode_phase(forward_batch)
|
||||
self.is_dcp_q_replicate_phase(forward_batch)
|
||||
and not self.use_deep_gemm_bmm
|
||||
and self.w_kc_qrep is not None
|
||||
and self.q_b_proj_qrep_weight is not None
|
||||
@@ -634,7 +632,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
if q_replicate_active:
|
||||
q = torch.nn.functional.linear(q, self.q_b_proj_qrep_weight).view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.qk_head_dim,
|
||||
)
|
||||
else:
|
||||
@@ -659,7 +657,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
hidden_states, self.q_b_proj_qrep_weight
|
||||
).view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.qk_head_dim,
|
||||
)
|
||||
else:
|
||||
@@ -740,7 +738,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
fuse_rope_for_trtllm_mla = self._fuse_rope_for_trtllm_mla(forward_batch)
|
||||
|
||||
force_rope_for_aiter_dcp_decode = (
|
||||
get_parallel().dcp_enabled
|
||||
self.dcp_size > 1
|
||||
and (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
@@ -771,8 +769,8 @@ class DeepseekMLARocmForwardMixin:
|
||||
q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
|
||||
|
||||
# all_gather q_pe, q_nope_out,take tp8 as an example, q_pe [B, H, ROPE_DIM], q_nope_out [B, H, NOPE_DIM] gathered to [B, H * dcp_world_size, ROPE_DIM] [B, H * dcp_world_size, NOPE_DIM] for decode batch, and all gather k_pe, k_nope for extend batch.
|
||||
if get_parallel().dcp_enabled:
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.dcp_size > 1:
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
if not q_replicate_active:
|
||||
q_nope_out, q_pe = all_gather_q_for_mla_decode(
|
||||
q_nope_out=q_nope_out,
|
||||
@@ -910,7 +908,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
"is_neox": self.rotary_emb.is_neox_style,
|
||||
"llama_4_scaling": llama_4_scaling,
|
||||
}
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
# set return_lse=True to correct attn_output
|
||||
attn_output, lse = self.attn_mqa_for_dcp_decode(
|
||||
q_nope_out,
|
||||
@@ -948,7 +946,7 @@ class DeepseekMLARocmForwardMixin:
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
or forward_batch.forward_mode.is_draft_extend_v2()
|
||||
)
|
||||
and get_parallel().dcp_enabled
|
||||
and self.dcp_size > 1
|
||||
):
|
||||
q = torch.cat([q_nope_out, q_pe], dim=-1)
|
||||
if llama_4_scaling is not None:
|
||||
@@ -1003,10 +1001,10 @@ class DeepseekMLARocmForwardMixin:
|
||||
)
|
||||
|
||||
# correct attn_output with respect to lse from other ranks
|
||||
if is_dcp_mla_decode_phase(forward_batch):
|
||||
if self.is_dcp_mla_decode_phase(forward_batch):
|
||||
attn_output = attn_output.view(
|
||||
-1,
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.kv_lora_rank,
|
||||
)
|
||||
if get_in_autotune_dummy_run():
|
||||
|
||||
@@ -2018,6 +2018,9 @@ class DeepseekV2AttentionMLA(
|
||||
self.is_nextn = is_nextn
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
self.use_dsa = is_deepseek_dsa(config)
|
||||
self.dcp_size = (
|
||||
1 if (is_nextn and self.use_dsa) else get_parallel().attn_dcp_size
|
||||
)
|
||||
self.num_heads = num_heads
|
||||
assert num_heads % attn_tp_size == 0
|
||||
self.num_local_heads = num_heads // attn_tp_size
|
||||
@@ -2156,9 +2159,9 @@ class DeepseekV2AttentionMLA(
|
||||
prefix=add_prefix("attn_mqa", prefix),
|
||||
)
|
||||
# use num_local_heads * dcp_world_size because q_nope, q_rope is all gathered from dcp ranks
|
||||
if get_parallel().dcp_enabled:
|
||||
if self.dcp_size > 1:
|
||||
self.attn_mqa_for_dcp_decode = RadixAttention(
|
||||
self.num_local_heads * get_parallel().attn_dcp_size,
|
||||
self.num_local_heads * self.dcp_size,
|
||||
self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=1,
|
||||
@@ -3441,7 +3444,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
kv_cache_device,
|
||||
create_chunked_prefix_cache_kv_indices_fn,
|
||||
):
|
||||
if _is_npu:
|
||||
if _is_npu or is_deepseek_dsa(self.config):
|
||||
return None
|
||||
return prepare_decode_context_parallel_metadata(
|
||||
seq_lens=seq_lens,
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import math
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton
|
||||
from sglang.kernels.ops.attention.dsa.transform_index import (
|
||||
transform_index_page_table_dcp,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.dsa_dcp import (
|
||||
dcp_trtllm_sparse_attention,
|
||||
trtllm_rows_per_call,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b", runner_config="4-gpu-b200")
|
||||
|
||||
TOPK = 2048
|
||||
SCALE = 1 / math.sqrt(192)
|
||||
|
||||
|
||||
def _reference_page_table(page_table, dcp_size, rank):
|
||||
rows, counts = [], []
|
||||
for row in page_table.tolist():
|
||||
owned = [
|
||||
slot // dcp_size for slot in row if slot >= 0 and slot % dcp_size == rank
|
||||
]
|
||||
rows.append(owned + [-1] * (len(row) - len(owned)))
|
||||
counts.append(len(owned))
|
||||
return torch.tensor(rows, dtype=page_table.dtype), torch.tensor(counts)
|
||||
|
||||
|
||||
def _sparse_attention(query, kv, page_table, seq_lens, workspace_bytes=512 << 20):
|
||||
rows = query.shape[0]
|
||||
return dcp_trtllm_sparse_attention(
|
||||
query=query,
|
||||
block_tables=page_table.unsqueeze(1),
|
||||
seq_lens=seq_lens,
|
||||
cu_seqlens_q=torch.arange(rows + 1, dtype=torch.int32, device="cuda"),
|
||||
kv_cache=kv.reshape(-1, 1, 64, 576),
|
||||
workspace_buffer=torch.zeros(workspace_bytes, dtype=torch.uint8, device="cuda"),
|
||||
qk_nope_head_dim=128,
|
||||
kv_lora_rank=512,
|
||||
qk_rope_head_dim=64,
|
||||
max_seq_len=TOPK,
|
||||
sparse_mla_top_k=TOPK,
|
||||
bmm1_scale=SCALE,
|
||||
backend="trtllm-gen",
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
|
||||
class TestDsaDcpAttention(CustomTestCase):
|
||||
def test_page_table_keeps_owned_slots_in_order(self):
|
||||
for width in (37, TOPK):
|
||||
page_table = torch.arange(3 * width, dtype=torch.int32).view(3, width)
|
||||
page_table = page_table.flip(1)
|
||||
page_table[0, ::3] = -1
|
||||
page_table[1] = -1
|
||||
for dcp_size in (2, 4):
|
||||
for rank in range(dcp_size):
|
||||
with self.subTest(width=width, dcp=dcp_size, rank=rank):
|
||||
expected, expected_counts = _reference_page_table(
|
||||
page_table, dcp_size, rank
|
||||
)
|
||||
actual, counts = transform_index_page_table_dcp(
|
||||
page_table.cuda(), dcp_size, rank
|
||||
)
|
||||
self.assertEqual(actual.cpu().tolist(), expected.tolist())
|
||||
self.assertEqual(
|
||||
counts.cpu().tolist(), expected_counts.tolist()
|
||||
)
|
||||
|
||||
def test_merged_shards_match_unsharded_attention(self):
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
capacity, heads = 8192, 64
|
||||
page_table = torch.full((3, TOPK), -1, dtype=torch.int32)
|
||||
page_table[0, 0] = capacity - 4
|
||||
page_table[1, :127] = (
|
||||
torch.randperm(capacity // 4, generator=generator)[:127] * 4
|
||||
)
|
||||
page_table[2] = torch.randperm(capacity, generator=generator)[:TOPK]
|
||||
page_table = page_table.cuda()
|
||||
seq_lens = torch.tensor([1, 127, TOPK], dtype=torch.int32, device="cuda")
|
||||
query_cpu = torch.randn(3, 1, heads, 576, generator=generator) * 0.25
|
||||
kv_cpu = torch.randn(capacity, 576, generator=generator) * 0.25
|
||||
|
||||
for dtype in (torch.bfloat16, torch.float8_e4m3fn):
|
||||
query, kv = query_cpu.cuda().to(dtype), kv_cpu.cuda().to(dtype)
|
||||
expected, expected_lse = _sparse_attention(query, kv, page_table, seq_lens)
|
||||
for dcp_size in (2, 4):
|
||||
with self.subTest(dtype=dtype, dcp_size=dcp_size):
|
||||
outputs, lses = [], []
|
||||
for rank in range(dcp_size):
|
||||
local_page_table, counts = transform_index_page_table_dcp(
|
||||
page_table, dcp_size, rank
|
||||
)
|
||||
output, lse = _sparse_attention(
|
||||
query,
|
||||
kv[rank::dcp_size].contiguous(),
|
||||
local_page_table,
|
||||
counts,
|
||||
)
|
||||
outputs.append(output.squeeze(1))
|
||||
lses.append(lse)
|
||||
merged, merged_lse = dcp_lse_combine_triton(
|
||||
torch.stack(outputs),
|
||||
torch.stack(lses),
|
||||
is_lse_base_on_e=False,
|
||||
return_lse=True,
|
||||
)
|
||||
atol = 0.008 if dtype == torch.float8_e4m3fn else 0.003
|
||||
torch.testing.assert_close(
|
||||
merged, expected.squeeze(1), atol=atol, rtol=0.02
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
merged_lse, expected_lse, atol=0.003, rtol=0.002
|
||||
)
|
||||
|
||||
def test_rows_split_to_fit_the_workspace(self):
|
||||
"""A workspace too small for one call's softmax slab must split the rows, not overflow."""
|
||||
generator = torch.Generator().manual_seed(1)
|
||||
rows, heads, capacity, length = 2400, 32, 4096, 128
|
||||
page_table = torch.full((rows, TOPK), -1, dtype=torch.int32)
|
||||
for row in range(rows):
|
||||
page_table[row, :length] = torch.randperm(capacity, generator=generator)[
|
||||
:length
|
||||
]
|
||||
page_table = page_table.cuda()
|
||||
seq_lens = torch.full((rows,), length, dtype=torch.int32, device="cuda")
|
||||
query = (torch.randn(rows, 1, heads, 576, generator=generator) * 0.25).cuda()
|
||||
kv = (torch.randn(capacity, 576, generator=generator) * 0.25).cuda()
|
||||
query, kv = query.to(torch.bfloat16), kv.to(torch.bfloat16)
|
||||
expected, expected_lse = _sparse_attention(query, kv, page_table, seq_lens)
|
||||
workspace_bytes = (128 << 20) + 64 * heads * 256 * 8
|
||||
self.assertEqual(
|
||||
trtllm_rows_per_call(workspace_bytes=workspace_bytes, heads=heads), 64
|
||||
)
|
||||
output, lse = _sparse_attention(
|
||||
query, kv, page_table, seq_lens, workspace_bytes=workspace_bytes
|
||||
)
|
||||
torch.testing.assert_close(output, expected, atol=0.003, rtol=0.02)
|
||||
torch.testing.assert_close(lse, expected_lse, atol=0.003, rtol=0.002)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,22 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.server_fixtures.dsa_mtp_fixture import (
|
||||
DsaMtpEvalConfigDefaults,
|
||||
DsaMtpServerBase,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=600, stage="extra-b", runner_config="4-gpu-b200")
|
||||
|
||||
|
||||
class TestGLM53NVFP4DCP2MTP(DsaMtpServerBase, DsaMtpEvalConfigDefaults, GSM8KMixin):
|
||||
model = "RadixArk/GLM-5.3-NVFP4"
|
||||
tp_size = 4
|
||||
mem_fraction_static = 0.8
|
||||
gsm8k_accuracy_thres = 0.88
|
||||
extra_server_args = ["--quantization", "modelopt_fp4", "--dcp-size", "2"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.mem_cache.pool_host.dsa import (
|
||||
make_dsa_indexer_pool_decl,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -19,6 +20,11 @@ register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
class TestDSAHostRelease(CustomTestCase):
|
||||
def setUp(self):
|
||||
cm = get_parallel().override(attn_dcp_size=1)
|
||||
cm.__enter__()
|
||||
self.addCleanup(cm.__exit__, None, None, None)
|
||||
|
||||
def test_destroy_releases_index_pages_for_registration_reuse(self):
|
||||
pool = DSATokenToKVPool(
|
||||
size=256,
|
||||
|
||||
@@ -185,6 +185,16 @@ class TestLSECombineEdgeCases(CustomTestCase):
|
||||
raise unittest.SkipTest("CUDA required")
|
||||
cls.device = "cuda"
|
||||
|
||||
def test_all_empty_rows_stay_zero(self):
|
||||
from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton
|
||||
|
||||
outputs = torch.ones(2, 2, 4, 64, device=self.device, dtype=torch.bfloat16)
|
||||
lses = torch.zeros(2, 2, 4, device=self.device)
|
||||
lses[:, 1] = -torch.inf
|
||||
combined, _ = dcp_lse_combine_triton(outputs, lses, is_lse_base_on_e=False)
|
||||
self.assertTrue((combined[0] == 1).all())
|
||||
self.assertTrue((combined[1] == 0).all())
|
||||
|
||||
def test_one_shard_dominant(self):
|
||||
"""One shard has much larger LSE -- output should be close to that shard."""
|
||||
from sglang.kernels.ops.attention.dcp_kernels import (
|
||||
|
||||
@@ -36,6 +36,7 @@ def _make_backend(*, allocate_counter: bool = True):
|
||||
backend = object.__new__(DeepseekSparseAttnBackend)
|
||||
backend.device = "cuda"
|
||||
backend.num_q_heads = _NUM_Q_HEADS
|
||||
backend.num_dcp_q_heads = _NUM_Q_HEADS
|
||||
backend.physical_page_size = 64
|
||||
backend.hisparse_coordinator = None
|
||||
backend.speculative_num_draft_tokens = _NUM_DRAFT_TOKENS
|
||||
|
||||
@@ -105,6 +105,8 @@ class TestDSABackendDPPadding(unittest.TestCase):
|
||||
_multi_ctas_kv_counter_buffer=None,
|
||||
device="cpu",
|
||||
num_q_heads=2,
|
||||
num_dcp_q_heads=2,
|
||||
dcp_size=1,
|
||||
)
|
||||
backend._pad_topk_indices = MethodType(
|
||||
DeepseekSparseAttnBackend._pad_topk_indices, backend
|
||||
|
||||
@@ -13,6 +13,7 @@ from sglang.srt.mem_cache.pool_host.dsa import (
|
||||
make_dsa_indexer_pool_decl,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
@@ -36,6 +37,9 @@ class TestDSAHiCacheTransfer(unittest.TestCase):
|
||||
self.skipTest("DSA host transfer tests only support CUDA/ROCm.")
|
||||
if not (is_cuda() or is_hip()):
|
||||
self.skipTest("CUDA/ROCm not available.")
|
||||
cm = get_parallel().override(attn_dcp_size=1)
|
||||
cm.__enter__()
|
||||
self.addCleanup(cm.__exit__, None, None, None)
|
||||
|
||||
@staticmethod
|
||||
def _token_indices_for_pages(pages: torch.Tensor, page_size: int, device: str):
|
||||
|
||||
@@ -33,6 +33,7 @@ def _attn(**overrides):
|
||||
rotary_emb=object(),
|
||||
current_attention_backend="dsa",
|
||||
use_deep_gemm_bmm=False,
|
||||
dcp_size=1,
|
||||
kv_b_proj=SimpleNamespace(set_lora=False),
|
||||
_skip_rope_for_dsa_tilelang_fused=lambda: True,
|
||||
)
|
||||
@@ -41,15 +42,6 @@ def _attn(**overrides):
|
||||
return attn
|
||||
|
||||
|
||||
def _patch_dcp(case, *, dcp_enabled: bool):
|
||||
"""Give the gate a parallel context; the real one needs an initialized runtime."""
|
||||
parallel = SimpleNamespace(dcp_enabled=dcp_enabled)
|
||||
saved = forward_mla_rocm.get_parallel
|
||||
forward_mla_rocm.get_parallel = lambda: parallel
|
||||
case.addCleanup(setattr, forward_mla_rocm, "get_parallel", saved)
|
||||
return parallel
|
||||
|
||||
|
||||
class TestFusedAbsorbGate(CustomTestCase):
|
||||
def setUp(self):
|
||||
# The first term is the platform, and it short-circuits everything else;
|
||||
@@ -57,7 +49,6 @@ class TestFusedAbsorbGate(CustomTestCase):
|
||||
self._saved = forward_mla_rocm._use_aiter_gfx95
|
||||
forward_mla_rocm._use_aiter_gfx95 = True
|
||||
self.addCleanup(setattr, forward_mla_rocm, "_use_aiter_gfx95", self._saved)
|
||||
self._parallel = _patch_dcp(self, dcp_enabled=False)
|
||||
|
||||
def test_dcp_turns_it_off(self):
|
||||
"""DCP decode all-gathers q_nope_out, which this path never produces.
|
||||
@@ -65,8 +56,7 @@ class TestFusedAbsorbGate(CustomTestCase):
|
||||
Without this term the fused branch sets q_nope_out to None and
|
||||
all_gather_q_for_mla_decode dereferences it.
|
||||
"""
|
||||
self._parallel.dcp_enabled = True
|
||||
self.assertFalse(_CAN_FUSE(_attn()))
|
||||
self.assertFalse(_CAN_FUSE(_attn(dcp_size=2)))
|
||||
|
||||
def test_takes_the_fused_path_when_every_term_holds(self):
|
||||
self.assertTrue(_CAN_FUSE(_attn()))
|
||||
|
||||
Reference in New Issue
Block a user