diff --git a/docs/docs/advanced_features/dcp.mdx b/docs/docs/advanced_features/dcp.mdx index 57e35a2d87..50707403fa 100644 --- a/docs/docs/advanced_features/dcp.mdx +++ b/docs/docs/advanced_features/dcp.mdx @@ -172,6 +172,29 @@ Before H2D or D2H, both index lists keep positions owned by this rank and map `i The current scope is MLA L1/L2. L3, LMCache, HiSparse, non-MLA KV host pools, speculative decoding, and PD decode are not supported in this combination. +### DCP × DSA (GLM-5.2, GLM-5.3) + +DeepSeek Sparse Attention attends over a top-k slice of the context, so DCP keeps the target's latent KV striped by position while the **index-K cache stays replicated** on every rank: each rank scores the whole sequence, takes the global top-k, and attends over the selected slots it owns. Decode, target verify and extend all follow the same path, so the prefix KV is never gathered. + +1. The index-K cache is addressed by the widened virtual slot and costs `c` times its usual share of each rank's KV budget (about 6% at DCP2 and 17% at DCP4 on GLM-5.3). +2. Each layer maps the global top-k to rank-local rows with one Triton kernel; layers that share the previous layer's top-k reuse the mapping. +3. The TRT-LLM sparse kernel returns the per-head LSE and the usual DCP merge combines the partial outputs. +4. The MTP draft (nextn) layer keeps a replicated KV cache over the widened slots and attends without the DCP group, so MTP index share feeds the draft directly. + +Requirements: CUDA SM100/SM103 with the `trtllm` DSA backends (selected automatically), an `fp8_e4m3` or `bf16` KV cache, `attn_tp_size % dcp_size == 0`, and, with speculative decoding, the checkpoint's own MTP draft as a single EAGLE chain (`--speculative-eagle-topk 1`, draft tokens equal to steps + 1). HiSparse, prefill CP, HiCache, PD disaggregation, unified memory and `index_kpool > 1` are not supported in this combination. + +```bash Command +sglang serve \ + --model-path zai-org/GLM-5.3 \ + --trust-remote-code \ + --tp-size 8 \ + --dcp-size 2 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 5 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 6 +``` + ## References - [DCP and Helix Parallelism roadmap](https://github.com/sgl-project/sglang/issues/29736) diff --git a/python/sglang/kernels/ops/attention/__init__.py b/python/sglang/kernels/ops/attention/__init__.py index 3e8b66f5f2..588d290ce2 100644 --- a/python/sglang/kernels/ops/attention/__init__.py +++ b/python/sglang/kernels/ops/attention/__init__.py @@ -181,6 +181,7 @@ for _mod, _fn in [ ("dsa.transform_index", "transform_index_page_table_prefill"), ("dsa.transform_index", "transform_index_page_table_decode"), ("dsa.transform_index", "prepare_trtllm_nope_sparse_metadata"), + ("dsa.transform_index", "transform_index_page_table_dcp"), ("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"), ("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"), ("dsv4.fp4_indexer", "store_fp4_index_k_cache"), diff --git a/python/sglang/kernels/ops/attention/dcp_kernels.py b/python/sglang/kernels/ops/attention/dcp_kernels.py index 82e23406db..62f0a5b269 100644 --- a/python/sglang/kernels/ops/attention/dcp_kernels.py +++ b/python/sglang/kernels/ops/attention/dcp_kernels.py @@ -605,7 +605,7 @@ def _dcp_lse_combine_kernel( partial_out = tl.load(recv_output_ptr + o_offsets).to(tl.float32) acc += partial_out * w - acc = acc / weight_sum + acc = acc / tl.maximum(weight_sum, 1.0) out_offsets = ( batch_idx * out_stride_B + head_idx * out_stride_H + d_offsets * out_stride_D @@ -710,7 +710,7 @@ def _lse_weighted_combine_cpu( weights = torch.pow(2.0, centered) weight_sum = weights.sum(dim=0, keepdim=True) - weights = weights / weight_sum + weights = weights / weight_sum.clamp_min(1.0) combined = (partial_outputs * weights.unsqueeze(-1)).sum(dim=0) return combined diff --git a/python/sglang/kernels/ops/attention/dsa/transform_index.py b/python/sglang/kernels/ops/attention/dsa/transform_index.py index 8c2d0c2f5a..0f9746109c 100644 --- a/python/sglang/kernels/ops/attention/dsa/transform_index.py +++ b/python/sglang/kernels/ops/attention/dsa/transform_index.py @@ -14,6 +14,55 @@ def transform_index_page_table_decode(**kwargs): return transform_index_page_table_decode_fast(**kwargs) +@triton.jit +def transform_index_page_table_dcp_kernel( + page_table_ptr, + result_ptr, + counts_ptr, + row_stride: tl.constexpr, + DCP_SIZE: tl.constexpr, + DCP_RANK: tl.constexpr, + TOPK: tl.constexpr, + BLOCK_TOPK: tl.constexpr, +): + row = tl.program_id(0) + offsets = tl.arange(0, BLOCK_TOPK) + slots = tl.load( + page_table_ptr + row * row_stride + offsets, mask=offsets < TOPK, other=-1 + ) + owned = (slots >= 0) & (slots % DCP_SIZE == DCP_RANK) + packed_offsets = tl.cumsum(owned.to(tl.int32), axis=0) - 1 + count = tl.sum(owned.to(tl.int32), axis=0) + tl.store(result_ptr + row * TOPK + packed_offsets, slots // DCP_SIZE, mask=owned) + tl.store(result_ptr + row * TOPK + count + offsets, -1, mask=offsets < TOPK - count) + tl.store(counts_ptr + row, count) + + +def transform_index_page_table_dcp( + page_table: torch.Tensor, + dcp_size: int, + dcp_rank: int, +) -> tuple[torch.Tensor, torch.Tensor]: + num_rows, topk = page_table.shape + result = torch.empty_like(page_table) + counts = torch.empty(num_rows, dtype=torch.int32, device=page_table.device) + if num_rows == 0 or topk == 0: + return result, counts.zero_() + assert page_table.stride(1) == 1 + transform_index_page_table_dcp_kernel[(num_rows,)]( + page_table, + result, + counts, + page_table.stride(0), + DCP_SIZE=dcp_size, + DCP_RANK=dcp_rank, + TOPK=topk, + BLOCK_TOPK=triton.next_power_of_2(topk), + num_warps=4, + ) + return result, counts + + @triton.jit def prepare_trtllm_nope_sparse_metadata_kernel( page_table_ptr: torch.Tensor, diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py index 881ff45493..a620c1eeba 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py @@ -15,6 +15,25 @@ from sglang.srt.runtime_context import get_platform logger = logging.getLogger(__name__) +def _dsa_dcp_overrides(cfg: Any) -> dict: + if cfg.dcp_size <= 1: + return {} + if cfg.enable_hisparse or cfg.enable_hierarchical_cache: + raise ValueError("DSA DCP does not support HiSparse or HiCache.") + if cfg.disaggregation_mode != "null": + raise ValueError("DSA DCP does not support PD disaggregation.") + if cfg.speculative_eagle_topk not in (None, 1): + raise ValueError("DSA DCP requires --speculative-eagle-topk 1.") + overrides = { + name: "trtllm" + for name in ("dsa_prefill_backend", "dsa_decode_backend") + if getattr(cfg, name) is None + } + if cfg.dcp_replicate_q_proj is None and cfg.dcp_comm_backend in ("a2a", "fi_a2a"): + overrides["dcp_replicate_q_proj"] = True + return overrides + + @_register_for( "DeepseekV3ForCausalLM", "DeepseekV32ForCausalLM", @@ -78,6 +97,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: ) if is_deepseek_dsa(hf_config): # DeepSeek 3.2/GLM 5 + overrides.update(_dsa_dcp_overrides(cfg)) # Set attention backend for DeepSeek if is_attention_backend_not_set(cfg): overrides["attention_backend"] = "dsa" diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index c081a28ba6..fdbc3e9add 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -240,6 +240,8 @@ def handle_decode_context_parallelism(server_args: Any): "--decode-context-parallel-size) must be >= 1, but got " f"dcp_size={cfg.dcp_size}." ) + if cfg.dcp_size > 1 and cfg.enable_prefill_cp: + raise ValueError("--dcp-size > 1 does not support --enable-prefill-cp.") if cfg.dcp_comm_backend in ("a2a", "fi_a2a") and cfg.dcp_size <= 1: raise ValueError( f"--dcp-comm-backend {cfg.dcp_comm_backend} only affects the " diff --git a/python/sglang/srt/layers/attention/dsa/dsa_dcp.py b/python/sglang/srt/layers/attention/dsa/dsa_dcp.py new file mode 100644 index 0000000000..b5fc67532e --- /dev/null +++ b/python/sglang/srt/layers/attention/dsa/dsa_dcp.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from typing import Optional + +import msgspec +import torch + +from sglang.kernels.ops.attention.dsa.transform_index import ( + transform_index_page_table_dcp, +) +from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows + +TRTLLM_MAX_BATCH_ROWS = 65535 +# FlashInfer carves heads * rows * 256 float2 softmax slots out of the workspace per call; +# the rest is kernel scratch. +_SOFTMAX_BYTES_PER_ROW_HEAD = 256 * 8 +_SCRATCH_RESERVE_BYTES = 128 << 20 + + +def trtllm_rows_per_call(*, workspace_bytes: int, heads: int) -> int: + rows = (workspace_bytes - _SCRATCH_RESERVE_BYTES) // ( + heads * _SOFTMAX_BYTES_PER_ROW_HEAD + ) + return max(1, min(TRTLLM_MAX_BATCH_ROWS, rows)) + + +class DcpPageTableCache(msgspec.Struct): + """NOTE(vincentzed): GLM-5.2/5.3 reuse the previous layer's top-k tensor in + 57 of 78 layers; remapping once per fresh top-k cuts 78 launches/step to 21. + """ + + metadata: object = None + layer_id: int = -1 + topk_indices: Optional[torch.Tensor] = None + page_table: Optional[torch.Tensor] = None + counts: Optional[torch.Tensor] = None + + +def localize_dcp_page_table( + cache: DcpPageTableCache, + *, + metadata: object, + layer_id: int, + topk_indices: Optional[torch.Tensor], + page_table: torch.Tensor, + dcp_size: int, + dcp_rank: int, +) -> tuple[torch.Tensor, torch.Tensor]: + reuse = ( + topk_indices is not None + and cache.metadata is metadata + and cache.layer_id == layer_id - 1 + and cache.topk_indices is topk_indices + ) + if not reuse: + cache.metadata = metadata + cache.topk_indices = topk_indices + cache.page_table, cache.counts = transform_index_page_table_dcp( + page_table, dcp_size, dcp_rank + ) + cache.layer_id = layer_id + return cache.page_table, cache.counts + + +def dcp_trtllm_sparse_attention( + *, + query: torch.Tensor, + block_tables: torch.Tensor, + seq_lens: torch.Tensor, + cu_seqlens_q: torch.Tensor, + **attention_kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + import flashinfer.decode + + rows, _, heads, _ = query.shape + out_dtype = torch.bfloat16 + if rows == 0: + return ( + torch.empty( + (0, 1, heads, attention_kwargs["kv_lora_rank"]), + dtype=out_dtype, + device=query.device, + ), + torch.empty((0, heads), dtype=torch.float32, device=query.device), + ) + lse = torch.empty((rows, heads), dtype=torch.float32, device=query.device) + workspace = attention_kwargs["workspace_buffer"] + step = trtllm_rows_per_call( + workspace_bytes=workspace.numel() * workspace.element_size(), heads=heads + ) + if rows > step: + attention_kwargs["enable_pdl"] = False + top_k_lens = attention_kwargs.pop("sparse_mla_top_k_lens", None) + outputs = [] + for start in range(0, rows, step): + end = min(start + step, rows) + output, _ = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla( + query=query[start:end], + block_tables=block_tables[start:end], + seq_lens=seq_lens[start:end], + return_lse=True, + lse=lse[start:end], + sparse_mla_top_k_lens=( + None if top_k_lens is None else top_k_lens[start:end] + ), + **attention_kwargs, + ) + output = output.view(end - start, heads, -1) + fixup_zero_kv_rows( + output, + lse[start:end], + seq_lens[start:end], + cu_seqlens_q[: end - start + 1], + 1, + ) + outputs.append(output) + output = outputs[0] if len(outputs) == 1 else torch.cat(outputs) + return output.view(rows, 1, heads, -1), lse diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py index 303372481d..c5a83aa73c 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py @@ -638,7 +638,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): pool.invalidate_index_buffer_for_layer(layer_id) kv_cache = ( pool.get_index_k_with_scale_buffer(layer_id=layer_id) - .view(-1, pool.page_size, self.head_dim + 4) + .view(-1, pool.index_page_size, self.head_dim + 4) .view(fp8_dtype) ) @@ -739,8 +739,8 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): return None pool = get_token_to_kv_pool() - if pool.page_size != PAGE_SIZE: - _decline_fused(f"pool page_size is {pool.page_size}, not {PAGE_SIZE}") + if pool.index_page_size != PAGE_SIZE: + _decline_fused(f"pool page_size is {pool.index_page_size}, not {PAGE_SIZE}") return None # DSA cache layer split: the standard path stores index-K per layer, and the # fused path addresses the same buffer directly. @@ -1000,7 +1000,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): if out_cache_loc is None: out_cache_loc = forward_batch.out_cache_loc pool = get_token_to_kv_pool() - page_size = pool.page_size + page_size = pool.index_page_size if hasattr(pool, "invalidate_index_buffer_for_layer"): pool.invalidate_index_buffer_for_layer(layer_id) if hasattr(pool, "_is_layer_owned") and not pool._is_layer_owned(layer_id): @@ -1199,7 +1199,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): if TYPE_CHECKING: assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool) - page_size = get_token_to_kv_pool().page_size + page_size = get_token_to_kv_pool().index_page_size # NOTE(dark): blocksize = 64 is hardcoded in deep_gemm if _is_hip: if _use_aiter_preshuffle: @@ -1463,7 +1463,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): assert forward_batch.forward_mode.is_extend_without_speculative() - page_size = get_token_to_kv_pool().page_size + page_size = get_token_to_kv_pool().index_page_size if _is_hip: if _use_aiter_preshuffle: assert page_size % 16 == 0, ( @@ -1808,7 +1808,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): and can_use_dsa_fused_store( key.dtype, out_cache_loc.dtype, - pool.page_size, + pool.index_page_size, ) ): # NOTE: wrapper already normalizes shape/contiguity and asserts dtypes. @@ -1817,7 +1817,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): key, buf, out_cache_loc, - pool.page_size, + pool.index_page_size, ) return @@ -1827,7 +1827,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): # layout with page_size=1; the same kv_cache.view works for both cases # because page_size is 1 there. if _use_aiter: - page_size = pool.page_size + page_size = pool.index_page_size buf = pool.get_index_k_with_scale_buffer(layer_id=layer_id) kv_cache = buf.view(-1, page_size, 132).view(fp8_dtype) out_loc = forward_batch.out_cache_loc diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index de11bd31f0..d81753d910 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -60,6 +60,11 @@ from sglang.srt.layers.attention.dsa.dsa_backend_mtp_precompute import ( PrecomputedMetadata, compute_cu_seqlens, ) +from sglang.srt.layers.attention.dsa.dsa_dcp import ( + DcpPageTableCache, + dcp_trtllm_sparse_attention, + localize_dcp_page_table, +) from sglang.srt.layers.attention.dsa.dsa_indexer_metadata import DSAIndexerMetadata from sglang.srt.layers.attention.dsa.dsa_metadata_manager import ( DSAMetadataManagementMixin, @@ -529,6 +534,13 @@ class DeepseekSparseAttnBackend( self.num_q_heads = ( model_runner.model_config.num_attention_heads // get_parallel().attn_tp_size ) + if model_runner.is_draft_worker: + self.dcp_size, self.dcp_rank = 1, 0 + else: + self.dcp_size = get_parallel().attn_dcp_size + self.dcp_rank = get_parallel().attn_dcp_rank + self.num_dcp_q_heads = self.num_q_heads * self.dcp_size + self._dcp_page_table_cache = DcpPageTableCache() self.kv_cache_dim = model_runner.token_to_kv_pool.kv_cache_dim self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim self.kv_lora_rank = model_runner.model_config.kv_lora_rank @@ -543,7 +555,7 @@ class DeepseekSparseAttnBackend( self.req_to_token = model_runner.req_to_token_pool.req_to_token self.use_mha: bool = False - self.supports_mha_one_shot: bool = True + self.supports_mha_one_shot: bool = self.dcp_size == 1 self.dsa_prefill_impl: _DSA_IMPL_T = get_exec().kernel.dsa_prefill_backend self.dsa_decode_impl: _DSA_IMPL_T = get_exec().kernel.dsa_decode_backend if ( @@ -742,7 +754,7 @@ class DeepseekSparseAttnBackend( self.workspace_buffer = get_buffer( "dsa_trtllm_workspace", lambda: torch.empty( - envs.SGLANG_FLASHINFER_WORKSPACE_SIZE.get(), + envs.SGLANG_FLASHINFER_WORKSPACE_SIZE.get() * self.dcp_size, dtype=torch.uint8, device=model_runner.device, ), @@ -750,7 +762,7 @@ class DeepseekSparseAttnBackend( self._multi_ctas_kv_counter_buffer = ( make_persistent_multi_ctas_kv_counter_buffer( torch.device(self.device), - self.num_q_heads, + self.num_dcp_q_heads, max_batch_size=model_runner.max_running_requests, ) ) @@ -1525,7 +1537,7 @@ class DeepseekSparseAttnBackend( counter = grow_multi_ctas_kv_counter_buffer_if_needed( buffer=self._multi_ctas_kv_counter_buffer, device=torch.device(self.device), - num_q_heads=self.num_q_heads, + num_q_heads=self.num_dcp_q_heads, batch_size=num_query_rows, ) # Capacity is set before capture, so a grow here is a broken invariant. @@ -1543,7 +1555,7 @@ class DeepseekSparseAttnBackend( grow_multi_ctas_kv_counter_buffer_if_needed( buffer=self._multi_ctas_kv_counter_buffer, device=torch.device(self.device), - num_q_heads=self.num_q_heads, + num_q_heads=self.num_dcp_q_heads, batch_size=num_query_rows, ) ) @@ -3774,6 +3786,8 @@ class DeepseekSparseAttnBackend( else: q_all = q.view(-1, layer.tp_q_head_num, layer.head_dim) + shared_topk_indices = topk_indices + # Eager DP attention can pad q beyond metadata that was deliberately # planned on the real draft batch. Pad top-k to the physical q shape, # then run decode attention only on metadata-backed rows. The output is @@ -3815,6 +3829,16 @@ class DeepseekSparseAttnBackend( page_table_1, sparse_mla_top_k = self._pad_trtllm_sparse_page_table( page_table_1 ) + if self.dcp_size > 1: + page_table_1, seq_lens = localize_dcp_page_table( + self._dcp_page_table_cache, + metadata=metadata, + layer_id=layer.layer_id, + topk_indices=shared_topk_indices, + page_table=page_table_1, + dcp_size=self.dcp_size, + dcp_rank=self.dcp_rank, + ) sparse_mla_top_k_lens = None if self.qk_rope_head_dim == 0: sparse_mla_top_k_lens = prepare_trtllm_nope_sparse_metadata(page_table_1) @@ -3827,8 +3851,7 @@ class DeepseekSparseAttnBackend( ) bmm1_scale = q_scale * k_scale * layer.scaling - batch_size = page_table_1.shape[0] - _, num_heads, head_dim = q_all.shape + batch_size, num_heads, head_dim = q_all.shape multi_ctas_kv_counter_buffer = self._multi_ctas_kv_counter_for(batch_size) @@ -3837,15 +3860,12 @@ class DeepseekSparseAttnBackend( block_tables = page_table_1.unsqueeze(1) seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens - out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla( - query=q, + attention_kwargs = dict( kv_cache=kv, workspace_buffer=self.workspace_buffer, qk_nope_head_dim=self.qk_nope_head_dim, kv_lora_rank=self.kv_lora_rank, qk_rope_head_dim=self.qk_rope_head_dim, - block_tables=block_tables, - seq_lens=seq_lens, max_seq_len=metadata.max_seq_len_k, sparse_mla_top_k=sparse_mla_top_k, bmm1_scale=bmm1_scale, @@ -3855,6 +3875,30 @@ class DeepseekSparseAttnBackend( multi_ctas_kv_counter_buffer=multi_ctas_kv_counter_buffer, ) + if self.dcp_size > 1: + out, lse = dcp_trtllm_sparse_attention( + query=q, + block_tables=block_tables, + seq_lens=seq_lens, + cu_seqlens_q=self.get_device_int32_arange(batch_size + 1), + **attention_kwargs, + ) + if num_decode_padding_rows: + lse = torch.cat( + [ + lse, + lse.new_full((num_decode_padding_rows, num_heads), -torch.inf), + ] + ) + return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows), lse + + out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla( + query=q, + block_tables=block_tables, + seq_lens=seq_lens, + **attention_kwargs, + ) + return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows) def _pad_topk_indices( diff --git a/python/sglang/srt/mem_cache/index_key_cache.py b/python/sglang/srt/mem_cache/index_key_cache.py index 2b74bbcfcb..6a45b6f6bd 100644 --- a/python/sglang/srt/mem_cache/index_key_cache.py +++ b/python/sglang/srt/mem_cache/index_key_cache.py @@ -35,10 +35,18 @@ class IndexKeyCache: pool = self.pool return ( num_pages, - pool.index_page_size + pool.page_size + // pool.index_kpool * (pool.index_head_dim + pool.index_head_dim // pool.quant_block_size * 4), ) + def _kernel_view(self, buffer: torch.Tensor) -> torch.Tensor: + pool = self.pool + kernel_pages_per_row = ( + pool.page_size // pool.index_kpool // pool.index_page_size + ) + return buffer.view(-1, buffer.shape[1] // kernel_pages_per_row) + def _layer_num_pages(self, layer_idx: int, num_pages: int) -> int: # Layers that reuse the previous layer's top-k never write index-K, so # they get a 0-row placeholder that keeps ``buffer`` layer-aligned. @@ -62,7 +70,7 @@ class IndexKeyCache: self.pool.layer_transfer_counter.wait_until( layer_id - self.pool.start_layer ) - return self.buffer[layer_id - self.pool.start_layer] + return self._kernel_view(self.buffer[layer_id - self.pool.start_layer]) def get_buffer(self, layer_id: int) -> torch.Tensor: return self.get_local_buffer(layer_id) @@ -92,7 +100,7 @@ class IndexKeyCache: index_k: torch.Tensor, index_k_scale: torch.Tensor, ) -> None: - buf = self.buffer[layer_id - self.pool.start_layer] + buf = self._kernel_view(self.buffer[layer_id - self.pool.start_layer]) index_buf_accessor.SetKAndS.execute( pool=self.pool, buf=buf, diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 6f556b272f..6fe104b842 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -2042,6 +2042,8 @@ class KVCacheConfigurator: pool_kwargs["layer_shard_size"] = dsa_cp_layer_shard_size else: PoolCls = dsa_pool_class + if self.loc_space_scale > 1: + pool_kwargs["dcp_replicated"] = True if _should_elide_dsa_index_k(is_draft_worker=self.is_draft_worker): pool_kwargs["skip_topk_layers"] = [ dsa_layer_skips_topk(self.model_config.hf_config, layer_id) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index bbec4bc42a..5cc72238dc 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -4684,6 +4684,7 @@ class MLATokenToKVPool(KVCache): end_layer: Optional[int] = None, use_dsa: bool = False, override_kv_cache_dim: Optional[int] = None, + dcp_replicated: bool = False, ): super().__init__( size, @@ -4699,6 +4700,7 @@ class MLATokenToKVPool(KVCache): self.kv_lora_rank = kv_lora_rank self.qk_rope_head_dim = qk_rope_head_dim self.use_dsa = use_dsa + self.dcp_replicated = dcp_replicated self.dsa_kv_cache_store_fp8 = ( use_dsa and dtype == torch.float8_e4m3fn @@ -4786,11 +4788,14 @@ class MLATokenToKVPool(KVCache): # False: this pool takes a WIDENED loc. The unified pool resolves it in # the plan's write translation (`translate_write_loc`) and flips this. write_loc_is_dcp_resolved = False + dcp_replicated = False @property def _write_loc_dcp_span(self) -> int: """How many logical ids one stored row spans in the write-loc space.""" - return 1 if self.write_loc_is_dcp_resolved else get_parallel().attn_dcp_size + if self.write_loc_is_dcp_resolved or self.dcp_replicated: + return 1 + return get_parallel().attn_dcp_size def _scatter_mla_rows( self, @@ -4799,7 +4804,7 @@ class MLATokenToKVPool(KVCache): cache_k_nope: torch.Tensor, cache_k_rope: torch.Tensor, ) -> None: - if self.write_loc_is_dcp_resolved: + if self.write_loc_is_dcp_resolved or self.dcp_replicated: set_mla_kv_buffer_triton(dst_buffer, loc, cache_k_nope, cache_k_rope) else: set_mla_kv_buffer_dcp_sharded_triton( @@ -4823,7 +4828,11 @@ class MLATokenToKVPool(KVCache): assert not self.dsa_kv_cache_store_fp8 # No DCP-aware variant is possible: the two backends reaching this door # disagree on the loc space (flashinfer-MLA widened, Triton collapsed). - assert self.write_loc_is_dcp_resolved or not get_parallel().dcp_enabled, ( + assert ( + self.write_loc_is_dcp_resolved + or self.dcp_replicated + or not get_parallel().dcp_enabled + ), ( "MLATokenToKVPool.set_kv_buffer has no DCP-aware write path. Under " "--dcp-size > 1 the MLA write must go through set_mla_kv_buffer, " "whose kernel resolves the owner rule; reaching the combined-row " @@ -5164,6 +5173,7 @@ class DSATokenToKVPool(MLATokenToKVPool): tail_extra_slots: int = 0, max_running_requests: Optional[int] = None, skip_topk_layers: Optional[List[bool]] = None, + dcp_replicated: bool = False, ): override_dim = ( kv_cache_dim if kv_cache_dim != kv_lora_rank + qk_rope_head_dim else None @@ -5182,6 +5192,7 @@ class DSATokenToKVPool(MLATokenToKVPool): end_layer, use_dsa=True, override_kv_cache_dim=override_dim, + dcp_replicated=dcp_replicated, ) # self.index_k_dtype = torch.float8_e4m3fn # self.index_k_scale_dtype = torch.float32 @@ -5193,8 +5204,15 @@ class DSATokenToKVPool(MLATokenToKVPool): f"page_size {self.page_size} must be a multiple of index_kpool {index_kpool}" ) self.index_page_size = self.page_size // index_kpool + if dcp_replicated: + self.index_page_size //= get_parallel().attn_dcp_size if index_buf_size is None: - index_buf_size = size + dcp_size = get_parallel().attn_dcp_size + index_buf_size = ( + size + if dcp_replicated + else (size + self.page_size) * dcp_size - self.page_size + ) self.index_buf_size = index_buf_size # num head == 1 and head dim == 128 for index_k in DSA assert index_head_dim == 128 @@ -5206,7 +5224,7 @@ class DSATokenToKVPool(MLATokenToKVPool): ) assert len(self.skip_topk_layers) == layer_num - physical_page_size = self.page_size // index_kpool + physical_page_size = self.index_page_size if _is_hip: if aiter_can_use_preshuffle_paged_mqa(): assert physical_page_size % 16 == 0, ( diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 7f20cc0953..b3dc523a52 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -270,8 +270,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): kvc=kvc, num_layers=num_layers, ) - if _is_npu and dcp_size > 1: - target_indexer_size *= dcp_size + target_indexer_size *= dcp_size target_kv_size = self._cell_size - target_indexer_size from sglang.srt.layers.cp.utils import ( get_glm_dsa_layer_split_effective_num_layers, @@ -288,11 +287,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): num_layers=draft_num_layers, allocate_all_layers=True, ) - # The draft pool is replicated and consumes the widened - # allocator-global slot space on NPU DCP. - if _is_npu and dcp_size > 1: - draft_kv_size *= dcp_size - draft_indexer_size *= dcp_size + draft_kv_size *= dcp_size + draft_indexer_size *= dcp_size self._cell_size += draft_kv_size + draft_indexer_size else: self._cell_size = int( @@ -388,7 +384,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): kvc=kvc, num_layers=num_layers, ) - if _is_npu and not kvc.is_draft_worker and dcp_size > 1: + if not kvc.is_draft_worker: indexer_cell_size *= dcp_size cell_size += indexer_cell_size elif is_minimax_sparse(model_config.hf_config): diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py index 3451942b4a..a625079f51 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py @@ -266,7 +266,7 @@ class DeepseekMHAForwardMixin: kv_a, k_pe = self._get_mla_kv_buffer_from_fp8_for_dsa(forward_batch) else: # BF16/FP16 path: directly fetch from cache - if get_parallel().dcp_enabled: + if self.dcp_size > 1: kv_a, k_pe = all_gather_kv_cache_for_mha_extend( get_token_to_kv_pool(), self.attn_mha, diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py index b34e45f20b..d631421747 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py @@ -34,7 +34,7 @@ from sglang.srt.models.deepseek_common.utils import ( _use_aiter_bpreshuffle_gfx95, _use_aiter_gfx95, ) -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_exec from sglang.srt.utils import BumpAllocator, get_bool_env_var if TYPE_CHECKING: @@ -232,7 +232,7 @@ class DeepseekMHARocmForwardMixin: kv_a, k_pe = self._get_mla_kv_buffer_from_fp8_for_dsa(forward_batch) else: # BF16/FP16 path: directly fetch from cache - if get_parallel().dcp_enabled: + if self.dcp_size > 1: kv_a, k_pe = all_gather_kv_cache_for_mha_extend( get_token_to_kv_pool(), self.attn_mha, diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py index a784b6de11..f96fad1076 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py @@ -92,15 +92,6 @@ def _select_local_dcp_heads_for_autotune( return attn_output.narrow(1, rank * num_local_heads, num_local_heads) -def is_dcp_mla_decode_phase(forward_batch: ForwardBatch) -> bool: - if not get_parallel().dcp_enabled: - return False - return ( - forward_batch.forward_mode.is_decode() - or forward_batch.forward_mode.is_target_verify() - ) - - def is_mla_dcp_lse_base_on_e(attention_backend: Optional[str]) -> bool: return attention_backend in {"flashmla", "cutedsl_mla", "aiter"} @@ -131,6 +122,26 @@ def _apply_attention_output_gate(module, attn_output, gate): class DeepseekMLAForwardMixin: + def is_dcp_mla_decode_phase( + self: DeepseekV2AttentionMLA, forward_batch: ForwardBatch + ) -> bool: + if self.dcp_size == 1: + return False + mode = forward_batch.forward_mode + return ( + mode.is_decode() + or mode.is_target_verify() + or (self.use_dsa and mode.is_extend()) + ) + + def is_dcp_q_replicate_phase( + self: DeepseekV2AttentionMLA, forward_batch: ForwardBatch + ) -> bool: + if not get_parallel().dcp_replicate_q_proj or self.dcp_size == 1: + return False + mode = forward_batch.forward_mode + return mode.is_decode() or mode.is_target_verify() + def init_mla_forward(self: DeepseekV2AttentionMLA): self.flashinfer_mla_disable_ragged = ( get_exec().kernel.flashinfer_mla_disable_ragged @@ -162,6 +173,8 @@ class DeepseekMLAForwardMixin: ) -> bool: if getattr(self, "_kimi_split_gguf_kv_b", False): return False + if self.dcp_size > 1: + return False # Shared activation surface with the DSA indexer graph dispatch # (in piecewise/breakable graph + non-speculative extend). Like the indexer # dispatch, this fusion is on by default on that surface. @@ -264,7 +277,7 @@ class DeepseekMLAForwardMixin: return None if get_is_capture_mode(): return None - if get_parallel().dcp_enabled: + if self.dcp_size > 1: return None # Kernel shape constraints (tl.arange / tl.dot / block tiling). K # (qk_nope_head_dim) needs only K % 16 == 0 and K <= 256: power-of-2 @@ -314,8 +327,7 @@ class DeepseekMLAForwardMixin: # --dcp-replicate-q-proj: project full-head Q locally from pre-gathered # weights and skip the per-layer Q all-gather (bf16 decode absorb only). q_replicate_active = ( - get_parallel().dcp_replicate_q_proj - and is_dcp_mla_decode_phase(forward_batch) + self.is_dcp_q_replicate_phase(forward_batch) and not self.use_deep_gemm_bmm and self.w_kc_qrep is not None and self.q_b_proj_qrep_weight is not None @@ -391,7 +403,7 @@ class DeepseekMLAForwardMixin: # full-head Q from the gathered weight (skips Q all-gather) q = torch.nn.functional.linear(q, self.q_b_proj_qrep_weight).view( -1, - self.num_local_heads * get_parallel().attn_dcp_size, + self.num_local_heads * self.dcp_size, self.qk_head_dim, ) else: @@ -463,7 +475,7 @@ class DeepseekMLAForwardMixin: hidden_states, self.q_b_proj_qrep_weight ).view( -1, - self.num_local_heads * get_parallel().attn_dcp_size, + self.num_local_heads * self.dcp_size, self.qk_head_dim, ) else: @@ -633,8 +645,8 @@ class DeepseekMLAForwardMixin: ) # all_gather q_pe, q_nope_out,take tp8 as an example, q_pe [B, H, ROPE_DIM], q_nope_out [B, H, NOPE_DIM] gathered to [B, H * dcp_world_size, ROPE_DIM] [B, H * dcp_world_size, NOPE_DIM] for decode batch, and all gather k_pe, k_nope for extend batch. - if get_parallel().dcp_enabled: - if is_dcp_mla_decode_phase(forward_batch): + if self.dcp_size > 1: + if self.is_dcp_mla_decode_phase(forward_batch): if not q_replicate_active: q_nope_out, q_pe = all_gather_q_for_mla_decode( q_nope_out=q_nope_out, @@ -727,7 +739,7 @@ class DeepseekMLAForwardMixin: topk_indices=topk_indices, ) attn_output = fusion_plan.attn_output_buf - elif is_dcp_mla_decode_phase(forward_batch): + elif self.is_dcp_mla_decode_phase(forward_batch): # set return_lse=True to correct attn_output attn_output, lse = self.attn_mqa_for_dcp_decode( q_nope_out, @@ -776,10 +788,10 @@ class DeepseekMLAForwardMixin: ) # correct attn_output with respect to lse from other ranks - if is_dcp_mla_decode_phase(forward_batch): + if self.is_dcp_mla_decode_phase(forward_batch): attn_output = attn_output.view( -1, - self.num_local_heads * get_parallel().attn_dcp_size, + self.num_local_heads * self.dcp_size, self.kv_lora_rank, ) if get_in_autotune_dummy_run(): diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py index 2b78a47c19..5b265075f6 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py @@ -51,7 +51,6 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo ) from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mla import ( _select_local_dcp_heads_for_autotune, - is_dcp_mla_decode_phase, is_mla_dcp_lse_base_on_e, ) from sglang.srt.models.deepseek_common.utils import ( @@ -441,7 +440,7 @@ def _can_fuse_bmm_rope_cat_and_cache(attn: DeepseekV2AttentionMLA) -> bool: """ return ( _use_aiter_gfx95 - and not get_parallel().dcp_enabled + and attn.dcp_size == 1 and attn.w_kc.dtype == torch.float8_e4m3fn and attn.rotary_emb is not None and attn._skip_rope_for_dsa_tilelang_fused() @@ -503,8 +502,7 @@ class DeepseekMLARocmForwardMixin: from sglang.srt.model_executor.runner import get_is_capture_mode q_replicate_active = ( - get_parallel().dcp_replicate_q_proj - and is_dcp_mla_decode_phase(forward_batch) + self.is_dcp_q_replicate_phase(forward_batch) and not self.use_deep_gemm_bmm and self.w_kc_qrep is not None and self.q_b_proj_qrep_weight is not None @@ -634,7 +632,7 @@ class DeepseekMLARocmForwardMixin: if q_replicate_active: q = torch.nn.functional.linear(q, self.q_b_proj_qrep_weight).view( -1, - self.num_local_heads * get_parallel().attn_dcp_size, + self.num_local_heads * self.dcp_size, self.qk_head_dim, ) else: @@ -659,7 +657,7 @@ class DeepseekMLARocmForwardMixin: hidden_states, self.q_b_proj_qrep_weight ).view( -1, - self.num_local_heads * get_parallel().attn_dcp_size, + self.num_local_heads * self.dcp_size, self.qk_head_dim, ) else: @@ -740,7 +738,7 @@ class DeepseekMLARocmForwardMixin: fuse_rope_for_trtllm_mla = self._fuse_rope_for_trtllm_mla(forward_batch) force_rope_for_aiter_dcp_decode = ( - get_parallel().dcp_enabled + self.dcp_size > 1 and ( forward_batch.forward_mode.is_decode() or forward_batch.forward_mode.is_target_verify() @@ -771,8 +769,8 @@ class DeepseekMLARocmForwardMixin: q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) # all_gather q_pe, q_nope_out,take tp8 as an example, q_pe [B, H, ROPE_DIM], q_nope_out [B, H, NOPE_DIM] gathered to [B, H * dcp_world_size, ROPE_DIM] [B, H * dcp_world_size, NOPE_DIM] for decode batch, and all gather k_pe, k_nope for extend batch. - if get_parallel().dcp_enabled: - if is_dcp_mla_decode_phase(forward_batch): + if self.dcp_size > 1: + if self.is_dcp_mla_decode_phase(forward_batch): if not q_replicate_active: q_nope_out, q_pe = all_gather_q_for_mla_decode( q_nope_out=q_nope_out, @@ -910,7 +908,7 @@ class DeepseekMLARocmForwardMixin: "is_neox": self.rotary_emb.is_neox_style, "llama_4_scaling": llama_4_scaling, } - if is_dcp_mla_decode_phase(forward_batch): + if self.is_dcp_mla_decode_phase(forward_batch): # set return_lse=True to correct attn_output attn_output, lse = self.attn_mqa_for_dcp_decode( q_nope_out, @@ -948,7 +946,7 @@ class DeepseekMLARocmForwardMixin: or forward_batch.forward_mode.is_target_verify() or forward_batch.forward_mode.is_draft_extend_v2() ) - and get_parallel().dcp_enabled + and self.dcp_size > 1 ): q = torch.cat([q_nope_out, q_pe], dim=-1) if llama_4_scaling is not None: @@ -1003,10 +1001,10 @@ class DeepseekMLARocmForwardMixin: ) # correct attn_output with respect to lse from other ranks - if is_dcp_mla_decode_phase(forward_batch): + if self.is_dcp_mla_decode_phase(forward_batch): attn_output = attn_output.view( -1, - self.num_local_heads * get_parallel().attn_dcp_size, + self.num_local_heads * self.dcp_size, self.kv_lora_rank, ) if get_in_autotune_dummy_run(): diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index e5d85047b9..4255da8467 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -2018,6 +2018,9 @@ class DeepseekV2AttentionMLA( self.is_nextn = is_nextn attn_tp_size = get_parallel().attn_tp_size self.use_dsa = is_deepseek_dsa(config) + self.dcp_size = ( + 1 if (is_nextn and self.use_dsa) else get_parallel().attn_dcp_size + ) self.num_heads = num_heads assert num_heads % attn_tp_size == 0 self.num_local_heads = num_heads // attn_tp_size @@ -2156,9 +2159,9 @@ class DeepseekV2AttentionMLA( prefix=add_prefix("attn_mqa", prefix), ) # use num_local_heads * dcp_world_size because q_nope, q_rope is all gathered from dcp ranks - if get_parallel().dcp_enabled: + if self.dcp_size > 1: self.attn_mqa_for_dcp_decode = RadixAttention( - self.num_local_heads * get_parallel().attn_dcp_size, + self.num_local_heads * self.dcp_size, self.kv_lora_rank + self.qk_rope_head_dim, self.scaling, num_kv_heads=1, @@ -3441,7 +3444,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): kv_cache_device, create_chunked_prefix_cache_kv_indices_fn, ): - if _is_npu: + if _is_npu or is_deepseek_dsa(self.config): return None return prepare_decode_context_parallel_metadata( seq_lens=seq_lens, diff --git a/test/registered/dcp/test_dsa_dcp_attention.py b/test/registered/dcp/test_dsa_dcp_attention.py new file mode 100644 index 0000000000..fd9613daad --- /dev/null +++ b/test/registered/dcp/test_dsa_dcp_attention.py @@ -0,0 +1,148 @@ +import math +import unittest + +import torch + +from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton +from sglang.kernels.ops.attention.dsa.transform_index import ( + transform_index_page_table_dcp, +) +from sglang.srt.layers.attention.dsa.dsa_dcp import ( + dcp_trtllm_sparse_attention, + trtllm_rows_per_call, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="base-b", runner_config="4-gpu-b200") + +TOPK = 2048 +SCALE = 1 / math.sqrt(192) + + +def _reference_page_table(page_table, dcp_size, rank): + rows, counts = [], [] + for row in page_table.tolist(): + owned = [ + slot // dcp_size for slot in row if slot >= 0 and slot % dcp_size == rank + ] + rows.append(owned + [-1] * (len(row) - len(owned))) + counts.append(len(owned)) + return torch.tensor(rows, dtype=page_table.dtype), torch.tensor(counts) + + +def _sparse_attention(query, kv, page_table, seq_lens, workspace_bytes=512 << 20): + rows = query.shape[0] + return dcp_trtllm_sparse_attention( + query=query, + block_tables=page_table.unsqueeze(1), + seq_lens=seq_lens, + cu_seqlens_q=torch.arange(rows + 1, dtype=torch.int32, device="cuda"), + kv_cache=kv.reshape(-1, 1, 64, 576), + workspace_buffer=torch.zeros(workspace_bytes, dtype=torch.uint8, device="cuda"), + qk_nope_head_dim=128, + kv_lora_rank=512, + qk_rope_head_dim=64, + max_seq_len=TOPK, + sparse_mla_top_k=TOPK, + bmm1_scale=SCALE, + backend="trtllm-gen", + ) + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class TestDsaDcpAttention(CustomTestCase): + def test_page_table_keeps_owned_slots_in_order(self): + for width in (37, TOPK): + page_table = torch.arange(3 * width, dtype=torch.int32).view(3, width) + page_table = page_table.flip(1) + page_table[0, ::3] = -1 + page_table[1] = -1 + for dcp_size in (2, 4): + for rank in range(dcp_size): + with self.subTest(width=width, dcp=dcp_size, rank=rank): + expected, expected_counts = _reference_page_table( + page_table, dcp_size, rank + ) + actual, counts = transform_index_page_table_dcp( + page_table.cuda(), dcp_size, rank + ) + self.assertEqual(actual.cpu().tolist(), expected.tolist()) + self.assertEqual( + counts.cpu().tolist(), expected_counts.tolist() + ) + + def test_merged_shards_match_unsharded_attention(self): + generator = torch.Generator().manual_seed(0) + capacity, heads = 8192, 64 + page_table = torch.full((3, TOPK), -1, dtype=torch.int32) + page_table[0, 0] = capacity - 4 + page_table[1, :127] = ( + torch.randperm(capacity // 4, generator=generator)[:127] * 4 + ) + page_table[2] = torch.randperm(capacity, generator=generator)[:TOPK] + page_table = page_table.cuda() + seq_lens = torch.tensor([1, 127, TOPK], dtype=torch.int32, device="cuda") + query_cpu = torch.randn(3, 1, heads, 576, generator=generator) * 0.25 + kv_cpu = torch.randn(capacity, 576, generator=generator) * 0.25 + + for dtype in (torch.bfloat16, torch.float8_e4m3fn): + query, kv = query_cpu.cuda().to(dtype), kv_cpu.cuda().to(dtype) + expected, expected_lse = _sparse_attention(query, kv, page_table, seq_lens) + for dcp_size in (2, 4): + with self.subTest(dtype=dtype, dcp_size=dcp_size): + outputs, lses = [], [] + for rank in range(dcp_size): + local_page_table, counts = transform_index_page_table_dcp( + page_table, dcp_size, rank + ) + output, lse = _sparse_attention( + query, + kv[rank::dcp_size].contiguous(), + local_page_table, + counts, + ) + outputs.append(output.squeeze(1)) + lses.append(lse) + merged, merged_lse = dcp_lse_combine_triton( + torch.stack(outputs), + torch.stack(lses), + is_lse_base_on_e=False, + return_lse=True, + ) + atol = 0.008 if dtype == torch.float8_e4m3fn else 0.003 + torch.testing.assert_close( + merged, expected.squeeze(1), atol=atol, rtol=0.02 + ) + torch.testing.assert_close( + merged_lse, expected_lse, atol=0.003, rtol=0.002 + ) + + def test_rows_split_to_fit_the_workspace(self): + """A workspace too small for one call's softmax slab must split the rows, not overflow.""" + generator = torch.Generator().manual_seed(1) + rows, heads, capacity, length = 2400, 32, 4096, 128 + page_table = torch.full((rows, TOPK), -1, dtype=torch.int32) + for row in range(rows): + page_table[row, :length] = torch.randperm(capacity, generator=generator)[ + :length + ] + page_table = page_table.cuda() + seq_lens = torch.full((rows,), length, dtype=torch.int32, device="cuda") + query = (torch.randn(rows, 1, heads, 576, generator=generator) * 0.25).cuda() + kv = (torch.randn(capacity, 576, generator=generator) * 0.25).cuda() + query, kv = query.to(torch.bfloat16), kv.to(torch.bfloat16) + expected, expected_lse = _sparse_attention(query, kv, page_table, seq_lens) + workspace_bytes = (128 << 20) + 64 * heads * 256 * 8 + self.assertEqual( + trtllm_rows_per_call(workspace_bytes=workspace_bytes, heads=heads), 64 + ) + output, lse = _sparse_attention( + query, kv, page_table, seq_lens, workspace_bytes=workspace_bytes + ) + torch.testing.assert_close(output, expected, atol=0.003, rtol=0.02) + torch.testing.assert_close(lse, expected_lse, atol=0.003, rtol=0.002) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/dcp/test_glm53_dcp2.py b/test/registered/dcp/test_glm53_dcp2.py new file mode 100644 index 0000000000..338101de83 --- /dev/null +++ b/test/registered/dcp/test_glm53_dcp2.py @@ -0,0 +1,22 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.dsa_mtp_fixture import ( + DsaMtpEvalConfigDefaults, + DsaMtpServerBase, +) + +register_cuda_ci(est_time=600, stage="extra-b", runner_config="4-gpu-b200") + + +class TestGLM53NVFP4DCP2MTP(DsaMtpServerBase, DsaMtpEvalConfigDefaults, GSM8KMixin): + model = "RadixArk/GLM-5.3-NVFP4" + tp_size = 4 + mem_fraction_static = 0.8 + gsm8k_accuracy_thres = 0.88 + extra_server_args = ["--quantization", "modelopt_fp4", "--dcp-size", "2"] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/hicache/test_dsa_host_release.py b/test/registered/hicache/test_dsa_host_release.py index 7bf4b8a4e2..1ed2eef62e 100644 --- a/test/registered/hicache/test_dsa_host_release.py +++ b/test/registered/hicache/test_dsa_host_release.py @@ -12,6 +12,7 @@ from sglang.srt.mem_cache.pool_host.dsa import ( make_dsa_indexer_pool_decl, ) from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -19,6 +20,11 @@ register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small") class TestDSAHostRelease(CustomTestCase): + def setUp(self): + cm = get_parallel().override(attn_dcp_size=1) + cm.__enter__() + self.addCleanup(cm.__exit__, None, None, None) + def test_destroy_releases_index_pages_for_registration_reuse(self): pool = DSATokenToKVPool( size=256, diff --git a/test/registered/kernels/ops/attention/test_dcp_lse_combine.py b/test/registered/kernels/ops/attention/test_dcp_lse_combine.py index f177178c1f..bf2ccbfbea 100644 --- a/test/registered/kernels/ops/attention/test_dcp_lse_combine.py +++ b/test/registered/kernels/ops/attention/test_dcp_lse_combine.py @@ -185,6 +185,16 @@ class TestLSECombineEdgeCases(CustomTestCase): raise unittest.SkipTest("CUDA required") cls.device = "cuda" + def test_all_empty_rows_stay_zero(self): + from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton + + outputs = torch.ones(2, 2, 4, 64, device=self.device, dtype=torch.bfloat16) + lses = torch.zeros(2, 2, 4, device=self.device) + lses[:, 1] = -torch.inf + combined, _ = dcp_lse_combine_triton(outputs, lses, is_lse_base_on_e=False) + self.assertTrue((combined[0] == 1).all()) + self.assertTrue((combined[1] == 0).all()) + def test_one_shard_dominant(self): """One shard has much larger LSE -- output should be close to that shard.""" from sglang.kernels.ops.attention.dcp_kernels import ( diff --git a/test/registered/kernels/ops/attention/test_dsa_multi_ctas_counter.py b/test/registered/kernels/ops/attention/test_dsa_multi_ctas_counter.py index ce8fbf5751..a88d5fbf1f 100644 --- a/test/registered/kernels/ops/attention/test_dsa_multi_ctas_counter.py +++ b/test/registered/kernels/ops/attention/test_dsa_multi_ctas_counter.py @@ -36,6 +36,7 @@ def _make_backend(*, allocate_counter: bool = True): backend = object.__new__(DeepseekSparseAttnBackend) backend.device = "cuda" backend.num_q_heads = _NUM_Q_HEADS + backend.num_dcp_q_heads = _NUM_Q_HEADS backend.physical_page_size = 64 backend.hisparse_coordinator = None backend.speculative_num_draft_tokens = _NUM_DRAFT_TOKENS diff --git a/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py b/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py index bf38ba47e1..59a395832f 100644 --- a/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py +++ b/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py @@ -105,6 +105,8 @@ class TestDSABackendDPPadding(unittest.TestCase): _multi_ctas_kv_counter_buffer=None, device="cpu", num_q_heads=2, + num_dcp_q_heads=2, + dcp_size=1, ) backend._pad_topk_indices = MethodType( DeepseekSparseAttnBackend._pad_topk_indices, backend diff --git a/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py b/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py index 1b025f21bb..15e70342e4 100644 --- a/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py +++ b/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py @@ -13,6 +13,7 @@ from sglang.srt.mem_cache.pool_host.dsa import ( make_dsa_indexer_pool_decl, ) from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -36,6 +37,9 @@ class TestDSAHiCacheTransfer(unittest.TestCase): self.skipTest("DSA host transfer tests only support CUDA/ROCm.") if not (is_cuda() or is_hip()): self.skipTest("CUDA/ROCm not available.") + cm = get_parallel().override(attn_dcp_size=1) + cm.__enter__() + self.addCleanup(cm.__exit__, None, None, None) @staticmethod def _token_indices_for_pages(pages: torch.Tensor, page_size: int, device: str): diff --git a/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py b/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py index fee491d922..4c6544e68e 100644 --- a/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py +++ b/test/registered/unit/models/test_rocm_mla_absorb_fusion_gate.py @@ -33,6 +33,7 @@ def _attn(**overrides): rotary_emb=object(), current_attention_backend="dsa", use_deep_gemm_bmm=False, + dcp_size=1, kv_b_proj=SimpleNamespace(set_lora=False), _skip_rope_for_dsa_tilelang_fused=lambda: True, ) @@ -41,15 +42,6 @@ def _attn(**overrides): return attn -def _patch_dcp(case, *, dcp_enabled: bool): - """Give the gate a parallel context; the real one needs an initialized runtime.""" - parallel = SimpleNamespace(dcp_enabled=dcp_enabled) - saved = forward_mla_rocm.get_parallel - forward_mla_rocm.get_parallel = lambda: parallel - case.addCleanup(setattr, forward_mla_rocm, "get_parallel", saved) - return parallel - - class TestFusedAbsorbGate(CustomTestCase): def setUp(self): # The first term is the platform, and it short-circuits everything else; @@ -57,7 +49,6 @@ class TestFusedAbsorbGate(CustomTestCase): self._saved = forward_mla_rocm._use_aiter_gfx95 forward_mla_rocm._use_aiter_gfx95 = True self.addCleanup(setattr, forward_mla_rocm, "_use_aiter_gfx95", self._saved) - self._parallel = _patch_dcp(self, dcp_enabled=False) def test_dcp_turns_it_off(self): """DCP decode all-gathers q_nope_out, which this path never produces. @@ -65,8 +56,7 @@ class TestFusedAbsorbGate(CustomTestCase): Without this term the fused branch sets q_nope_out to None and all_gather_q_for_mla_decode dereferences it. """ - self._parallel.dcp_enabled = True - self.assertFalse(_CAN_FUSE(_attn())) + self.assertFalse(_CAN_FUSE(_attn(dcp_size=2))) def test_takes_the_fused_path_when_every_term_holds(self): self.assertTrue(_CAN_FUSE(_attn()))