[DCP] Decode context parallelism for DSA (GLM-5.2 / GLM-5.3) with MTP (#43735)

Co-authored-by: vincentzed <207368749+vincentzed@users.noreply.github.com>
This commit is contained in:
Khoa Pham
2026-10-13 11:35:34 -07:00
committed by GitHub
co-authored by vincentzed
parent 6ade23696d
commit 8bf145d119
26 changed files with 565 additions and 88 deletions
@@ -0,0 +1,148 @@
import math
import unittest
import torch
from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton
from sglang.kernels.ops.attention.dsa.transform_index import (
transform_index_page_table_dcp,
)
from sglang.srt.layers.attention.dsa.dsa_dcp import (
dcp_trtllm_sparse_attention,
trtllm_rows_per_call,
)
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=30, stage="base-b", runner_config="4-gpu-b200")
TOPK = 2048
SCALE = 1 / math.sqrt(192)
def _reference_page_table(page_table, dcp_size, rank):
rows, counts = [], []
for row in page_table.tolist():
owned = [
slot // dcp_size for slot in row if slot >= 0 and slot % dcp_size == rank
]
rows.append(owned + [-1] * (len(row) - len(owned)))
counts.append(len(owned))
return torch.tensor(rows, dtype=page_table.dtype), torch.tensor(counts)
def _sparse_attention(query, kv, page_table, seq_lens, workspace_bytes=512 << 20):
rows = query.shape[0]
return dcp_trtllm_sparse_attention(
query=query,
block_tables=page_table.unsqueeze(1),
seq_lens=seq_lens,
cu_seqlens_q=torch.arange(rows + 1, dtype=torch.int32, device="cuda"),
kv_cache=kv.reshape(-1, 1, 64, 576),
workspace_buffer=torch.zeros(workspace_bytes, dtype=torch.uint8, device="cuda"),
qk_nope_head_dim=128,
kv_lora_rank=512,
qk_rope_head_dim=64,
max_seq_len=TOPK,
sparse_mla_top_k=TOPK,
bmm1_scale=SCALE,
backend="trtllm-gen",
)
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
class TestDsaDcpAttention(CustomTestCase):
def test_page_table_keeps_owned_slots_in_order(self):
for width in (37, TOPK):
page_table = torch.arange(3 * width, dtype=torch.int32).view(3, width)
page_table = page_table.flip(1)
page_table[0, ::3] = -1
page_table[1] = -1
for dcp_size in (2, 4):
for rank in range(dcp_size):
with self.subTest(width=width, dcp=dcp_size, rank=rank):
expected, expected_counts = _reference_page_table(
page_table, dcp_size, rank
)
actual, counts = transform_index_page_table_dcp(
page_table.cuda(), dcp_size, rank
)
self.assertEqual(actual.cpu().tolist(), expected.tolist())
self.assertEqual(
counts.cpu().tolist(), expected_counts.tolist()
)
def test_merged_shards_match_unsharded_attention(self):
generator = torch.Generator().manual_seed(0)
capacity, heads = 8192, 64
page_table = torch.full((3, TOPK), -1, dtype=torch.int32)
page_table[0, 0] = capacity - 4
page_table[1, :127] = (
torch.randperm(capacity // 4, generator=generator)[:127] * 4
)
page_table[2] = torch.randperm(capacity, generator=generator)[:TOPK]
page_table = page_table.cuda()
seq_lens = torch.tensor([1, 127, TOPK], dtype=torch.int32, device="cuda")
query_cpu = torch.randn(3, 1, heads, 576, generator=generator) * 0.25
kv_cpu = torch.randn(capacity, 576, generator=generator) * 0.25
for dtype in (torch.bfloat16, torch.float8_e4m3fn):
query, kv = query_cpu.cuda().to(dtype), kv_cpu.cuda().to(dtype)
expected, expected_lse = _sparse_attention(query, kv, page_table, seq_lens)
for dcp_size in (2, 4):
with self.subTest(dtype=dtype, dcp_size=dcp_size):
outputs, lses = [], []
for rank in range(dcp_size):
local_page_table, counts = transform_index_page_table_dcp(
page_table, dcp_size, rank
)
output, lse = _sparse_attention(
query,
kv[rank::dcp_size].contiguous(),
local_page_table,
counts,
)
outputs.append(output.squeeze(1))
lses.append(lse)
merged, merged_lse = dcp_lse_combine_triton(
torch.stack(outputs),
torch.stack(lses),
is_lse_base_on_e=False,
return_lse=True,
)
atol = 0.008 if dtype == torch.float8_e4m3fn else 0.003
torch.testing.assert_close(
merged, expected.squeeze(1), atol=atol, rtol=0.02
)
torch.testing.assert_close(
merged_lse, expected_lse, atol=0.003, rtol=0.002
)
def test_rows_split_to_fit_the_workspace(self):
"""A workspace too small for one call's softmax slab must split the rows, not overflow."""
generator = torch.Generator().manual_seed(1)
rows, heads, capacity, length = 2400, 32, 4096, 128
page_table = torch.full((rows, TOPK), -1, dtype=torch.int32)
for row in range(rows):
page_table[row, :length] = torch.randperm(capacity, generator=generator)[
:length
]
page_table = page_table.cuda()
seq_lens = torch.full((rows,), length, dtype=torch.int32, device="cuda")
query = (torch.randn(rows, 1, heads, 576, generator=generator) * 0.25).cuda()
kv = (torch.randn(capacity, 576, generator=generator) * 0.25).cuda()
query, kv = query.to(torch.bfloat16), kv.to(torch.bfloat16)
expected, expected_lse = _sparse_attention(query, kv, page_table, seq_lens)
workspace_bytes = (128 << 20) + 64 * heads * 256 * 8
self.assertEqual(
trtllm_rows_per_call(workspace_bytes=workspace_bytes, heads=heads), 64
)
output, lse = _sparse_attention(
query, kv, page_table, seq_lens, workspace_bytes=workspace_bytes
)
torch.testing.assert_close(output, expected, atol=0.003, rtol=0.02)
torch.testing.assert_close(lse, expected_lse, atol=0.003, rtol=0.002)
if __name__ == "__main__":
unittest.main()
+22
View File
@@ -0,0 +1,22 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.dsa_mtp_fixture import (
DsaMtpEvalConfigDefaults,
DsaMtpServerBase,
)
register_cuda_ci(est_time=600, stage="extra-b", runner_config="4-gpu-b200")
class TestGLM53NVFP4DCP2MTP(DsaMtpServerBase, DsaMtpEvalConfigDefaults, GSM8KMixin):
model = "RadixArk/GLM-5.3-NVFP4"
tp_size = 4
mem_fraction_static = 0.8
gsm8k_accuracy_thres = 0.88
extra_server_args = ["--quantization", "modelopt_fp4", "--dcp-size", "2"]
if __name__ == "__main__":
unittest.main()
@@ -12,6 +12,7 @@ from sglang.srt.mem_cache.pool_host.dsa import (
make_dsa_indexer_pool_decl,
)
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
@@ -19,6 +20,11 @@ register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small")
class TestDSAHostRelease(CustomTestCase):
def setUp(self):
cm = get_parallel().override(attn_dcp_size=1)
cm.__enter__()
self.addCleanup(cm.__exit__, None, None, None)
def test_destroy_releases_index_pages_for_registration_reuse(self):
pool = DSATokenToKVPool(
size=256,
@@ -185,6 +185,16 @@ class TestLSECombineEdgeCases(CustomTestCase):
raise unittest.SkipTest("CUDA required")
cls.device = "cuda"
def test_all_empty_rows_stay_zero(self):
from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton
outputs = torch.ones(2, 2, 4, 64, device=self.device, dtype=torch.bfloat16)
lses = torch.zeros(2, 2, 4, device=self.device)
lses[:, 1] = -torch.inf
combined, _ = dcp_lse_combine_triton(outputs, lses, is_lse_base_on_e=False)
self.assertTrue((combined[0] == 1).all())
self.assertTrue((combined[1] == 0).all())
def test_one_shard_dominant(self):
"""One shard has much larger LSE -- output should be close to that shard."""
from sglang.kernels.ops.attention.dcp_kernels import (
@@ -36,6 +36,7 @@ def _make_backend(*, allocate_counter: bool = True):
backend = object.__new__(DeepseekSparseAttnBackend)
backend.device = "cuda"
backend.num_q_heads = _NUM_Q_HEADS
backend.num_dcp_q_heads = _NUM_Q_HEADS
backend.physical_page_size = 64
backend.hisparse_coordinator = None
backend.speculative_num_draft_tokens = _NUM_DRAFT_TOKENS
@@ -105,6 +105,8 @@ class TestDSABackendDPPadding(unittest.TestCase):
_multi_ctas_kv_counter_buffer=None,
device="cpu",
num_q_heads=2,
num_dcp_q_heads=2,
dcp_size=1,
)
backend._pad_topk_indices = MethodType(
DeepseekSparseAttnBackend._pad_topk_indices, backend
@@ -13,6 +13,7 @@ from sglang.srt.mem_cache.pool_host.dsa import (
make_dsa_indexer_pool_decl,
)
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
@@ -36,6 +37,9 @@ class TestDSAHiCacheTransfer(unittest.TestCase):
self.skipTest("DSA host transfer tests only support CUDA/ROCm.")
if not (is_cuda() or is_hip()):
self.skipTest("CUDA/ROCm not available.")
cm = get_parallel().override(attn_dcp_size=1)
cm.__enter__()
self.addCleanup(cm.__exit__, None, None, None)
@staticmethod
def _token_indices_for_pages(pages: torch.Tensor, page_size: int, device: str):
@@ -33,6 +33,7 @@ def _attn(**overrides):
rotary_emb=object(),
current_attention_backend="dsa",
use_deep_gemm_bmm=False,
dcp_size=1,
kv_b_proj=SimpleNamespace(set_lora=False),
_skip_rope_for_dsa_tilelang_fused=lambda: True,
)
@@ -41,15 +42,6 @@ def _attn(**overrides):
return attn
def _patch_dcp(case, *, dcp_enabled: bool):
"""Give the gate a parallel context; the real one needs an initialized runtime."""
parallel = SimpleNamespace(dcp_enabled=dcp_enabled)
saved = forward_mla_rocm.get_parallel
forward_mla_rocm.get_parallel = lambda: parallel
case.addCleanup(setattr, forward_mla_rocm, "get_parallel", saved)
return parallel
class TestFusedAbsorbGate(CustomTestCase):
def setUp(self):
# The first term is the platform, and it short-circuits everything else;
@@ -57,7 +49,6 @@ class TestFusedAbsorbGate(CustomTestCase):
self._saved = forward_mla_rocm._use_aiter_gfx95
forward_mla_rocm._use_aiter_gfx95 = True
self.addCleanup(setattr, forward_mla_rocm, "_use_aiter_gfx95", self._saved)
self._parallel = _patch_dcp(self, dcp_enabled=False)
def test_dcp_turns_it_off(self):
"""DCP decode all-gathers q_nope_out, which this path never produces.
@@ -65,8 +56,7 @@ class TestFusedAbsorbGate(CustomTestCase):
Without this term the fused branch sets q_nope_out to None and
all_gather_q_for_mla_decode dereferences it.
"""
self._parallel.dcp_enabled = True
self.assertFalse(_CAN_FUSE(_attn()))
self.assertFalse(_CAN_FUSE(_attn(dcp_size=2)))
def test_takes_the_fused_path_when_every_term_holds(self):
self.assertTrue(_CAN_FUSE(_attn()))