[DCP] Decode context parallelism for DSA (GLM-5.2 / GLM-5.3) with MTP (#43735)
Co-authored-by: vincentzed <207368749+vincentzed@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
import math
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton
|
||||
from sglang.kernels.ops.attention.dsa.transform_index import (
|
||||
transform_index_page_table_dcp,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsa.dsa_dcp import (
|
||||
dcp_trtllm_sparse_attention,
|
||||
trtllm_rows_per_call,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b", runner_config="4-gpu-b200")
|
||||
|
||||
TOPK = 2048
|
||||
SCALE = 1 / math.sqrt(192)
|
||||
|
||||
|
||||
def _reference_page_table(page_table, dcp_size, rank):
|
||||
rows, counts = [], []
|
||||
for row in page_table.tolist():
|
||||
owned = [
|
||||
slot // dcp_size for slot in row if slot >= 0 and slot % dcp_size == rank
|
||||
]
|
||||
rows.append(owned + [-1] * (len(row) - len(owned)))
|
||||
counts.append(len(owned))
|
||||
return torch.tensor(rows, dtype=page_table.dtype), torch.tensor(counts)
|
||||
|
||||
|
||||
def _sparse_attention(query, kv, page_table, seq_lens, workspace_bytes=512 << 20):
|
||||
rows = query.shape[0]
|
||||
return dcp_trtllm_sparse_attention(
|
||||
query=query,
|
||||
block_tables=page_table.unsqueeze(1),
|
||||
seq_lens=seq_lens,
|
||||
cu_seqlens_q=torch.arange(rows + 1, dtype=torch.int32, device="cuda"),
|
||||
kv_cache=kv.reshape(-1, 1, 64, 576),
|
||||
workspace_buffer=torch.zeros(workspace_bytes, dtype=torch.uint8, device="cuda"),
|
||||
qk_nope_head_dim=128,
|
||||
kv_lora_rank=512,
|
||||
qk_rope_head_dim=64,
|
||||
max_seq_len=TOPK,
|
||||
sparse_mla_top_k=TOPK,
|
||||
bmm1_scale=SCALE,
|
||||
backend="trtllm-gen",
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
|
||||
class TestDsaDcpAttention(CustomTestCase):
|
||||
def test_page_table_keeps_owned_slots_in_order(self):
|
||||
for width in (37, TOPK):
|
||||
page_table = torch.arange(3 * width, dtype=torch.int32).view(3, width)
|
||||
page_table = page_table.flip(1)
|
||||
page_table[0, ::3] = -1
|
||||
page_table[1] = -1
|
||||
for dcp_size in (2, 4):
|
||||
for rank in range(dcp_size):
|
||||
with self.subTest(width=width, dcp=dcp_size, rank=rank):
|
||||
expected, expected_counts = _reference_page_table(
|
||||
page_table, dcp_size, rank
|
||||
)
|
||||
actual, counts = transform_index_page_table_dcp(
|
||||
page_table.cuda(), dcp_size, rank
|
||||
)
|
||||
self.assertEqual(actual.cpu().tolist(), expected.tolist())
|
||||
self.assertEqual(
|
||||
counts.cpu().tolist(), expected_counts.tolist()
|
||||
)
|
||||
|
||||
def test_merged_shards_match_unsharded_attention(self):
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
capacity, heads = 8192, 64
|
||||
page_table = torch.full((3, TOPK), -1, dtype=torch.int32)
|
||||
page_table[0, 0] = capacity - 4
|
||||
page_table[1, :127] = (
|
||||
torch.randperm(capacity // 4, generator=generator)[:127] * 4
|
||||
)
|
||||
page_table[2] = torch.randperm(capacity, generator=generator)[:TOPK]
|
||||
page_table = page_table.cuda()
|
||||
seq_lens = torch.tensor([1, 127, TOPK], dtype=torch.int32, device="cuda")
|
||||
query_cpu = torch.randn(3, 1, heads, 576, generator=generator) * 0.25
|
||||
kv_cpu = torch.randn(capacity, 576, generator=generator) * 0.25
|
||||
|
||||
for dtype in (torch.bfloat16, torch.float8_e4m3fn):
|
||||
query, kv = query_cpu.cuda().to(dtype), kv_cpu.cuda().to(dtype)
|
||||
expected, expected_lse = _sparse_attention(query, kv, page_table, seq_lens)
|
||||
for dcp_size in (2, 4):
|
||||
with self.subTest(dtype=dtype, dcp_size=dcp_size):
|
||||
outputs, lses = [], []
|
||||
for rank in range(dcp_size):
|
||||
local_page_table, counts = transform_index_page_table_dcp(
|
||||
page_table, dcp_size, rank
|
||||
)
|
||||
output, lse = _sparse_attention(
|
||||
query,
|
||||
kv[rank::dcp_size].contiguous(),
|
||||
local_page_table,
|
||||
counts,
|
||||
)
|
||||
outputs.append(output.squeeze(1))
|
||||
lses.append(lse)
|
||||
merged, merged_lse = dcp_lse_combine_triton(
|
||||
torch.stack(outputs),
|
||||
torch.stack(lses),
|
||||
is_lse_base_on_e=False,
|
||||
return_lse=True,
|
||||
)
|
||||
atol = 0.008 if dtype == torch.float8_e4m3fn else 0.003
|
||||
torch.testing.assert_close(
|
||||
merged, expected.squeeze(1), atol=atol, rtol=0.02
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
merged_lse, expected_lse, atol=0.003, rtol=0.002
|
||||
)
|
||||
|
||||
def test_rows_split_to_fit_the_workspace(self):
|
||||
"""A workspace too small for one call's softmax slab must split the rows, not overflow."""
|
||||
generator = torch.Generator().manual_seed(1)
|
||||
rows, heads, capacity, length = 2400, 32, 4096, 128
|
||||
page_table = torch.full((rows, TOPK), -1, dtype=torch.int32)
|
||||
for row in range(rows):
|
||||
page_table[row, :length] = torch.randperm(capacity, generator=generator)[
|
||||
:length
|
||||
]
|
||||
page_table = page_table.cuda()
|
||||
seq_lens = torch.full((rows,), length, dtype=torch.int32, device="cuda")
|
||||
query = (torch.randn(rows, 1, heads, 576, generator=generator) * 0.25).cuda()
|
||||
kv = (torch.randn(capacity, 576, generator=generator) * 0.25).cuda()
|
||||
query, kv = query.to(torch.bfloat16), kv.to(torch.bfloat16)
|
||||
expected, expected_lse = _sparse_attention(query, kv, page_table, seq_lens)
|
||||
workspace_bytes = (128 << 20) + 64 * heads * 256 * 8
|
||||
self.assertEqual(
|
||||
trtllm_rows_per_call(workspace_bytes=workspace_bytes, heads=heads), 64
|
||||
)
|
||||
output, lse = _sparse_attention(
|
||||
query, kv, page_table, seq_lens, workspace_bytes=workspace_bytes
|
||||
)
|
||||
torch.testing.assert_close(output, expected, atol=0.003, rtol=0.02)
|
||||
torch.testing.assert_close(lse, expected_lse, atol=0.003, rtol=0.002)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,22 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.server_fixtures.dsa_mtp_fixture import (
|
||||
DsaMtpEvalConfigDefaults,
|
||||
DsaMtpServerBase,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=600, stage="extra-b", runner_config="4-gpu-b200")
|
||||
|
||||
|
||||
class TestGLM53NVFP4DCP2MTP(DsaMtpServerBase, DsaMtpEvalConfigDefaults, GSM8KMixin):
|
||||
model = "RadixArk/GLM-5.3-NVFP4"
|
||||
tp_size = 4
|
||||
mem_fraction_static = 0.8
|
||||
gsm8k_accuracy_thres = 0.88
|
||||
extra_server_args = ["--quantization", "modelopt_fp4", "--dcp-size", "2"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.mem_cache.pool_host.dsa import (
|
||||
make_dsa_indexer_pool_decl,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -19,6 +20,11 @@ register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
class TestDSAHostRelease(CustomTestCase):
|
||||
def setUp(self):
|
||||
cm = get_parallel().override(attn_dcp_size=1)
|
||||
cm.__enter__()
|
||||
self.addCleanup(cm.__exit__, None, None, None)
|
||||
|
||||
def test_destroy_releases_index_pages_for_registration_reuse(self):
|
||||
pool = DSATokenToKVPool(
|
||||
size=256,
|
||||
|
||||
@@ -185,6 +185,16 @@ class TestLSECombineEdgeCases(CustomTestCase):
|
||||
raise unittest.SkipTest("CUDA required")
|
||||
cls.device = "cuda"
|
||||
|
||||
def test_all_empty_rows_stay_zero(self):
|
||||
from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton
|
||||
|
||||
outputs = torch.ones(2, 2, 4, 64, device=self.device, dtype=torch.bfloat16)
|
||||
lses = torch.zeros(2, 2, 4, device=self.device)
|
||||
lses[:, 1] = -torch.inf
|
||||
combined, _ = dcp_lse_combine_triton(outputs, lses, is_lse_base_on_e=False)
|
||||
self.assertTrue((combined[0] == 1).all())
|
||||
self.assertTrue((combined[1] == 0).all())
|
||||
|
||||
def test_one_shard_dominant(self):
|
||||
"""One shard has much larger LSE -- output should be close to that shard."""
|
||||
from sglang.kernels.ops.attention.dcp_kernels import (
|
||||
|
||||
@@ -36,6 +36,7 @@ def _make_backend(*, allocate_counter: bool = True):
|
||||
backend = object.__new__(DeepseekSparseAttnBackend)
|
||||
backend.device = "cuda"
|
||||
backend.num_q_heads = _NUM_Q_HEADS
|
||||
backend.num_dcp_q_heads = _NUM_Q_HEADS
|
||||
backend.physical_page_size = 64
|
||||
backend.hisparse_coordinator = None
|
||||
backend.speculative_num_draft_tokens = _NUM_DRAFT_TOKENS
|
||||
|
||||
@@ -105,6 +105,8 @@ class TestDSABackendDPPadding(unittest.TestCase):
|
||||
_multi_ctas_kv_counter_buffer=None,
|
||||
device="cpu",
|
||||
num_q_heads=2,
|
||||
num_dcp_q_heads=2,
|
||||
dcp_size=1,
|
||||
)
|
||||
backend._pad_topk_indices = MethodType(
|
||||
DeepseekSparseAttnBackend._pad_topk_indices, backend
|
||||
|
||||
@@ -13,6 +13,7 @@ from sglang.srt.mem_cache.pool_host.dsa import (
|
||||
make_dsa_indexer_pool_decl,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
@@ -36,6 +37,9 @@ class TestDSAHiCacheTransfer(unittest.TestCase):
|
||||
self.skipTest("DSA host transfer tests only support CUDA/ROCm.")
|
||||
if not (is_cuda() or is_hip()):
|
||||
self.skipTest("CUDA/ROCm not available.")
|
||||
cm = get_parallel().override(attn_dcp_size=1)
|
||||
cm.__enter__()
|
||||
self.addCleanup(cm.__exit__, None, None, None)
|
||||
|
||||
@staticmethod
|
||||
def _token_indices_for_pages(pages: torch.Tensor, page_size: int, device: str):
|
||||
|
||||
@@ -33,6 +33,7 @@ def _attn(**overrides):
|
||||
rotary_emb=object(),
|
||||
current_attention_backend="dsa",
|
||||
use_deep_gemm_bmm=False,
|
||||
dcp_size=1,
|
||||
kv_b_proj=SimpleNamespace(set_lora=False),
|
||||
_skip_rope_for_dsa_tilelang_fused=lambda: True,
|
||||
)
|
||||
@@ -41,15 +42,6 @@ def _attn(**overrides):
|
||||
return attn
|
||||
|
||||
|
||||
def _patch_dcp(case, *, dcp_enabled: bool):
|
||||
"""Give the gate a parallel context; the real one needs an initialized runtime."""
|
||||
parallel = SimpleNamespace(dcp_enabled=dcp_enabled)
|
||||
saved = forward_mla_rocm.get_parallel
|
||||
forward_mla_rocm.get_parallel = lambda: parallel
|
||||
case.addCleanup(setattr, forward_mla_rocm, "get_parallel", saved)
|
||||
return parallel
|
||||
|
||||
|
||||
class TestFusedAbsorbGate(CustomTestCase):
|
||||
def setUp(self):
|
||||
# The first term is the platform, and it short-circuits everything else;
|
||||
@@ -57,7 +49,6 @@ class TestFusedAbsorbGate(CustomTestCase):
|
||||
self._saved = forward_mla_rocm._use_aiter_gfx95
|
||||
forward_mla_rocm._use_aiter_gfx95 = True
|
||||
self.addCleanup(setattr, forward_mla_rocm, "_use_aiter_gfx95", self._saved)
|
||||
self._parallel = _patch_dcp(self, dcp_enabled=False)
|
||||
|
||||
def test_dcp_turns_it_off(self):
|
||||
"""DCP decode all-gathers q_nope_out, which this path never produces.
|
||||
@@ -65,8 +56,7 @@ class TestFusedAbsorbGate(CustomTestCase):
|
||||
Without this term the fused branch sets q_nope_out to None and
|
||||
all_gather_q_for_mla_decode dereferences it.
|
||||
"""
|
||||
self._parallel.dcp_enabled = True
|
||||
self.assertFalse(_CAN_FUSE(_attn()))
|
||||
self.assertFalse(_CAN_FUSE(_attn(dcp_size=2)))
|
||||
|
||||
def test_takes_the_fused_path_when_every_term_holds(self):
|
||||
self.assertTrue(_CAN_FUSE(_attn()))
|
||||
|
||||
Reference in New Issue
Block a user