[Perf] Improve GLM-5.3-Flash DFlash decoding on B300 (#42489)

Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
leonardHONG
2026-10-06 12:05:37 +08:00
committed by GitHub
co-authored by lianghougong Xiaoyu Zhang
parent fcbbefa82a
commit 1e1bdeedbb
8 changed files with 157 additions and 61 deletions
@@ -7,7 +7,7 @@ import torch.nn as nn
import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.kernels.jit.utils import get_jit_cuda_arch, is_arch_support_pdl
from sglang.srt.utils import (
cdiv,
cpu_has_amx_support,
@@ -223,10 +223,16 @@ def layer_norm_gated_fwd(
# heuristics for number of warps
if D <= 512:
use_pdl = is_arch_support_pdl()
BT = 32
pdl_kwargs = (
{"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {}
)
use_bt8 = 8 <= T <= 256 if is_rms_norm and activation == "sigmoid" else T == 64
if use_pdl and use_bt8 and (D, x.dtype) == (128, torch.bfloat16):
arch = get_jit_cuda_arch()
if (arch.major, arch.minor) == (10, 3):
# Small sigmoid-gated RMSNorm batches use eight rows per CTA.
# Other norm modes retain their T=64 selection.
BT = 8
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if use_pdl else {}
layer_norm_gated_fwd_kernel[(cdiv(T, BT),)](
x=x,
g=g,
@@ -5,11 +5,18 @@ import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.srt.utils import is_gfx95_supported, is_hip, is_sm90_supported
from sglang.srt.utils import (
get_device_sm,
is_cuda,
is_gfx95_supported,
is_hip,
is_sm90_supported,
)
_is_hip = is_hip()
_is_gfx95 = is_gfx95_supported()
_is_sm90 = is_sm90_supported()
_is_sm103 = is_cuda() and get_device_sm() == 103
def _select_recurrent_launch_config(
@@ -20,8 +27,10 @@ def _select_recurrent_launch_config(
v: int,
is_kda: bool,
target_verify: bool = False,
*,
cache_steps: int = 0,
) -> tuple[int, int]:
"""Select the value tile and warp count for recurrent GDN."""
"""Select the value tile and warp count for recurrent GDN/KDA."""
if (
_is_hip
and _is_gfx95
@@ -46,6 +55,14 @@ def _select_recurrent_launch_config(
# with narrow tiles but not bit-identical to BV=32. Only the dense
# intermediate-state verify sets target_verify; cache_ring keeps BV=32.
return 4, 1
if (
_is_sm103
and is_kda
and target_verify
and (n, h, hv, k, v, cache_steps) == (1, 8, 8, 128, 128, 8)
):
# GLM TP8 DFlash verification: 256 CTAs instead of 32.
return 4, 1
return min(triton.next_power_of_2(v), 32), 1
@@ -448,6 +465,16 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
# Update hidden state: h += k[:, None] * v[None, :]
if MATCH_CUTEDSL_DECODE:
b_h = tl.fma(b_k[:, None], b_v[None, :], b_h)
elif IS_KDA and V == 128 and BV == 4:
# The narrow KDA tile keeps the separate FP32 multiply/add rounding.
b_h = tl.inline_asm_elementwise(
"add.rn.f32 $0, $1, $2;",
constraints="=f,f,f",
args=[b_h, b_k[:, None] * b_v[None, :]],
dtype=tl.float32,
is_pure=True,
pack=1,
)
else:
b_h += b_k[:, None] * b_v[None, :]
@@ -567,8 +594,21 @@ def fused_sigmoid_gating_delta_rule_update(
stride_a = a.stride()[1] if a.ndim == 4 else a.stride()[-2]
HV = v.shape[2]
N = B if cu_seqlens is None else len(cu_seqlens) - 1
use_pdl = is_arch_support_pdl()
target_verify = (
disable_state_update and not match_cutedsl_decode
if is_kda
else intermediate_states_buffer is not None
)
BV, num_warps = _select_recurrent_launch_config(
N, H, HV, K, V, is_kda, target_verify=intermediate_states_buffer is not None
N,
H,
HV,
K,
V,
is_kda,
target_verify=target_verify,
cache_steps=cache_steps,
)
BK = triton.next_power_of_2(K)
NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)
@@ -630,7 +670,7 @@ def fused_sigmoid_gating_delta_rule_update(
# PDL (sm90+): chain this kernel behind its producer conv1d_update, which
# already launches dependents. Bit-exact (scheduling only) — benefits both
# KDA and GDN recurrent paths.
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {}
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if use_pdl else {}
# CuTeDSL specializes on the persisted state dtype. Its BF16 decode path
# writes BF16 after every token and reloads FP32 for the next recurrence;
@@ -102,16 +102,17 @@ class DeepseekSparseAttnBackendKPoolMixin:
full_seqlens_expanded=kpool_inputs.full_seqlens_expanded,
)
metadata = init_pooled_paged_mqa_metadata(
metadata,
metadata.cache_seqlens_int32,
forward_mode,
pool_size=self.dsa_index_kpool,
real_page_size=self.real_page_size,
slots_per_page=slots_per_page,
build_schedule_metadata=build_schedule_metadata,
)
if forward_mode.is_decode_or_idle():
metadata = init_pooled_paged_mqa_metadata(
metadata,
metadata.cache_seqlens_int32,
forward_mode,
pool_size=self.dsa_index_kpool,
real_page_size=self.real_page_size,
slots_per_page=slots_per_page,
build_schedule_metadata=build_schedule_metadata,
)
return init_kpool_write_plan(
metadata,
forward_batch,
@@ -170,16 +171,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
slots_per_page = self._kpool_slots_per_page()
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
if forward_mode.is_decode_or_idle():
metadata = init_pooled_paged_mqa_metadata(
metadata,
metadata.cache_seqlens_int32,
forward_mode,
pool_size=self.dsa_index_kpool,
real_page_size=self.real_page_size,
slots_per_page=slots_per_page,
build_schedule_metadata=build_schedule_metadata,
)
metadata = init_pooled_paged_mqa_metadata(
metadata,
metadata.cache_seqlens_int32,
forward_mode,
pool_size=self.dsa_index_kpool,
real_page_size=self.real_page_size,
slots_per_page=slots_per_page,
build_schedule_metadata=build_schedule_metadata,
)
if (
forward_mode.is_decode_or_idle()
@@ -218,16 +218,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
slots_per_page = self._kpool_slots_per_page()
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
if forward_mode.is_decode_or_idle():
update_pooled_paged_mqa_metadata(
metadata,
metadata.cache_seqlens_int32,
forward_mode,
pool_size=self.dsa_index_kpool,
real_page_size=self.real_page_size,
slots_per_page=slots_per_page,
build_schedule_metadata=build_schedule_metadata,
)
update_pooled_paged_mqa_metadata(
metadata,
metadata.cache_seqlens_int32,
forward_mode,
pool_size=self.dsa_index_kpool,
real_page_size=self.real_page_size,
slots_per_page=slots_per_page,
build_schedule_metadata=build_schedule_metadata,
)
if not (
forward_mode.is_decode_or_idle()
@@ -270,16 +269,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
slots_per_page = self._kpool_slots_per_page()
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
if forward_mode.is_decode_or_idle():
update_pooled_paged_mqa_metadata(
metadata,
precomputed.cache_seqlens,
forward_mode,
pool_size=self.dsa_index_kpool,
real_page_size=self.real_page_size,
slots_per_page=slots_per_page,
build_schedule_metadata=build_schedule_metadata,
)
update_pooled_paged_mqa_metadata(
metadata,
precomputed.cache_seqlens,
forward_mode,
pool_size=self.dsa_index_kpool,
real_page_size=self.real_page_size,
slots_per_page=slots_per_page,
build_schedule_metadata=build_schedule_metadata,
)
if not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()):
return
@@ -805,9 +805,14 @@ class IndexerKPool(MultiPlatformOp):
if plan is not None and plan.pool_seqlens_per_q is not None:
pool_seqlens = plan.pool_seqlens_per_q[: seqlens_32.shape[0]]
pool_context_lens = pool_seqlens.contiguous().view(-1, 1)
pool_block_tables = build_pooled_page_table_64(
block_tables, self.index_kpool
).contiguous()
pool_block_tables = attn_metadata.pooled_real_page_table
if (
pool_block_tables is None
or attn_metadata.pooled_index_kpool != self.index_kpool
):
pool_block_tables = build_pooled_page_table_64(
block_tables, self.index_kpool
).contiguous()
pool_schedule_metadata = plan.pool_schedule_metadata
if pool_schedule_metadata is None and build_schedule_metadata:
pool_schedule_metadata = deep_gemm.get_paged_mqa_logits_metadata(
@@ -585,16 +585,23 @@ def init_pooled_paged_mqa_metadata(
if (
not _is_kpool_layout_enabled(pool_size, real_page_size)
or not is_cuda()
or not forward_mode.is_decode_or_idle()
or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify())
):
return metadata
pooled_page_table = build_pooled_page_table_64(
metadata.real_page_table, pool_size
).contiguous()
if forward_mode.is_target_verify():
return dataclasses.replace(
metadata,
pooled_index_kpool=pool_size,
pooled_real_page_table=pooled_page_table,
)
pool_seqlens = torch.div(seqlens_32, pool_size, rounding_mode="floor").to(
torch.int32
)
pooled_page_table = build_pooled_page_table_64(
metadata.real_page_table, pool_size
).contiguous()
schedule = (
_compute_pool_schedule_metadata(
pool_seqlens,
@@ -625,10 +632,14 @@ def update_pooled_paged_mqa_metadata(
if (
not _is_kpool_layout_enabled(pool_size, real_page_size)
or not is_cuda()
or not forward_mode.is_decode_or_idle()
or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify())
):
return
if forward_mode.is_target_verify():
metadata.pooled_real_page_table.copy_(metadata.real_page_table[:, ::pool_size])
return
if (
metadata.pooled_index_kpool != pool_size
or metadata.pooled_cache_seqlens_int32 is None
@@ -1157,7 +1157,6 @@ class DFlashWorkerV2(BaseSpecWorker):
*,
batch_seq_lens_cpu: Optional[torch.Tensor],
nxt_kv_lens_cpu: Optional[torch.Tensor],
draft_prefix_lens: torch.Tensor,
out: torch.Tensor,
) -> None:
"""Fill the seq_lens_cpu planning bound, sync-free when a host-side
@@ -1168,8 +1167,11 @@ class DFlashWorkerV2(BaseSpecWorker):
elif nxt_kv_lens_cpu is not None:
self._compute_compact_draft_seq_lens_host(nxt_kv_lens_cpu, out=out)
else:
# Last resort: the legacy blocking D2H copy.
out.copy_(draft_prefix_lens)
# Compact device lengths never exceed the window plus one page.
out.fill_(
int(self.draft_window_size)
+ (self.page_size if self.page_size > 1 else 0)
)
def _rebuild_compact_draft_cache(
self,
@@ -2489,7 +2491,6 @@ class DFlashWorkerV2(BaseSpecWorker):
self._fill_compact_seq_lens_cpu_bound(
batch_seq_lens_cpu=batch.seq_lens_cpu,
nxt_kv_lens_cpu=draft_input.nxt_kv_lens_cpu,
draft_prefix_lens=draft_prefix_lens,
out=seq_lens_cpu,
)
self._rebuild_compact_draft_cache(
@@ -2517,8 +2518,10 @@ class DFlashWorkerV2(BaseSpecWorker):
seq_lens_cpu.copy_(draft_input.nxt_kv_lens_cpu)
draft_seq_lens_sum = int(draft_input.nxt_kv_lens_sum)
else:
seq_lens_cpu.copy_(prefix_lens.to("cpu", dtype=torch.int32))
draft_seq_lens_sum = int(prefix_lens.sum().item())
# Allocated lengths include the in-flight verify reservation.
for i, req in enumerate(batch.reqs):
seq_lens_cpu[i] = req.kv.kv_allocated_len
draft_seq_lens_sum = int(seq_lens_cpu.sum())
forward_batch = ForwardBatch(
forward_mode=ForwardMode.TARGET_VERIFY,
@@ -6,11 +6,14 @@ from unittest.mock import patch
import torch
from sglang.srt.layers.attention.dsa.dsa_indexer_kpool import IndexerKPool
from sglang.srt.layers.attention.dsa.kpool_fp8_index import build_pooled_page_table_64
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.dsa_metadata_kit import (
BS,
NEXT_N,
POOL,
ROUNDS,
addresses,
apply_metadata,
@@ -51,6 +54,23 @@ class TestDSAMetadataReplay(CustomTestCase):
self, fused.forward_metadata, ordinary.forward_metadata
)
self.assertEqual(pointers, addresses(fused.forward_metadata))
if mode.is_target_verify():
metadata = fused.forward_metadata
_, _, table, _ = IndexerKPool._get_kpool_decode_metadata(
SimpleNamespace(index_kpool=POOL),
SimpleNamespace(attn_metadata=metadata),
metadata.real_page_table,
metadata.dsa_seqlens_expanded,
64,
build_schedule_metadata=False,
)
self.assertIs(table, metadata.pooled_real_page_table)
torch.testing.assert_close(
table,
build_pooled_page_table_64(metadata.real_page_table, POOL),
rtol=0,
atol=0,
)
def test_precomputed_verify_retains_live_tail(self):
mode = ForwardMode.TARGET_VERIFY
@@ -47,6 +47,19 @@ class TestCompactSeqLensHostBound(CustomTestCase):
f"host bound under-shoots exact at window={window} page={page}",
)
def test_without_host_lengths_uses_window_bound(self):
from sglang.srt.speculative.dflash_worker_v2 import DFlashWorkerV2
seq = torch.arange(16385)
for page in (1, 64):
worker = SimpleNamespace(draft_window_size=2048, page_size=page)
bound = torch.empty(seq.numel(), dtype=torch.int32)
DFlashWorkerV2._fill_compact_seq_lens_cpu_bound(
worker, batch_seq_lens_cpu=None, nxt_kv_lens_cpu=None, out=bound
)
exact = _compact_lens_exact(seq, 2048, page)
self.assertTrue(bool((bound >= exact).all()))
def test_sawtooth_counterexample(self):
# exact(4160) = 4096 < exact(4100) = 4100 at window=4096 page=64:
# a host mirror of the exact math fed the reserved over-estimate