[Perf] Improve GLM-5.3-Flash DFlash decoding on B300 (#42489)
Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
lianghougong
Xiaoyu Zhang
parent
fcbbefa82a
commit
1e1bdeedbb
@@ -7,7 +7,7 @@ import torch.nn as nn
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.kernels.jit.utils import get_jit_cuda_arch, is_arch_support_pdl
|
||||
from sglang.srt.utils import (
|
||||
cdiv,
|
||||
cpu_has_amx_support,
|
||||
@@ -223,10 +223,16 @@ def layer_norm_gated_fwd(
|
||||
# heuristics for number of warps
|
||||
|
||||
if D <= 512:
|
||||
use_pdl = is_arch_support_pdl()
|
||||
BT = 32
|
||||
pdl_kwargs = (
|
||||
{"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {}
|
||||
)
|
||||
use_bt8 = 8 <= T <= 256 if is_rms_norm and activation == "sigmoid" else T == 64
|
||||
if use_pdl and use_bt8 and (D, x.dtype) == (128, torch.bfloat16):
|
||||
arch = get_jit_cuda_arch()
|
||||
if (arch.major, arch.minor) == (10, 3):
|
||||
# Small sigmoid-gated RMSNorm batches use eight rows per CTA.
|
||||
# Other norm modes retain their T=64 selection.
|
||||
BT = 8
|
||||
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if use_pdl else {}
|
||||
layer_norm_gated_fwd_kernel[(cdiv(T, BT),)](
|
||||
x=x,
|
||||
g=g,
|
||||
|
||||
@@ -5,11 +5,18 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip, is_sm90_supported
|
||||
from sglang.srt.utils import (
|
||||
get_device_sm,
|
||||
is_cuda,
|
||||
is_gfx95_supported,
|
||||
is_hip,
|
||||
is_sm90_supported,
|
||||
)
|
||||
|
||||
_is_hip = is_hip()
|
||||
_is_gfx95 = is_gfx95_supported()
|
||||
_is_sm90 = is_sm90_supported()
|
||||
_is_sm103 = is_cuda() and get_device_sm() == 103
|
||||
|
||||
|
||||
def _select_recurrent_launch_config(
|
||||
@@ -20,8 +27,10 @@ def _select_recurrent_launch_config(
|
||||
v: int,
|
||||
is_kda: bool,
|
||||
target_verify: bool = False,
|
||||
*,
|
||||
cache_steps: int = 0,
|
||||
) -> tuple[int, int]:
|
||||
"""Select the value tile and warp count for recurrent GDN."""
|
||||
"""Select the value tile and warp count for recurrent GDN/KDA."""
|
||||
if (
|
||||
_is_hip
|
||||
and _is_gfx95
|
||||
@@ -46,6 +55,14 @@ def _select_recurrent_launch_config(
|
||||
# with narrow tiles but not bit-identical to BV=32. Only the dense
|
||||
# intermediate-state verify sets target_verify; cache_ring keeps BV=32.
|
||||
return 4, 1
|
||||
if (
|
||||
_is_sm103
|
||||
and is_kda
|
||||
and target_verify
|
||||
and (n, h, hv, k, v, cache_steps) == (1, 8, 8, 128, 128, 8)
|
||||
):
|
||||
# GLM TP8 DFlash verification: 256 CTAs instead of 32.
|
||||
return 4, 1
|
||||
return min(triton.next_power_of_2(v), 32), 1
|
||||
|
||||
|
||||
@@ -448,6 +465,16 @@ def fused_sigmoid_gating_delta_rule_update_kernel(
|
||||
# Update hidden state: h += k[:, None] * v[None, :]
|
||||
if MATCH_CUTEDSL_DECODE:
|
||||
b_h = tl.fma(b_k[:, None], b_v[None, :], b_h)
|
||||
elif IS_KDA and V == 128 and BV == 4:
|
||||
# The narrow KDA tile keeps the separate FP32 multiply/add rounding.
|
||||
b_h = tl.inline_asm_elementwise(
|
||||
"add.rn.f32 $0, $1, $2;",
|
||||
constraints="=f,f,f",
|
||||
args=[b_h, b_k[:, None] * b_v[None, :]],
|
||||
dtype=tl.float32,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
else:
|
||||
b_h += b_k[:, None] * b_v[None, :]
|
||||
|
||||
@@ -567,8 +594,21 @@ def fused_sigmoid_gating_delta_rule_update(
|
||||
stride_a = a.stride()[1] if a.ndim == 4 else a.stride()[-2]
|
||||
HV = v.shape[2]
|
||||
N = B if cu_seqlens is None else len(cu_seqlens) - 1
|
||||
use_pdl = is_arch_support_pdl()
|
||||
target_verify = (
|
||||
disable_state_update and not match_cutedsl_decode
|
||||
if is_kda
|
||||
else intermediate_states_buffer is not None
|
||||
)
|
||||
BV, num_warps = _select_recurrent_launch_config(
|
||||
N, H, HV, K, V, is_kda, target_verify=intermediate_states_buffer is not None
|
||||
N,
|
||||
H,
|
||||
HV,
|
||||
K,
|
||||
V,
|
||||
is_kda,
|
||||
target_verify=target_verify,
|
||||
cache_steps=cache_steps,
|
||||
)
|
||||
BK = triton.next_power_of_2(K)
|
||||
NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV)
|
||||
@@ -630,7 +670,7 @@ def fused_sigmoid_gating_delta_rule_update(
|
||||
# PDL (sm90+): chain this kernel behind its producer conv1d_update, which
|
||||
# already launches dependents. Bit-exact (scheduling only) — benefits both
|
||||
# KDA and GDN recurrent paths.
|
||||
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {}
|
||||
pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if use_pdl else {}
|
||||
|
||||
# CuTeDSL specializes on the persisted state dtype. Its BF16 decode path
|
||||
# writes BF16 after every token and reloads FP32 for the next recurrence;
|
||||
|
||||
@@ -102,16 +102,17 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
full_seqlens_expanded=kpool_inputs.full_seqlens_expanded,
|
||||
)
|
||||
|
||||
metadata = init_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
metadata.cache_seqlens_int32,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
if forward_mode.is_decode_or_idle():
|
||||
metadata = init_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
metadata.cache_seqlens_int32,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
return init_kpool_write_plan(
|
||||
metadata,
|
||||
forward_batch,
|
||||
@@ -170,16 +171,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
|
||||
slots_per_page = self._kpool_slots_per_page()
|
||||
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
|
||||
if forward_mode.is_decode_or_idle():
|
||||
metadata = init_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
metadata.cache_seqlens_int32,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
metadata = init_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
metadata.cache_seqlens_int32,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
if (
|
||||
forward_mode.is_decode_or_idle()
|
||||
@@ -218,16 +218,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
|
||||
slots_per_page = self._kpool_slots_per_page()
|
||||
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
|
||||
if forward_mode.is_decode_or_idle():
|
||||
update_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
metadata.cache_seqlens_int32,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
update_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
metadata.cache_seqlens_int32,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
if not (
|
||||
forward_mode.is_decode_or_idle()
|
||||
@@ -270,16 +269,15 @@ class DeepseekSparseAttnBackendKPoolMixin:
|
||||
|
||||
slots_per_page = self._kpool_slots_per_page()
|
||||
build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata()
|
||||
if forward_mode.is_decode_or_idle():
|
||||
update_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
precomputed.cache_seqlens,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
update_pooled_paged_mqa_metadata(
|
||||
metadata,
|
||||
precomputed.cache_seqlens,
|
||||
forward_mode,
|
||||
pool_size=self.dsa_index_kpool,
|
||||
real_page_size=self.real_page_size,
|
||||
slots_per_page=slots_per_page,
|
||||
build_schedule_metadata=build_schedule_metadata,
|
||||
)
|
||||
|
||||
if not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()):
|
||||
return
|
||||
|
||||
@@ -805,9 +805,14 @@ class IndexerKPool(MultiPlatformOp):
|
||||
if plan is not None and plan.pool_seqlens_per_q is not None:
|
||||
pool_seqlens = plan.pool_seqlens_per_q[: seqlens_32.shape[0]]
|
||||
pool_context_lens = pool_seqlens.contiguous().view(-1, 1)
|
||||
pool_block_tables = build_pooled_page_table_64(
|
||||
block_tables, self.index_kpool
|
||||
).contiguous()
|
||||
pool_block_tables = attn_metadata.pooled_real_page_table
|
||||
if (
|
||||
pool_block_tables is None
|
||||
or attn_metadata.pooled_index_kpool != self.index_kpool
|
||||
):
|
||||
pool_block_tables = build_pooled_page_table_64(
|
||||
block_tables, self.index_kpool
|
||||
).contiguous()
|
||||
pool_schedule_metadata = plan.pool_schedule_metadata
|
||||
if pool_schedule_metadata is None and build_schedule_metadata:
|
||||
pool_schedule_metadata = deep_gemm.get_paged_mqa_logits_metadata(
|
||||
|
||||
@@ -585,16 +585,23 @@ def init_pooled_paged_mqa_metadata(
|
||||
if (
|
||||
not _is_kpool_layout_enabled(pool_size, real_page_size)
|
||||
or not is_cuda()
|
||||
or not forward_mode.is_decode_or_idle()
|
||||
or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify())
|
||||
):
|
||||
return metadata
|
||||
|
||||
pooled_page_table = build_pooled_page_table_64(
|
||||
metadata.real_page_table, pool_size
|
||||
).contiguous()
|
||||
if forward_mode.is_target_verify():
|
||||
return dataclasses.replace(
|
||||
metadata,
|
||||
pooled_index_kpool=pool_size,
|
||||
pooled_real_page_table=pooled_page_table,
|
||||
)
|
||||
|
||||
pool_seqlens = torch.div(seqlens_32, pool_size, rounding_mode="floor").to(
|
||||
torch.int32
|
||||
)
|
||||
pooled_page_table = build_pooled_page_table_64(
|
||||
metadata.real_page_table, pool_size
|
||||
).contiguous()
|
||||
schedule = (
|
||||
_compute_pool_schedule_metadata(
|
||||
pool_seqlens,
|
||||
@@ -625,10 +632,14 @@ def update_pooled_paged_mqa_metadata(
|
||||
if (
|
||||
not _is_kpool_layout_enabled(pool_size, real_page_size)
|
||||
or not is_cuda()
|
||||
or not forward_mode.is_decode_or_idle()
|
||||
or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify())
|
||||
):
|
||||
return
|
||||
|
||||
if forward_mode.is_target_verify():
|
||||
metadata.pooled_real_page_table.copy_(metadata.real_page_table[:, ::pool_size])
|
||||
return
|
||||
|
||||
if (
|
||||
metadata.pooled_index_kpool != pool_size
|
||||
or metadata.pooled_cache_seqlens_int32 is None
|
||||
|
||||
@@ -1157,7 +1157,6 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
*,
|
||||
batch_seq_lens_cpu: Optional[torch.Tensor],
|
||||
nxt_kv_lens_cpu: Optional[torch.Tensor],
|
||||
draft_prefix_lens: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""Fill the seq_lens_cpu planning bound, sync-free when a host-side
|
||||
@@ -1168,8 +1167,11 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
elif nxt_kv_lens_cpu is not None:
|
||||
self._compute_compact_draft_seq_lens_host(nxt_kv_lens_cpu, out=out)
|
||||
else:
|
||||
# Last resort: the legacy blocking D2H copy.
|
||||
out.copy_(draft_prefix_lens)
|
||||
# Compact device lengths never exceed the window plus one page.
|
||||
out.fill_(
|
||||
int(self.draft_window_size)
|
||||
+ (self.page_size if self.page_size > 1 else 0)
|
||||
)
|
||||
|
||||
def _rebuild_compact_draft_cache(
|
||||
self,
|
||||
@@ -2489,7 +2491,6 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._fill_compact_seq_lens_cpu_bound(
|
||||
batch_seq_lens_cpu=batch.seq_lens_cpu,
|
||||
nxt_kv_lens_cpu=draft_input.nxt_kv_lens_cpu,
|
||||
draft_prefix_lens=draft_prefix_lens,
|
||||
out=seq_lens_cpu,
|
||||
)
|
||||
self._rebuild_compact_draft_cache(
|
||||
@@ -2517,8 +2518,10 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
seq_lens_cpu.copy_(draft_input.nxt_kv_lens_cpu)
|
||||
draft_seq_lens_sum = int(draft_input.nxt_kv_lens_sum)
|
||||
else:
|
||||
seq_lens_cpu.copy_(prefix_lens.to("cpu", dtype=torch.int32))
|
||||
draft_seq_lens_sum = int(prefix_lens.sum().item())
|
||||
# Allocated lengths include the in-flight verify reservation.
|
||||
for i, req in enumerate(batch.reqs):
|
||||
seq_lens_cpu[i] = req.kv.kv_allocated_len
|
||||
draft_seq_lens_sum = int(seq_lens_cpu.sum())
|
||||
|
||||
forward_batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.TARGET_VERIFY,
|
||||
|
||||
@@ -6,11 +6,14 @@ from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer_kpool import IndexerKPool
|
||||
from sglang.srt.layers.attention.dsa.kpool_fp8_index import build_pooled_page_table_64
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.dsa_metadata_kit import (
|
||||
BS,
|
||||
NEXT_N,
|
||||
POOL,
|
||||
ROUNDS,
|
||||
addresses,
|
||||
apply_metadata,
|
||||
@@ -51,6 +54,23 @@ class TestDSAMetadataReplay(CustomTestCase):
|
||||
self, fused.forward_metadata, ordinary.forward_metadata
|
||||
)
|
||||
self.assertEqual(pointers, addresses(fused.forward_metadata))
|
||||
if mode.is_target_verify():
|
||||
metadata = fused.forward_metadata
|
||||
_, _, table, _ = IndexerKPool._get_kpool_decode_metadata(
|
||||
SimpleNamespace(index_kpool=POOL),
|
||||
SimpleNamespace(attn_metadata=metadata),
|
||||
metadata.real_page_table,
|
||||
metadata.dsa_seqlens_expanded,
|
||||
64,
|
||||
build_schedule_metadata=False,
|
||||
)
|
||||
self.assertIs(table, metadata.pooled_real_page_table)
|
||||
torch.testing.assert_close(
|
||||
table,
|
||||
build_pooled_page_table_64(metadata.real_page_table, POOL),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
def test_precomputed_verify_retains_live_tail(self):
|
||||
mode = ForwardMode.TARGET_VERIFY
|
||||
|
||||
@@ -47,6 +47,19 @@ class TestCompactSeqLensHostBound(CustomTestCase):
|
||||
f"host bound under-shoots exact at window={window} page={page}",
|
||||
)
|
||||
|
||||
def test_without_host_lengths_uses_window_bound(self):
|
||||
from sglang.srt.speculative.dflash_worker_v2 import DFlashWorkerV2
|
||||
|
||||
seq = torch.arange(16385)
|
||||
for page in (1, 64):
|
||||
worker = SimpleNamespace(draft_window_size=2048, page_size=page)
|
||||
bound = torch.empty(seq.numel(), dtype=torch.int32)
|
||||
DFlashWorkerV2._fill_compact_seq_lens_cpu_bound(
|
||||
worker, batch_seq_lens_cpu=None, nxt_kv_lens_cpu=None, out=bound
|
||||
)
|
||||
exact = _compact_lens_exact(seq, 2048, page)
|
||||
self.assertTrue(bool((bound >= exact).all()))
|
||||
|
||||
def test_sawtooth_counterexample(self):
|
||||
# exact(4160) = 4096 < exact(4100) = 4100 at window=4096 page=64:
|
||||
# a host mirror of the exact math fed the reserved over-estimate
|
||||
|
||||
Reference in New Issue
Block a user