From 1e1bdeedbbe670ea6938afa7bdac67a54d4ff800 Mon Sep 17 00:00:00 2001 From: leonardHONG <2695316095@qq.com> Date: Tue, 6 Oct 2026 12:05:37 +0800 Subject: [PATCH] [Perf] Improve GLM-5.3-Flash DFlash decoding on B300 (#42489) Co-authored-by: lianghougong Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../ops/attention/fla/fused_norm_gate.py | 14 +++- .../fla/fused_sigmoid_gating_recurrent.py | 48 +++++++++++- .../layers/attention/dsa/dsa_backend_kpool.py | 76 +++++++++---------- .../layers/attention/dsa/dsa_indexer_kpool.py | 11 ++- .../srt/layers/attention/dsa/kpool_plan.py | 21 +++-- .../srt/speculative/dflash_worker_v2.py | 15 ++-- .../ops/attention/test_dsa_metadata_replay.py | 20 +++++ .../unit/spec/test_dflash_overlap_hostsync.py | 13 ++++ 8 files changed, 157 insertions(+), 61 deletions(-) diff --git a/python/sglang/kernels/ops/attention/fla/fused_norm_gate.py b/python/sglang/kernels/ops/attention/fla/fused_norm_gate.py index cdc3df1cc4..d13be4b877 100644 --- a/python/sglang/kernels/ops/attention/fla/fused_norm_gate.py +++ b/python/sglang/kernels/ops/attention/fla/fused_norm_gate.py @@ -7,7 +7,7 @@ import torch.nn as nn import triton import triton.language as tl -from sglang.kernels.jit.utils import is_arch_support_pdl +from sglang.kernels.jit.utils import get_jit_cuda_arch, is_arch_support_pdl from sglang.srt.utils import ( cdiv, cpu_has_amx_support, @@ -223,10 +223,16 @@ def layer_norm_gated_fwd( # heuristics for number of warps if D <= 512: + use_pdl = is_arch_support_pdl() BT = 32 - pdl_kwargs = ( - {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} - ) + use_bt8 = 8 <= T <= 256 if is_rms_norm and activation == "sigmoid" else T == 64 + if use_pdl and use_bt8 and (D, x.dtype) == (128, torch.bfloat16): + arch = get_jit_cuda_arch() + if (arch.major, arch.minor) == (10, 3): + # Small sigmoid-gated RMSNorm batches use eight rows per CTA. + # Other norm modes retain their T=64 selection. + BT = 8 + pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if use_pdl else {} layer_norm_gated_fwd_kernel[(cdiv(T, BT),)]( x=x, g=g, diff --git a/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py b/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py index 374182c437..03a628bcfb 100644 --- a/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py +++ b/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py @@ -5,11 +5,18 @@ import triton import triton.language as tl from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.srt.utils import is_gfx95_supported, is_hip, is_sm90_supported +from sglang.srt.utils import ( + get_device_sm, + is_cuda, + is_gfx95_supported, + is_hip, + is_sm90_supported, +) _is_hip = is_hip() _is_gfx95 = is_gfx95_supported() _is_sm90 = is_sm90_supported() +_is_sm103 = is_cuda() and get_device_sm() == 103 def _select_recurrent_launch_config( @@ -20,8 +27,10 @@ def _select_recurrent_launch_config( v: int, is_kda: bool, target_verify: bool = False, + *, + cache_steps: int = 0, ) -> tuple[int, int]: - """Select the value tile and warp count for recurrent GDN.""" + """Select the value tile and warp count for recurrent GDN/KDA.""" if ( _is_hip and _is_gfx95 @@ -46,6 +55,14 @@ def _select_recurrent_launch_config( # with narrow tiles but not bit-identical to BV=32. Only the dense # intermediate-state verify sets target_verify; cache_ring keeps BV=32. return 4, 1 + if ( + _is_sm103 + and is_kda + and target_verify + and (n, h, hv, k, v, cache_steps) == (1, 8, 8, 128, 128, 8) + ): + # GLM TP8 DFlash verification: 256 CTAs instead of 32. + return 4, 1 return min(triton.next_power_of_2(v), 32), 1 @@ -448,6 +465,16 @@ def fused_sigmoid_gating_delta_rule_update_kernel( # Update hidden state: h += k[:, None] * v[None, :] if MATCH_CUTEDSL_DECODE: b_h = tl.fma(b_k[:, None], b_v[None, :], b_h) + elif IS_KDA and V == 128 and BV == 4: + # The narrow KDA tile keeps the separate FP32 multiply/add rounding. + b_h = tl.inline_asm_elementwise( + "add.rn.f32 $0, $1, $2;", + constraints="=f,f,f", + args=[b_h, b_k[:, None] * b_v[None, :]], + dtype=tl.float32, + is_pure=True, + pack=1, + ) else: b_h += b_k[:, None] * b_v[None, :] @@ -567,8 +594,21 @@ def fused_sigmoid_gating_delta_rule_update( stride_a = a.stride()[1] if a.ndim == 4 else a.stride()[-2] HV = v.shape[2] N = B if cu_seqlens is None else len(cu_seqlens) - 1 + use_pdl = is_arch_support_pdl() + target_verify = ( + disable_state_update and not match_cutedsl_decode + if is_kda + else intermediate_states_buffer is not None + ) BV, num_warps = _select_recurrent_launch_config( - N, H, HV, K, V, is_kda, target_verify=intermediate_states_buffer is not None + N, + H, + HV, + K, + V, + is_kda, + target_verify=target_verify, + cache_steps=cache_steps, ) BK = triton.next_power_of_2(K) NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV) @@ -630,7 +670,7 @@ def fused_sigmoid_gating_delta_rule_update( # PDL (sm90+): chain this kernel behind its producer conv1d_update, which # already launches dependents. Bit-exact (scheduling only) — benefits both # KDA and GDN recurrent paths. - pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} + pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if use_pdl else {} # CuTeDSL specializes on the persisted state dtype. Its BF16 decode path # writes BF16 after every token and reloads FP32 for the next recurrence; diff --git a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py index 236a00c59a..5cc0f907db 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py @@ -102,16 +102,17 @@ class DeepseekSparseAttnBackendKPoolMixin: full_seqlens_expanded=kpool_inputs.full_seqlens_expanded, ) + metadata = init_pooled_paged_mqa_metadata( + metadata, + metadata.cache_seqlens_int32, + forward_mode, + pool_size=self.dsa_index_kpool, + real_page_size=self.real_page_size, + slots_per_page=slots_per_page, + build_schedule_metadata=build_schedule_metadata, + ) + if forward_mode.is_decode_or_idle(): - metadata = init_pooled_paged_mqa_metadata( - metadata, - metadata.cache_seqlens_int32, - forward_mode, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, - slots_per_page=slots_per_page, - build_schedule_metadata=build_schedule_metadata, - ) return init_kpool_write_plan( metadata, forward_batch, @@ -170,16 +171,15 @@ class DeepseekSparseAttnBackendKPoolMixin: slots_per_page = self._kpool_slots_per_page() build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata() - if forward_mode.is_decode_or_idle(): - metadata = init_pooled_paged_mqa_metadata( - metadata, - metadata.cache_seqlens_int32, - forward_mode, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, - slots_per_page=slots_per_page, - build_schedule_metadata=build_schedule_metadata, - ) + metadata = init_pooled_paged_mqa_metadata( + metadata, + metadata.cache_seqlens_int32, + forward_mode, + pool_size=self.dsa_index_kpool, + real_page_size=self.real_page_size, + slots_per_page=slots_per_page, + build_schedule_metadata=build_schedule_metadata, + ) if ( forward_mode.is_decode_or_idle() @@ -218,16 +218,15 @@ class DeepseekSparseAttnBackendKPoolMixin: slots_per_page = self._kpool_slots_per_page() build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata() - if forward_mode.is_decode_or_idle(): - update_pooled_paged_mqa_metadata( - metadata, - metadata.cache_seqlens_int32, - forward_mode, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, - slots_per_page=slots_per_page, - build_schedule_metadata=build_schedule_metadata, - ) + update_pooled_paged_mqa_metadata( + metadata, + metadata.cache_seqlens_int32, + forward_mode, + pool_size=self.dsa_index_kpool, + real_page_size=self.real_page_size, + slots_per_page=slots_per_page, + build_schedule_metadata=build_schedule_metadata, + ) if not ( forward_mode.is_decode_or_idle() @@ -270,16 +269,15 @@ class DeepseekSparseAttnBackendKPoolMixin: slots_per_page = self._kpool_slots_per_page() build_schedule_metadata = self._build_kpool_paged_mqa_schedule_metadata() - if forward_mode.is_decode_or_idle(): - update_pooled_paged_mqa_metadata( - metadata, - precomputed.cache_seqlens, - forward_mode, - pool_size=self.dsa_index_kpool, - real_page_size=self.real_page_size, - slots_per_page=slots_per_page, - build_schedule_metadata=build_schedule_metadata, - ) + update_pooled_paged_mqa_metadata( + metadata, + precomputed.cache_seqlens, + forward_mode, + pool_size=self.dsa_index_kpool, + real_page_size=self.real_page_size, + slots_per_page=slots_per_page, + build_schedule_metadata=build_schedule_metadata, + ) if not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()): return diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py index 8e4d29cbe2..a2259755ad 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py @@ -805,9 +805,14 @@ class IndexerKPool(MultiPlatformOp): if plan is not None and plan.pool_seqlens_per_q is not None: pool_seqlens = plan.pool_seqlens_per_q[: seqlens_32.shape[0]] pool_context_lens = pool_seqlens.contiguous().view(-1, 1) - pool_block_tables = build_pooled_page_table_64( - block_tables, self.index_kpool - ).contiguous() + pool_block_tables = attn_metadata.pooled_real_page_table + if ( + pool_block_tables is None + or attn_metadata.pooled_index_kpool != self.index_kpool + ): + pool_block_tables = build_pooled_page_table_64( + block_tables, self.index_kpool + ).contiguous() pool_schedule_metadata = plan.pool_schedule_metadata if pool_schedule_metadata is None and build_schedule_metadata: pool_schedule_metadata = deep_gemm.get_paged_mqa_logits_metadata( diff --git a/python/sglang/srt/layers/attention/dsa/kpool_plan.py b/python/sglang/srt/layers/attention/dsa/kpool_plan.py index d367d08962..f9b088243f 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_plan.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_plan.py @@ -585,16 +585,23 @@ def init_pooled_paged_mqa_metadata( if ( not _is_kpool_layout_enabled(pool_size, real_page_size) or not is_cuda() - or not forward_mode.is_decode_or_idle() + or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()) ): return metadata + pooled_page_table = build_pooled_page_table_64( + metadata.real_page_table, pool_size + ).contiguous() + if forward_mode.is_target_verify(): + return dataclasses.replace( + metadata, + pooled_index_kpool=pool_size, + pooled_real_page_table=pooled_page_table, + ) + pool_seqlens = torch.div(seqlens_32, pool_size, rounding_mode="floor").to( torch.int32 ) - pooled_page_table = build_pooled_page_table_64( - metadata.real_page_table, pool_size - ).contiguous() schedule = ( _compute_pool_schedule_metadata( pool_seqlens, @@ -625,10 +632,14 @@ def update_pooled_paged_mqa_metadata( if ( not _is_kpool_layout_enabled(pool_size, real_page_size) or not is_cuda() - or not forward_mode.is_decode_or_idle() + or not (forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()) ): return + if forward_mode.is_target_verify(): + metadata.pooled_real_page_table.copy_(metadata.real_page_table[:, ::pool_size]) + return + if ( metadata.pooled_index_kpool != pool_size or metadata.pooled_cache_seqlens_int32 is None diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index f48870c1c7..ff123e8945 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -1157,7 +1157,6 @@ class DFlashWorkerV2(BaseSpecWorker): *, batch_seq_lens_cpu: Optional[torch.Tensor], nxt_kv_lens_cpu: Optional[torch.Tensor], - draft_prefix_lens: torch.Tensor, out: torch.Tensor, ) -> None: """Fill the seq_lens_cpu planning bound, sync-free when a host-side @@ -1168,8 +1167,11 @@ class DFlashWorkerV2(BaseSpecWorker): elif nxt_kv_lens_cpu is not None: self._compute_compact_draft_seq_lens_host(nxt_kv_lens_cpu, out=out) else: - # Last resort: the legacy blocking D2H copy. - out.copy_(draft_prefix_lens) + # Compact device lengths never exceed the window plus one page. + out.fill_( + int(self.draft_window_size) + + (self.page_size if self.page_size > 1 else 0) + ) def _rebuild_compact_draft_cache( self, @@ -2489,7 +2491,6 @@ class DFlashWorkerV2(BaseSpecWorker): self._fill_compact_seq_lens_cpu_bound( batch_seq_lens_cpu=batch.seq_lens_cpu, nxt_kv_lens_cpu=draft_input.nxt_kv_lens_cpu, - draft_prefix_lens=draft_prefix_lens, out=seq_lens_cpu, ) self._rebuild_compact_draft_cache( @@ -2517,8 +2518,10 @@ class DFlashWorkerV2(BaseSpecWorker): seq_lens_cpu.copy_(draft_input.nxt_kv_lens_cpu) draft_seq_lens_sum = int(draft_input.nxt_kv_lens_sum) else: - seq_lens_cpu.copy_(prefix_lens.to("cpu", dtype=torch.int32)) - draft_seq_lens_sum = int(prefix_lens.sum().item()) + # Allocated lengths include the in-flight verify reservation. + for i, req in enumerate(batch.reqs): + seq_lens_cpu[i] = req.kv.kv_allocated_len + draft_seq_lens_sum = int(seq_lens_cpu.sum()) forward_batch = ForwardBatch( forward_mode=ForwardMode.TARGET_VERIFY, diff --git a/test/registered/kernels/ops/attention/test_dsa_metadata_replay.py b/test/registered/kernels/ops/attention/test_dsa_metadata_replay.py index e47b7aefa7..8e203cb1f0 100644 --- a/test/registered/kernels/ops/attention/test_dsa_metadata_replay.py +++ b/test/registered/kernels/ops/attention/test_dsa_metadata_replay.py @@ -6,11 +6,14 @@ from unittest.mock import patch import torch +from sglang.srt.layers.attention.dsa.dsa_indexer_kpool import IndexerKPool +from sglang.srt.layers.attention.dsa.kpool_fp8_index import build_pooled_page_table_64 from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.dsa_metadata_kit import ( BS, NEXT_N, + POOL, ROUNDS, addresses, apply_metadata, @@ -51,6 +54,23 @@ class TestDSAMetadataReplay(CustomTestCase): self, fused.forward_metadata, ordinary.forward_metadata ) self.assertEqual(pointers, addresses(fused.forward_metadata)) + if mode.is_target_verify(): + metadata = fused.forward_metadata + _, _, table, _ = IndexerKPool._get_kpool_decode_metadata( + SimpleNamespace(index_kpool=POOL), + SimpleNamespace(attn_metadata=metadata), + metadata.real_page_table, + metadata.dsa_seqlens_expanded, + 64, + build_schedule_metadata=False, + ) + self.assertIs(table, metadata.pooled_real_page_table) + torch.testing.assert_close( + table, + build_pooled_page_table_64(metadata.real_page_table, POOL), + rtol=0, + atol=0, + ) def test_precomputed_verify_retains_live_tail(self): mode = ForwardMode.TARGET_VERIFY diff --git a/test/registered/unit/spec/test_dflash_overlap_hostsync.py b/test/registered/unit/spec/test_dflash_overlap_hostsync.py index 809f3466a7..bc8c5c8618 100644 --- a/test/registered/unit/spec/test_dflash_overlap_hostsync.py +++ b/test/registered/unit/spec/test_dflash_overlap_hostsync.py @@ -47,6 +47,19 @@ class TestCompactSeqLensHostBound(CustomTestCase): f"host bound under-shoots exact at window={window} page={page}", ) + def test_without_host_lengths_uses_window_bound(self): + from sglang.srt.speculative.dflash_worker_v2 import DFlashWorkerV2 + + seq = torch.arange(16385) + for page in (1, 64): + worker = SimpleNamespace(draft_window_size=2048, page_size=page) + bound = torch.empty(seq.numel(), dtype=torch.int32) + DFlashWorkerV2._fill_compact_seq_lens_cpu_bound( + worker, batch_seq_lens_cpu=None, nxt_kv_lens_cpu=None, out=bound + ) + exact = _compact_lens_exact(seq, 2048, page) + self.assertTrue(bool((bound >= exact).all())) + def test_sawtooth_counterexample(self): # exact(4160) = 4096 < exact(4100) = 4100 at window=4096 page=64: # a host mirror of the exact math fed the reserved over-estimate