From cb3ba2f2b77b4812313f7b13da67759d2b5e3a52 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Singh <9626333+SKRohit@users.noreply.github.com> Date: Wed, 30 Sep 2026 09:35:35 +0530 Subject: [PATCH] [XPU] Add dynamic_expert_bias and track_state support (#39618) Co-authored-by: github-actions[bot] Co-authored-by: Singh Co-authored-by: Singh Co-authored-by: Ma Mingfei --- .../xpu/kernels/fla/chunk_delta_h.py | 50 +++- python/sglang/srt/layers/moe/topk.py | 2 + .../xpu/test_kda_track_state_xpu.py | 242 ++++++++++++++++++ 3 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 test/registered/xpu/test_kda_track_state_xpu.py diff --git a/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_delta_h.py b/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_delta_h.py index e3b7da63a4..0102ab4cad 100644 --- a/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_delta_h.py +++ b/python/sglang/srt/hardware_backend/xpu/kernels/fla/chunk_delta_h.py @@ -41,8 +41,12 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop( h, initial_state, initial_state_indices, + stride_init_state, cu_seqlens, chunk_offsets, + track_state, + track_chunk_idx, + stride_track_state, T, H: tl.constexpr, Hg: tl.constexpr, @@ -58,6 +62,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop( IS_VARLEN: tl.constexpr, NT_BUCKET: tl.constexpr, # this arg is kept to align with the triton kernel for CUDA USE_EXP2: tl.constexpr, + TRACK_STATE: tl.constexpr, ): i_v, i_nh = tl.program_id(0), tl.program_id(1) i_n, i_h = i_nh // H, i_nh % H @@ -112,17 +117,30 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop( block_shape=(BT, BV), ) - index = tl.load(initial_state_indices + i_n).to(tl.int32) + # Slot pitch comes from the caller (initial_state.stride(0)): the state pool + # may be an envelope-strided view (page-major / unified memory), where the + # per-slot pitch spans ALL layers' state, not H*V*K. int64: envelope pitches + # overflow an int32 index product. + index = tl.load(initial_state_indices + i_n).to(tl.int64) # Padded rows carry the -1 sentinel; without this guard the sentinel # reaches pointer arithmetic and addresses before the state pool. valid_state = index >= 0 - h0 = initial_state + index * stride_h - ht = initial_state + index * stride_h + h0 = initial_state + index * stride_init_state + ht = initial_state + index * stride_init_state if USE_INITIAL_STATE: h0 = h0 + i_h * V * K if INPLACE_UPDATE: ht = ht + i_h * V * K + if TRACK_STATE: + i_track = tl.load(track_chunk_idx + i_n).to(tl.int32) + p_track_base = track_state + (i_n * stride_track_state + i_h * V * K).to( + tl.int64 + ) + else: + i_track = -1 + p_track_base = track_state + # main recurrence — time is the outer loop for i_t in range(NT): ######################################################################## @@ -158,6 +176,14 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop( ) tl.store(p_ho, b_h.to(p_ho.dtype.element_ty), boundary_check=(0, 1)) + # Mid-sequence checkpoint for the mamba prefix cache: the same + # pre-update state, kept in fp32 so the caller rounds only once. + if TRACK_STATE and i_t == i_track: + p_track = tl.make_block_ptr( + p_track_base, (V, K), (K, 1), (i_v * BV, k_blk), (BV, 64), (1, 0) + ) + tl.store(p_track, b_h, boundary_check=(0, 1)) + # Accumulate correction: w_k @ h_k^T b_w = w_desc.load([i_t * BT, k_blk]) b_v_corr += tl.dot(b_w, tl.trans(b_h).to(b_w.dtype)) @@ -243,6 +269,8 @@ def chunk_gated_delta_rule_fwd_h( chunk_indices: Optional[torch.LongTensor] = None, use_exp2: bool = False, inplace_update: bool = True, + track_state: Optional[torch.Tensor] = None, + track_chunk_idx: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: if not inplace_update: raise NotImplementedError( @@ -251,6 +279,15 @@ def chunk_gated_delta_rule_fwd_h( assert not (use_exp2 and g is not None), ( "use_exp2 covers only the per-channel gk path; scalar g stays natural-exp" ) + assert (track_state is None) == (track_chunk_idx is None), ( + "track_state and track_chunk_idx must be passed together" + ) + if track_state is not None: + # The caller rounds once to the pool dtype; a narrower buffer would + # silently double-round the snapshot. + assert track_state.dtype == torch.float32, ( + f"track_state must be fp32, got {track_state.dtype}" + ) B, T, Hg, K, V = *k.shape, u.shape[-1] H = u.shape[-2] BT = CHUNK_SIZE @@ -287,8 +324,14 @@ def chunk_gated_delta_rule_fwd_h( h=h, initial_state=initial_state, initial_state_indices=initial_state_indices, + # Envelope-strided state pools (page-major / unified memory) have a + # per-slot pitch != H*V*K; contiguous pools pass exactly H*V*K. + stride_init_state=(initial_state.stride(0) if initial_state is not None else 0), cu_seqlens=cu_seqlens, chunk_offsets=chunk_offsets, + track_state=track_state, + track_chunk_idx=track_chunk_idx, + stride_track_state=(track_state.stride(0) if track_state is not None else 0), T=T, H=H, Hg=Hg, @@ -303,5 +346,6 @@ def chunk_gated_delta_rule_fwd_h( IS_VARLEN=cu_seqlens is not None, NT_BUCKET=(0 if NT <= 32 else (1 if NT <= 128 else 2)), USE_EXP2=use_exp2, + TRACK_STATE=track_state is not None, ) return h, v_new diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 8e72a5a8b7..c827500704 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -836,6 +836,7 @@ class TopK(BaseFusedOp): *, num_token_non_padded: Optional[torch.Tensor] = None, expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + dynamic_expert_bias: Optional[torch.Tensor] = None, ) -> TopKOutput: self.topk_config.torch_native = True # [NOTE] XPU device support for topk kernels @@ -852,6 +853,7 @@ class TopK(BaseFusedOp): topk_config=self.topk_config, num_token_non_padded=num_token_non_padded, expert_location_dispatch_info=expert_location_dispatch_info, + dynamic_expert_bias=dynamic_expert_bias, ) diff --git a/test/registered/xpu/test_kda_track_state_xpu.py b/test/registered/xpu/test_kda_track_state_xpu.py new file mode 100644 index 0000000000..9bcf6aa3ec --- /dev/null +++ b/test/registered/xpu/test_kda_track_state_xpu.py @@ -0,0 +1,242 @@ +import unittest + +import torch + +from sglang.kernels.ops.attention.fla.kda import chunk_kda +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import CustomTestCase + +register_xpu_ci(est_time=300, suite="stage-b-test-1-gpu-xpu") + +CHUNK_SIZE = 64 +# Gap between the pool's per-slot pitch and H*V*K. A kernel that assumes a +# contiguous pool addresses slot i at i*H*V*K, which lands in this gap. +ENVELOPE_PAD = 256 +PAD_SENTINEL = -12345.0 + + +def _make_varlen_inputs(seed, lens, num_heads=2, head_dim=128, device="xpu"): + """Packed varlen KDA inputs: [1, sum(lens), H, D] plus a zero fp32 state pool.""" + generator = torch.Generator().manual_seed(seed) + total = sum(lens) + + def randn(*shape, dtype=torch.bfloat16): + # CPU generator + transfer: torch.Generator(device="xpu") is not a + # portable construction across the XPU torch builds this suite runs on. + return torch.randn(*shape, generator=generator, dtype=torch.float32).to( + device=device, dtype=dtype + ) + + q = randn(1, total, num_heads, head_dim) + k = randn(1, total, num_heads, head_dim) + v = (0.1 * randn(1, total, num_heads, head_dim, dtype=torch.float32)).to( + torch.bfloat16 + ) + gate = randn(1, total, num_heads, head_dim) + beta = torch.sigmoid(randn(1, total, num_heads, dtype=torch.float32)).to( + torch.bfloat16 + ) + a_log = randn(num_heads, dtype=torch.float32) + dt_bias = randn(num_heads * head_dim, dtype=torch.float32) + state = torch.zeros( + len(lens), num_heads, head_dim, head_dim, device=device, dtype=torch.float32 + ) + cu_seqlens = torch.tensor( + [0, *torch.tensor(lens).cumsum(0).tolist()], dtype=torch.int32, device=device + ) + return q, k, v, gate, beta, a_log, dt_bias, state, cu_seqlens + + +def _run_chunk_kda(q, k, v, gate, beta, a_log, dt_bias, state, cu_seqlens, **kwargs): + return chunk_kda( + # chunk_kda writes in place (the attention output lands in v, the gate + # cumsum in g); hand every run fresh copies so runs stay independent. + q=q.clone(), + k=k.clone(), + v=v.clone(), + g=gate.clone(), + beta=beta.clone(), + scale=q.shape[-1] ** -0.5, + initial_state=state, + initial_state_indices=torch.arange( + state.shape[0], device=state.device, dtype=torch.int32 + ), + use_qk_l2norm_in_kernel=True, + cu_seqlens=cu_seqlens, + A_log=a_log, + dt_bias=dt_bias, + lower_bound=-5.0, + **kwargs, + ) + + +@unittest.skipUnless( + torch.xpu.is_available(), + "Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestKdaTrackStateXpu(CustomTestCase): + @torch.inference_mode() + def test_track_state_snapshots_fp32_accumulator(self): + """Bug regression: the XPU chunk kernel had no track path at all, so the + mamba radix prefix cache could not snapshot the SSM state at the last + chunk boundary of an unaligned sequence. `track_state` must carry the + in-kernel fp32 accumulator: identical to the fp32 final state of a run + truncated at the boundary, and strictly more precise than the bf16 `h` + row for the same boundary. + """ + # seq0: 100 tokens, unaligned -> snapshot at the 64-token boundary + # (start of chunk 1). seq1: 64 tokens, aligned -> not tracked. + lens = [100, 64] + q, k, v, gate, beta, a_log, dt_bias, state, cu_seqlens = _make_varlen_inputs( + 0, lens + ) + num_heads, head_dim = q.shape[2], q.shape[3] + + track_state = torch.full( + (len(lens), num_heads, head_dim, head_dim), + float("nan"), + device=state.device, + dtype=torch.float32, + ) + track_chunk_idx = torch.tensor([1, -1], dtype=torch.int32, device=state.device) + _, h = _run_chunk_kda( + q, + k, + v, + gate, + beta, + a_log, + dt_bias, + state, + cu_seqlens, + output_intermediate_states=True, + track_state=track_state, + track_chunk_idx=track_chunk_idx, + ) + + # The untracked row must stay untouched; the tracked row must be finite. + self.assertTrue(torch.all(torch.isnan(track_state[1]))) + self.assertFalse(torch.any(torch.isnan(track_state[0]))) + + # Reference: truncate seq0 at the boundary; the pool's fp32 row then + # receives the in-place final state for the same prefix -- the + # established fp32 path the snapshot must agree with. + ref_state = torch.zeros( + 1, num_heads, head_dim, head_dim, device=state.device, dtype=torch.float32 + ) + ref_cu_seqlens = torch.tensor( + [0, CHUNK_SIZE], dtype=torch.int32, device=state.device + ) + _run_chunk_kda( + q[:, :CHUNK_SIZE], + k[:, :CHUNK_SIZE], + v[:, :CHUNK_SIZE], + gate[:, :CHUNK_SIZE], + beta[:, :CHUNK_SIZE], + a_log, + dt_bias, + ref_state, + ref_cu_seqlens, + ) + torch.testing.assert_close(track_state[0], ref_state[0], rtol=1e-5, atol=1e-5) + + # The guard: h packs one row per (seq, chunk); row 1 is seq0's state at + # the boundary, rounded to bf16. If the snapshot were re-routed through + # h, it could not match the fp32 reference above. + self.assertTrue( + torch.equal(h[0, 1].float(), track_state[0].to(torch.bfloat16).float()), + "h row should be exactly the bf16 rounding of the fp32 snapshot", + ) + self.assertFalse( + torch.equal(track_state[0], track_state[0].to(torch.bfloat16).float()), + "test inputs must make bf16 rounding lossy", + ) + + @torch.inference_mode() + def test_envelope_strided_initial_state_pool(self): + """Bug regression: the XPU chunk kernel addressed a state slot as + index*H*V*K, so a pool whose stride(0) spans all layers (page-major / + unified memory) had every slot but the first read and written at the + wrong offset. Slot pitch must come from initial_state.stride(0): results + match a contiguous pool, and no write leaves a slot. + """ + lens = [100, 64] + q, k, v, gate, beta, a_log, dt_bias, state, cu_seqlens = _make_varlen_inputs( + 0, lens + ) + num_slots, num_heads, v_dim, k_dim = state.shape + slot_elems = num_heads * v_dim * k_dim + pitch = slot_elems + ENVELOPE_PAD + + # Distinct non-zero content per slot: against an all-zero pool, reading + # the wrong slot returns zeros either way and the bug stays invisible. + generator = torch.Generator().manual_seed(7) + base = ( + 0.1 + * torch.randn( + num_slots, + num_heads, + v_dim, + k_dim, + generator=generator, + dtype=torch.float32, + ) + ).to(state.device) + track_chunk_idx = torch.tensor([1, -1], dtype=torch.int32, device=state.device) + + def run(pool): + track = torch.full( + (num_slots, num_heads, v_dim, k_dim), + float("nan"), + device=state.device, + dtype=torch.float32, + ) + out, _ = _run_chunk_kda( + q, + k, + v, + gate, + beta, + a_log, + dt_bias, + pool, + cu_seqlens, + output_intermediate_states=True, + track_state=track, + track_chunk_idx=track_chunk_idx, + ) + return out, track + + contiguous_pool = base.clone() + self.assertEqual(contiguous_pool.stride(0), slot_elems) + expected_out, expected_track = run(contiguous_pool) + + envelope = torch.full( + (num_slots, pitch), PAD_SENTINEL, device=state.device, dtype=torch.float32 + ) + strided_pool = torch.as_strided( + envelope, + (num_slots, num_heads, v_dim, k_dim), + (pitch, v_dim * k_dim, k_dim, 1), + ) + strided_pool.copy_(base) + self.assertEqual(strided_pool.stride(0), pitch) + actual_out, actual_track = run(strided_pool) + + torch.testing.assert_close( + actual_out.float(), expected_out.float(), rtol=1e-2, atol=1e-2 + ) + # INPLACE_UPDATE: the pool holds the final state, so this covers the + # write path as well as the read. + torch.testing.assert_close(strided_pool, contiguous_pool, rtol=1e-5, atol=1e-5) + torch.testing.assert_close( + actual_track, expected_track, rtol=1e-5, atol=1e-5, equal_nan=True + ) + self.assertTrue( + torch.all(envelope[:, slot_elems:] == PAD_SENTINEL), + "kernel wrote past a pool slot", + ) + + +if __name__ == "__main__": + unittest.main()