[XPU] Add dynamic_expert_bias and track_state support (#39618)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Singh <rohitsi2@iil-login.iind.intel.com>
Co-authored-by: Singh <rohitsi2@iil-gnrap01.iind.intel.com>
Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
This commit is contained in:
Rohit Kumar Singh
2026-09-30 12:05:35 +08:00
committed by GitHub
co-authored by github-actions[bot] Singh Singh Ma Mingfei
parent 8854857a99
commit cb3ba2f2b7
3 changed files with 291 additions and 3 deletions
@@ -41,8 +41,12 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop(
h,
initial_state,
initial_state_indices,
stride_init_state,
cu_seqlens,
chunk_offsets,
track_state,
track_chunk_idx,
stride_track_state,
T,
H: tl.constexpr,
Hg: tl.constexpr,
@@ -58,6 +62,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop(
IS_VARLEN: tl.constexpr,
NT_BUCKET: tl.constexpr, # this arg is kept to align with the triton kernel for CUDA
USE_EXP2: tl.constexpr,
TRACK_STATE: tl.constexpr,
):
i_v, i_nh = tl.program_id(0), tl.program_id(1)
i_n, i_h = i_nh // H, i_nh % H
@@ -112,17 +117,30 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop(
block_shape=(BT, BV),
)
index = tl.load(initial_state_indices + i_n).to(tl.int32)
# Slot pitch comes from the caller (initial_state.stride(0)): the state pool
# may be an envelope-strided view (page-major / unified memory), where the
# per-slot pitch spans ALL layers' state, not H*V*K. int64: envelope pitches
# overflow an int32 index product.
index = tl.load(initial_state_indices + i_n).to(tl.int64)
# Padded rows carry the -1 sentinel; without this guard the sentinel
# reaches pointer arithmetic and addresses before the state pool.
valid_state = index >= 0
h0 = initial_state + index * stride_h
ht = initial_state + index * stride_h
h0 = initial_state + index * stride_init_state
ht = initial_state + index * stride_init_state
if USE_INITIAL_STATE:
h0 = h0 + i_h * V * K
if INPLACE_UPDATE:
ht = ht + i_h * V * K
if TRACK_STATE:
i_track = tl.load(track_chunk_idx + i_n).to(tl.int32)
p_track_base = track_state + (i_n * stride_track_state + i_h * V * K).to(
tl.int64
)
else:
i_track = -1
p_track_base = track_state
# main recurrence — time is the outer loop
for i_t in range(NT):
########################################################################
@@ -158,6 +176,14 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop(
)
tl.store(p_ho, b_h.to(p_ho.dtype.element_ty), boundary_check=(0, 1))
# Mid-sequence checkpoint for the mamba prefix cache: the same
# pre-update state, kept in fp32 so the caller rounds only once.
if TRACK_STATE and i_t == i_track:
p_track = tl.make_block_ptr(
p_track_base, (V, K), (K, 1), (i_v * BV, k_blk), (BV, 64), (1, 0)
)
tl.store(p_track, b_h, boundary_check=(0, 1))
# Accumulate correction: w_k @ h_k^T
b_w = w_desc.load([i_t * BT, k_blk])
b_v_corr += tl.dot(b_w, tl.trans(b_h).to(b_w.dtype))
@@ -243,6 +269,8 @@ def chunk_gated_delta_rule_fwd_h(
chunk_indices: Optional[torch.LongTensor] = None,
use_exp2: bool = False,
inplace_update: bool = True,
track_state: Optional[torch.Tensor] = None,
track_chunk_idx: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
if not inplace_update:
raise NotImplementedError(
@@ -251,6 +279,15 @@ def chunk_gated_delta_rule_fwd_h(
assert not (use_exp2 and g is not None), (
"use_exp2 covers only the per-channel gk path; scalar g stays natural-exp"
)
assert (track_state is None) == (track_chunk_idx is None), (
"track_state and track_chunk_idx must be passed together"
)
if track_state is not None:
# The caller rounds once to the pool dtype; a narrower buffer would
# silently double-round the snapshot.
assert track_state.dtype == torch.float32, (
f"track_state must be fp32, got {track_state.dtype}"
)
B, T, Hg, K, V = *k.shape, u.shape[-1]
H = u.shape[-2]
BT = CHUNK_SIZE
@@ -287,8 +324,14 @@ def chunk_gated_delta_rule_fwd_h(
h=h,
initial_state=initial_state,
initial_state_indices=initial_state_indices,
# Envelope-strided state pools (page-major / unified memory) have a
# per-slot pitch != H*V*K; contiguous pools pass exactly H*V*K.
stride_init_state=(initial_state.stride(0) if initial_state is not None else 0),
cu_seqlens=cu_seqlens,
chunk_offsets=chunk_offsets,
track_state=track_state,
track_chunk_idx=track_chunk_idx,
stride_track_state=(track_state.stride(0) if track_state is not None else 0),
T=T,
H=H,
Hg=Hg,
@@ -303,5 +346,6 @@ def chunk_gated_delta_rule_fwd_h(
IS_VARLEN=cu_seqlens is not None,
NT_BUCKET=(0 if NT <= 32 else (1 if NT <= 128 else 2)),
USE_EXP2=use_exp2,
TRACK_STATE=track_state is not None,
)
return h, v_new
+2
View File
@@ -836,6 +836,7 @@ class TopK(BaseFusedOp):
*,
num_token_non_padded: Optional[torch.Tensor] = None,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
) -> TopKOutput:
self.topk_config.torch_native = True
# [NOTE] XPU device support for topk kernels
@@ -852,6 +853,7 @@ class TopK(BaseFusedOp):
topk_config=self.topk_config,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
dynamic_expert_bias=dynamic_expert_bias,
)
@@ -0,0 +1,242 @@
import unittest
import torch
from sglang.kernels.ops.attention.fla.kda import chunk_kda
from sglang.test.ci.ci_register import register_xpu_ci
from sglang.test.test_utils import CustomTestCase
register_xpu_ci(est_time=300, suite="stage-b-test-1-gpu-xpu")
CHUNK_SIZE = 64
# Gap between the pool's per-slot pitch and H*V*K. A kernel that assumes a
# contiguous pool addresses slot i at i*H*V*K, which lands in this gap.
ENVELOPE_PAD = 256
PAD_SENTINEL = -12345.0
def _make_varlen_inputs(seed, lens, num_heads=2, head_dim=128, device="xpu"):
"""Packed varlen KDA inputs: [1, sum(lens), H, D] plus a zero fp32 state pool."""
generator = torch.Generator().manual_seed(seed)
total = sum(lens)
def randn(*shape, dtype=torch.bfloat16):
# CPU generator + transfer: torch.Generator(device="xpu") is not a
# portable construction across the XPU torch builds this suite runs on.
return torch.randn(*shape, generator=generator, dtype=torch.float32).to(
device=device, dtype=dtype
)
q = randn(1, total, num_heads, head_dim)
k = randn(1, total, num_heads, head_dim)
v = (0.1 * randn(1, total, num_heads, head_dim, dtype=torch.float32)).to(
torch.bfloat16
)
gate = randn(1, total, num_heads, head_dim)
beta = torch.sigmoid(randn(1, total, num_heads, dtype=torch.float32)).to(
torch.bfloat16
)
a_log = randn(num_heads, dtype=torch.float32)
dt_bias = randn(num_heads * head_dim, dtype=torch.float32)
state = torch.zeros(
len(lens), num_heads, head_dim, head_dim, device=device, dtype=torch.float32
)
cu_seqlens = torch.tensor(
[0, *torch.tensor(lens).cumsum(0).tolist()], dtype=torch.int32, device=device
)
return q, k, v, gate, beta, a_log, dt_bias, state, cu_seqlens
def _run_chunk_kda(q, k, v, gate, beta, a_log, dt_bias, state, cu_seqlens, **kwargs):
return chunk_kda(
# chunk_kda writes in place (the attention output lands in v, the gate
# cumsum in g); hand every run fresh copies so runs stay independent.
q=q.clone(),
k=k.clone(),
v=v.clone(),
g=gate.clone(),
beta=beta.clone(),
scale=q.shape[-1] ** -0.5,
initial_state=state,
initial_state_indices=torch.arange(
state.shape[0], device=state.device, dtype=torch.int32
),
use_qk_l2norm_in_kernel=True,
cu_seqlens=cu_seqlens,
A_log=a_log,
dt_bias=dt_bias,
lower_bound=-5.0,
**kwargs,
)
@unittest.skipUnless(
torch.xpu.is_available(),
"Intel XPU not available (torch.xpu.is_available() returned False)",
)
class TestKdaTrackStateXpu(CustomTestCase):
@torch.inference_mode()
def test_track_state_snapshots_fp32_accumulator(self):
"""Bug regression: the XPU chunk kernel had no track path at all, so the
mamba radix prefix cache could not snapshot the SSM state at the last
chunk boundary of an unaligned sequence. `track_state` must carry the
in-kernel fp32 accumulator: identical to the fp32 final state of a run
truncated at the boundary, and strictly more precise than the bf16 `h`
row for the same boundary.
"""
# seq0: 100 tokens, unaligned -> snapshot at the 64-token boundary
# (start of chunk 1). seq1: 64 tokens, aligned -> not tracked.
lens = [100, 64]
q, k, v, gate, beta, a_log, dt_bias, state, cu_seqlens = _make_varlen_inputs(
0, lens
)
num_heads, head_dim = q.shape[2], q.shape[3]
track_state = torch.full(
(len(lens), num_heads, head_dim, head_dim),
float("nan"),
device=state.device,
dtype=torch.float32,
)
track_chunk_idx = torch.tensor([1, -1], dtype=torch.int32, device=state.device)
_, h = _run_chunk_kda(
q,
k,
v,
gate,
beta,
a_log,
dt_bias,
state,
cu_seqlens,
output_intermediate_states=True,
track_state=track_state,
track_chunk_idx=track_chunk_idx,
)
# The untracked row must stay untouched; the tracked row must be finite.
self.assertTrue(torch.all(torch.isnan(track_state[1])))
self.assertFalse(torch.any(torch.isnan(track_state[0])))
# Reference: truncate seq0 at the boundary; the pool's fp32 row then
# receives the in-place final state for the same prefix -- the
# established fp32 path the snapshot must agree with.
ref_state = torch.zeros(
1, num_heads, head_dim, head_dim, device=state.device, dtype=torch.float32
)
ref_cu_seqlens = torch.tensor(
[0, CHUNK_SIZE], dtype=torch.int32, device=state.device
)
_run_chunk_kda(
q[:, :CHUNK_SIZE],
k[:, :CHUNK_SIZE],
v[:, :CHUNK_SIZE],
gate[:, :CHUNK_SIZE],
beta[:, :CHUNK_SIZE],
a_log,
dt_bias,
ref_state,
ref_cu_seqlens,
)
torch.testing.assert_close(track_state[0], ref_state[0], rtol=1e-5, atol=1e-5)
# The guard: h packs one row per (seq, chunk); row 1 is seq0's state at
# the boundary, rounded to bf16. If the snapshot were re-routed through
# h, it could not match the fp32 reference above.
self.assertTrue(
torch.equal(h[0, 1].float(), track_state[0].to(torch.bfloat16).float()),
"h row should be exactly the bf16 rounding of the fp32 snapshot",
)
self.assertFalse(
torch.equal(track_state[0], track_state[0].to(torch.bfloat16).float()),
"test inputs must make bf16 rounding lossy",
)
@torch.inference_mode()
def test_envelope_strided_initial_state_pool(self):
"""Bug regression: the XPU chunk kernel addressed a state slot as
index*H*V*K, so a pool whose stride(0) spans all layers (page-major /
unified memory) had every slot but the first read and written at the
wrong offset. Slot pitch must come from initial_state.stride(0): results
match a contiguous pool, and no write leaves a slot.
"""
lens = [100, 64]
q, k, v, gate, beta, a_log, dt_bias, state, cu_seqlens = _make_varlen_inputs(
0, lens
)
num_slots, num_heads, v_dim, k_dim = state.shape
slot_elems = num_heads * v_dim * k_dim
pitch = slot_elems + ENVELOPE_PAD
# Distinct non-zero content per slot: against an all-zero pool, reading
# the wrong slot returns zeros either way and the bug stays invisible.
generator = torch.Generator().manual_seed(7)
base = (
0.1
* torch.randn(
num_slots,
num_heads,
v_dim,
k_dim,
generator=generator,
dtype=torch.float32,
)
).to(state.device)
track_chunk_idx = torch.tensor([1, -1], dtype=torch.int32, device=state.device)
def run(pool):
track = torch.full(
(num_slots, num_heads, v_dim, k_dim),
float("nan"),
device=state.device,
dtype=torch.float32,
)
out, _ = _run_chunk_kda(
q,
k,
v,
gate,
beta,
a_log,
dt_bias,
pool,
cu_seqlens,
output_intermediate_states=True,
track_state=track,
track_chunk_idx=track_chunk_idx,
)
return out, track
contiguous_pool = base.clone()
self.assertEqual(contiguous_pool.stride(0), slot_elems)
expected_out, expected_track = run(contiguous_pool)
envelope = torch.full(
(num_slots, pitch), PAD_SENTINEL, device=state.device, dtype=torch.float32
)
strided_pool = torch.as_strided(
envelope,
(num_slots, num_heads, v_dim, k_dim),
(pitch, v_dim * k_dim, k_dim, 1),
)
strided_pool.copy_(base)
self.assertEqual(strided_pool.stride(0), pitch)
actual_out, actual_track = run(strided_pool)
torch.testing.assert_close(
actual_out.float(), expected_out.float(), rtol=1e-2, atol=1e-2
)
# INPLACE_UPDATE: the pool holds the final state, so this covers the
# write path as well as the read.
torch.testing.assert_close(strided_pool, contiguous_pool, rtol=1e-5, atol=1e-5)
torch.testing.assert_close(
actual_track, expected_track, rtol=1e-5, atol=1e-5, equal_nan=True
)
self.assertTrue(
torch.all(envelope[:, slot_elems:] == PAD_SENTINEL),
"kernel wrote past a pool slot",
)
if __name__ == "__main__":
unittest.main()