[XPU] Add dynamic_expert_bias and track_state support (#39618)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Singh <rohitsi2@iil-login.iind.intel.com>
Co-authored-by: Singh <rohitsi2@iil-gnrap01.iind.intel.com>
Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
This commit is contained in:
Rohit Kumar Singh
2026-09-30 12:05:35 +08:00
committed by GitHub
co-authored by github-actions[bot] Singh Singh Ma Mingfei
parent 8854857a99
commit cb3ba2f2b7
3 changed files with 291 additions and 3 deletions
@@ -41,8 +41,12 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop(
h,
initial_state,
initial_state_indices,
stride_init_state,
cu_seqlens,
chunk_offsets,
track_state,
track_chunk_idx,
stride_track_state,
T,
H: tl.constexpr,
Hg: tl.constexpr,
@@ -58,6 +62,7 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop(
IS_VARLEN: tl.constexpr,
NT_BUCKET: tl.constexpr, # this arg is kept to align with the triton kernel for CUDA
USE_EXP2: tl.constexpr,
TRACK_STATE: tl.constexpr,
):
i_v, i_nh = tl.program_id(0), tl.program_id(1)
i_n, i_h = i_nh // H, i_nh % H
@@ -112,17 +117,30 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop(
block_shape=(BT, BV),
)
index = tl.load(initial_state_indices + i_n).to(tl.int32)
# Slot pitch comes from the caller (initial_state.stride(0)): the state pool
# may be an envelope-strided view (page-major / unified memory), where the
# per-slot pitch spans ALL layers' state, not H*V*K. int64: envelope pitches
# overflow an int32 index product.
index = tl.load(initial_state_indices + i_n).to(tl.int64)
# Padded rows carry the -1 sentinel; without this guard the sentinel
# reaches pointer arithmetic and addresses before the state pool.
valid_state = index >= 0
h0 = initial_state + index * stride_h
ht = initial_state + index * stride_h
h0 = initial_state + index * stride_init_state
ht = initial_state + index * stride_init_state
if USE_INITIAL_STATE:
h0 = h0 + i_h * V * K
if INPLACE_UPDATE:
ht = ht + i_h * V * K
if TRACK_STATE:
i_track = tl.load(track_chunk_idx + i_n).to(tl.int32)
p_track_base = track_state + (i_n * stride_track_state + i_h * V * K).to(
tl.int64
)
else:
i_track = -1
p_track_base = track_state
# main recurrence — time is the outer loop
for i_t in range(NT):
########################################################################
@@ -158,6 +176,14 @@ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64_k_loop(
)
tl.store(p_ho, b_h.to(p_ho.dtype.element_ty), boundary_check=(0, 1))
# Mid-sequence checkpoint for the mamba prefix cache: the same
# pre-update state, kept in fp32 so the caller rounds only once.
if TRACK_STATE and i_t == i_track:
p_track = tl.make_block_ptr(
p_track_base, (V, K), (K, 1), (i_v * BV, k_blk), (BV, 64), (1, 0)
)
tl.store(p_track, b_h, boundary_check=(0, 1))
# Accumulate correction: w_k @ h_k^T
b_w = w_desc.load([i_t * BT, k_blk])
b_v_corr += tl.dot(b_w, tl.trans(b_h).to(b_w.dtype))
@@ -243,6 +269,8 @@ def chunk_gated_delta_rule_fwd_h(
chunk_indices: Optional[torch.LongTensor] = None,
use_exp2: bool = False,
inplace_update: bool = True,
track_state: Optional[torch.Tensor] = None,
track_chunk_idx: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
if not inplace_update:
raise NotImplementedError(
@@ -251,6 +279,15 @@ def chunk_gated_delta_rule_fwd_h(
assert not (use_exp2 and g is not None), (
"use_exp2 covers only the per-channel gk path; scalar g stays natural-exp"
)
assert (track_state is None) == (track_chunk_idx is None), (
"track_state and track_chunk_idx must be passed together"
)
if track_state is not None:
# The caller rounds once to the pool dtype; a narrower buffer would
# silently double-round the snapshot.
assert track_state.dtype == torch.float32, (
f"track_state must be fp32, got {track_state.dtype}"
)
B, T, Hg, K, V = *k.shape, u.shape[-1]
H = u.shape[-2]
BT = CHUNK_SIZE
@@ -287,8 +324,14 @@ def chunk_gated_delta_rule_fwd_h(
h=h,
initial_state=initial_state,
initial_state_indices=initial_state_indices,
# Envelope-strided state pools (page-major / unified memory) have a
# per-slot pitch != H*V*K; contiguous pools pass exactly H*V*K.
stride_init_state=(initial_state.stride(0) if initial_state is not None else 0),
cu_seqlens=cu_seqlens,
chunk_offsets=chunk_offsets,
track_state=track_state,
track_chunk_idx=track_chunk_idx,
stride_track_state=(track_state.stride(0) if track_state is not None else 0),
T=T,
H=H,
Hg=Hg,
@@ -303,5 +346,6 @@ def chunk_gated_delta_rule_fwd_h(
IS_VARLEN=cu_seqlens is not None,
NT_BUCKET=(0 if NT <= 32 else (1 if NT <= 128 else 2)),
USE_EXP2=use_exp2,
TRACK_STATE=track_state is not None,
)
return h, v_new
+2
View File
@@ -836,6 +836,7 @@ class TopK(BaseFusedOp):
*,
num_token_non_padded: Optional[torch.Tensor] = None,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
) -> TopKOutput:
self.topk_config.torch_native = True
# [NOTE] XPU device support for topk kernels
@@ -852,6 +853,7 @@ class TopK(BaseFusedOp):
topk_config=self.topk_config,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
dynamic_expert_bias=dynamic_expert_bias,
)