[DSpark] Mask uncommitted slots before commit-inject cache lookups (#42624)

This commit is contained in:
Chunan Zeng
2026-10-08 00:16:54 -07:00
committed by GitHub
parent 54b551df8e
commit 785005d21b
2 changed files with 20 additions and 28 deletions
@@ -702,32 +702,20 @@ def build_commit_inject_layout(
commit_lens: torch.Tensor,
stride: int,
) -> CommitInjectLayoutResult:
from sglang.kernels.ops.speculative.cache_locs import (
assign_extend_cache_locs_func,
)
bs = req_pool_indices.shape[0]
device = req_pool_indices.device
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets[:stride]
positions = positions_2d.reshape(-1).to(dtype=torch.int64)
cache_loc = assign_extend_cache_locs_func(
req_pool_indices=req_pool_indices,
req_to_token=req_to_token,
start_offset=prefix_lens,
end_offset=prefix_lens + stride,
batch_size=bs,
draft_token_num=stride,
device=device,
).to(dtype=torch.int64)
swa_loc = full_to_swa_mapping[cache_loc].to(torch.int32)
col = torch.arange(stride, device=device).view(1, -1)
committed = (col < commit_lens.to(torch.long).view(-1, 1)).reshape(-1)
swa_loc = torch.where(committed, swa_loc, torch.full_like(swa_loc, -1))
return CommitInjectLayoutResult(swa_loc=swa_loc, positions=positions)
committed = col < commit_lens.to(torch.long).view(-1, 1)
req_rows = req_pool_indices.to(torch.long).view(-1, 1).expand(bs, stride)
rows = torch.where(committed, req_rows, 0)
cols = torch.where(committed, positions_2d.long(), 0)
cache_loc = torch.where(committed, req_to_token[rows, cols].long(), 0)
swa_loc = torch.where(committed, full_to_swa_mapping[cache_loc], -1)
return CommitInjectLayoutResult(
swa_loc=swa_loc.to(torch.int32).reshape(-1), positions=positions
)
@triton.jit
@@ -753,13 +741,14 @@ def _commit_inject_layout_kernel(
prefix = tl.load(prefix_lens_ptr + r, mask=mask, other=0).to(tl.int64)
pos_off = tl.load(block_pos_offsets_ptr + c, mask=mask, other=0).to(tl.int64)
rp = tl.load(req_pool_ptr + r, mask=mask, other=0).to(tl.int64)
full_loc = tl.load(
req_to_token_ptr + rp * rt_stride + prefix + pos_off, mask=mask, other=0
).to(tl.int64)
swa = tl.load(full_to_swa_ptr + full_loc, mask=mask, other=-1).to(tl.int32)
commit_len = tl.load(commit_lens_ptr + r, mask=mask, other=0).to(tl.int64)
swa = tl.where(c.to(tl.int64) < commit_len, swa, -1)
committed = mask & (c.to(tl.int64) < commit_len)
full_loc = tl.load(
req_to_token_ptr + rp * rt_stride + prefix + pos_off,
mask=committed,
other=0,
).to(tl.int64)
swa = tl.load(full_to_swa_ptr + full_loc, mask=committed, other=-1).to(tl.int32)
tl.store(swa_loc_ptr + offs, swa, mask=mask)
tl.store(positions_ptr + offs, prefix + pos_off, mask=mask)