[DSpark] Mask uncommitted slots before commit-inject cache lookups (#42624)
This commit is contained in:
@@ -702,32 +702,20 @@ def build_commit_inject_layout(
|
||||
commit_lens: torch.Tensor,
|
||||
stride: int,
|
||||
) -> CommitInjectLayoutResult:
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
assign_extend_cache_locs_func,
|
||||
)
|
||||
|
||||
bs = req_pool_indices.shape[0]
|
||||
device = req_pool_indices.device
|
||||
|
||||
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets[:stride]
|
||||
positions = positions_2d.reshape(-1).to(dtype=torch.int64)
|
||||
|
||||
cache_loc = assign_extend_cache_locs_func(
|
||||
req_pool_indices=req_pool_indices,
|
||||
req_to_token=req_to_token,
|
||||
start_offset=prefix_lens,
|
||||
end_offset=prefix_lens + stride,
|
||||
batch_size=bs,
|
||||
draft_token_num=stride,
|
||||
device=device,
|
||||
).to(dtype=torch.int64)
|
||||
swa_loc = full_to_swa_mapping[cache_loc].to(torch.int32)
|
||||
|
||||
col = torch.arange(stride, device=device).view(1, -1)
|
||||
committed = (col < commit_lens.to(torch.long).view(-1, 1)).reshape(-1)
|
||||
swa_loc = torch.where(committed, swa_loc, torch.full_like(swa_loc, -1))
|
||||
|
||||
return CommitInjectLayoutResult(swa_loc=swa_loc, positions=positions)
|
||||
committed = col < commit_lens.to(torch.long).view(-1, 1)
|
||||
req_rows = req_pool_indices.to(torch.long).view(-1, 1).expand(bs, stride)
|
||||
rows = torch.where(committed, req_rows, 0)
|
||||
cols = torch.where(committed, positions_2d.long(), 0)
|
||||
cache_loc = torch.where(committed, req_to_token[rows, cols].long(), 0)
|
||||
swa_loc = torch.where(committed, full_to_swa_mapping[cache_loc], -1)
|
||||
return CommitInjectLayoutResult(
|
||||
swa_loc=swa_loc.to(torch.int32).reshape(-1), positions=positions
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -753,13 +741,14 @@ def _commit_inject_layout_kernel(
|
||||
prefix = tl.load(prefix_lens_ptr + r, mask=mask, other=0).to(tl.int64)
|
||||
pos_off = tl.load(block_pos_offsets_ptr + c, mask=mask, other=0).to(tl.int64)
|
||||
rp = tl.load(req_pool_ptr + r, mask=mask, other=0).to(tl.int64)
|
||||
full_loc = tl.load(
|
||||
req_to_token_ptr + rp * rt_stride + prefix + pos_off, mask=mask, other=0
|
||||
).to(tl.int64)
|
||||
swa = tl.load(full_to_swa_ptr + full_loc, mask=mask, other=-1).to(tl.int32)
|
||||
|
||||
commit_len = tl.load(commit_lens_ptr + r, mask=mask, other=0).to(tl.int64)
|
||||
swa = tl.where(c.to(tl.int64) < commit_len, swa, -1)
|
||||
committed = mask & (c.to(tl.int64) < commit_len)
|
||||
full_loc = tl.load(
|
||||
req_to_token_ptr + rp * rt_stride + prefix + pos_off,
|
||||
mask=committed,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
swa = tl.load(full_to_swa_ptr + full_loc, mask=committed, other=-1).to(tl.int32)
|
||||
|
||||
tl.store(swa_loc_ptr + offs, swa, mask=mask)
|
||||
tl.store(positions_ptr + offs, prefix + pos_off, mask=mask)
|
||||
|
||||
Reference in New Issue
Block a user