[Sampler] Sort only the max(top_k) leading columns in the torch sampler (#43346)

This commit is contained in:
jiaqiang-dot-liu
2026-10-11 01:00:59 -07:00
committed by GitHub
parent dab108b9f3
commit 41652225d0
5 changed files with 190 additions and 2 deletions
+19 -2
View File
@@ -450,6 +450,7 @@ class Sampler(nn.Module):
sampling_info.sampling_seed,
positions,
return_filtered_probs=return_sampling_mask,
max_top_k=sampling_info.max_top_k,
)
if return_sampling_mask:
(
@@ -759,6 +760,7 @@ def top_k_top_p_min_p_sampling_from_probs_torch(
positions: torch.Tensor,
*,
return_filtered_probs: bool = False,
max_top_k: Optional[int] = None,
):
"""
A top-k, top-p and min-p sampling implementation with native pytorch operations.
@@ -768,11 +770,26 @@ def top_k_top_p_min_p_sampling_from_probs_torch(
By default, returns only sampled token IDs. With return_filtered_probs=True,
also returns the actual filtered weights, their token-ID permutation, and
the selected weights.
max_top_k, when given, must be >= every value in top_ks.
"""
probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
# The top-k mask below zeroes every column at rank >= top_k, so the max_top_k
# leading columns carry all the mass that survives and sorting only those is
# exact. The seeded path keeps the full sort because the order of tied
# probabilities may depend on k, which is a batch-wide value, and
# deterministic inference must not depend on the batch composition.
if (
max_top_k is not None
and max_top_k < probs.shape[-1]
and sampling_seed is None
and not return_filtered_probs
):
probs_sort, probs_idx = torch.topk(probs, k=max_top_k, dim=-1, sorted=True)
else:
probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
probs_sum = torch.cumsum(probs_sort, dim=-1)
probs_sort[
torch.arange(0, probs.shape[-1], device=probs.device).view(1, -1)
torch.arange(0, probs_sort.shape[-1], device=probs.device).view(1, -1)
>= top_ks.view(-1, 1)
] = 0.0
probs_sort[(probs_sum - probs_sort) > top_ps.view(-1, 1)] = 0.0
@@ -111,6 +111,11 @@ class SamplingBatchInfo:
# device avoids a scalar synchronization in the per-token sampling path.
npu_top_k_top_p_eligible: bool = False
# Host-side upper bound on every row's top_k, or None when some row samples
# from the whole vocabulary. Any value >= the true max stays exact, so
# filter_batch keeps it unchanged.
max_top_k: Optional[int] = None
@classmethod
def from_schedule_batch(cls, batch: ScheduleBatch, vocab_size: int):
enable_deterministic = get_exec().deterministic.enable_deterministic_inference
@@ -263,6 +268,11 @@ class SamplingBatchInfo:
npu_top_k_top_p_eligible=all(
1 <= r.sampling_params.top_k <= 1024 for r in reqs
),
max_top_k=(
max(r.sampling_params.top_k for r in reqs)
if reqs and all(1 <= r.sampling_params.top_k < TOP_K_ALL for r in reqs)
else None
),
vocab_size=vocab_size,
penalizer_orchestrator=penalizer_orchestrator,
has_custom_logit_processor=has_custom_logit_processor,
@@ -572,6 +582,11 @@ class SamplingBatchInfo:
self.need_top_k_sampling |= other.need_top_k_sampling
self.need_min_p_sampling |= other.need_min_p_sampling
self.npu_top_k_top_p_eligible &= other.npu_top_k_top_p_eligible
self.max_top_k = (
None
if self.max_top_k is None or other.max_top_k is None
else max(self.max_top_k, other.max_top_k)
)
self.adjusted_merge_batch(other)