[NPU] Avoid device synchronization in Ascend sampling (#39404)

python/sglang/srt/layers/sampler.py

@@ -615,6 +615,7 @@ class Sampler(nn.Module):
                 sampling_info.need_min_p_sampling,
                 sampling_info.sampling_seed,
                 positions,
+                npu_top_k_top_p_eligible=sampling_info.npu_top_k_top_p_eligible,
             )
             return batch_next_token_ids.to(torch.int32)
 
@@ -782,15 +783,14 @@ def top_k_top_p_min_p_sampling_from_logits_ascend(
     need_min_p_sampling: bool,
     sampling_seed: Optional[torch.Tensor],
     positions: torch.Tensor,
+    npu_top_k_top_p_eligible: bool = False,
 ):
     """A top-k, top-p and min-p sampling implementation for ascend npu with torch_npu interface.
 
     Takes temperature-scaled logits as input (softmax is applied internally).
     """
     # torch_npu.npu_top_k_top_p requires top_k value range in [1, 1024]
-    if hasattr(torch_npu, "npu_top_k_top_p") and torch.all(
-        (top_ks <= 1024) & (top_ks >= 1)
-    ):
+    if hasattr(torch_npu, "npu_top_k_top_p") and npu_top_k_top_p_eligible:
         logits_top_k_top_p = torch_npu.npu_top_k_top_p(logits, top_ps, top_ks)
         probs_top_k_top_p = logits_top_k_top_p.softmax(dim=-1)
 

python/sglang/srt/sampling/sampling_batch_info.py

@@ -85,6 +85,10 @@ class SamplingBatchInfo:
     # Handle logit bias
     logit_bias: Optional[torch.Tensor] = None
 
+    # Host-side eligibility for torch_npu.npu_top_k_top_p. Keeping this off the
+    # device avoids a scalar synchronization in the per-token sampling path.
+    npu_top_k_top_p_eligible: bool = False
+
     @classmethod
     def from_schedule_batch(cls, batch: ScheduleBatch, vocab_size: int):
         enable_deterministic = get_exec().deterministic.enable_deterministic_inference
@@ -207,6 +211,9 @@ class SamplingBatchInfo:
             need_top_p_sampling=any(r.sampling_params.top_p != 1.0 for r in reqs),
             need_top_k_sampling=any(r.sampling_params.top_k != TOP_K_ALL for r in reqs),
             need_min_p_sampling=any(r.sampling_params.min_p > 0 for r in reqs),
+            npu_top_k_top_p_eligible=all(
+                1 <= r.sampling_params.top_k <= 1024 for r in reqs
+            ),
             vocab_size=vocab_size,
             penalizer_orchestrator=penalizer_orchestrator,
             has_custom_logit_processor=has_custom_logit_processor,
@@ -502,6 +509,7 @@ class SamplingBatchInfo:
         self.need_top_p_sampling |= other.need_top_p_sampling
         self.need_top_k_sampling |= other.need_top_k_sampling
         self.need_min_p_sampling |= other.need_min_p_sampling
+        self.npu_top_k_top_p_eligible &= other.npu_top_k_top_p_eligible
 
         self.adjusted_merge_batch(other)