From bdc4ec15ce4fbe2a05c43b3d6b79d2d657232863 Mon Sep 17 00:00:00 2001 From: kk <43161300+kkHuang-amd@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:13:37 +0800 Subject: [PATCH] [Perf] [AMD] dspark: No per-step host sync in decode (1/4) (#44405) Co-authored-by: wunhuang --- .../ops/speculative/dspark/simulated_bonus.py | 93 +++++++++++++++++++ python/sglang/srt/environ.py | 3 + python/sglang/srt/managers/overlap_utils.py | 8 +- python/sglang/srt/managers/scheduler.py | 12 ++- .../dspark_components/dspark_verify.py | 37 ++++++++ .../dspark_components/dspark_worker_v2.py | 18 +++- 6 files changed, 166 insertions(+), 5 deletions(-) diff --git a/python/sglang/kernels/ops/speculative/dspark/simulated_bonus.py b/python/sglang/kernels/ops/speculative/dspark/simulated_bonus.py index 39b72d86a8..ec1e073db1 100644 --- a/python/sglang/kernels/ops/speculative/dspark/simulated_bonus.py +++ b/python/sglang/kernels/ops/speculative/dspark/simulated_bonus.py @@ -60,6 +60,99 @@ def _simulated_bonus_select_kernel( ) +@triton.jit +def _simulated_bonus_row_partial_kernel( + logits_ptr, + logits_stride, + temperatures_ptr, + correct_len_ptr, + seed_ptr, + part_val_ptr, + part_idx_ptr, + vocab, + ROWS: tl.constexpr, + NSPLIT: tl.constexpr, + BLOCK: tl.constexpr, +): + req = tl.program_id(0) + split = tl.program_id(1) + row = req * ROWS + tl.load(correct_len_ptr + req) + t = tl.load(temperatures_ptr + req).to(tl.float32) + offs = split * BLOCK + tl.arange(0, BLOCK) + mask = offs < vocab + x = tl.load( + logits_ptr + row.to(tl.int64) * logits_stride + offs, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + seed = tl.load(seed_ptr) + u = tl.rand(seed, req * vocab + offs) + noisy = x / tl.maximum(t, 1e-5) - tl.log(-tl.log(u)) + v = tl.where(mask, tl.where(t > 0.0, noisy, x), float("-inf")) + top = tl.max(v, axis=0) + idx = tl.min(tl.where(v == top, offs, vocab), axis=0) + tl.store(part_val_ptr + req * NSPLIT + split, top) + tl.store(part_idx_ptr + req * NSPLIT + split, idx) + + +@triton.jit +def _simulated_bonus_row_select_kernel( + part_val_ptr, + part_idx_ptr, + out_ptr, + vocab, + NSPLIT: tl.constexpr, +): + req = tl.program_id(0) + s = tl.arange(0, NSPLIT) + val = tl.load(part_val_ptr + req * NSPLIT + s) + idx = tl.load(part_idx_ptr + req * NSPLIT + s) + top = tl.max(val, axis=0) + tl.store( + out_ptr + req, tl.min(tl.where(val == top, idx, vocab), axis=0).to(tl.int64) + ) + + +def simulated_bonus_sample_graph( + *, + target_logits: torch.Tensor, + correct_len: torch.Tensor, + temperatures: torch.Tensor, + seed: torch.Tensor, + bs: int, + rows_per_request: int, +) -> torch.Tensor: + """Graph-safe variant of simulated_bonus_sample: seed ([1] int) and per-request + temperatures ([bs] float; <= 0 means argmax) are read from device buffers, and + only the row at ``correct_len`` is sampled. [bs] int64.""" + assert target_logits.stride(1) == 1 + device = target_logits.device + vocab = target_logits.shape[1] + block = 4096 + nsplit = triton.next_power_of_2(triton.cdiv(vocab, block)) + part_val = torch.empty((bs, nsplit), dtype=torch.float32, device=device) + part_idx = torch.empty((bs, nsplit), dtype=torch.int32, device=device) + _simulated_bonus_row_partial_kernel[(bs, nsplit)]( + target_logits, + target_logits.stride(0), + temperatures, + correct_len.contiguous(), + seed, + part_val, + part_idx, + vocab, + ROWS=rows_per_request, + NSPLIT=nsplit, + BLOCK=block, + num_warps=8, + ) + out = torch.empty((bs,), dtype=torch.int64, device=device) + _simulated_bonus_row_select_kernel[(bs,)]( + part_val, part_idx, out, vocab, NSPLIT=nsplit + ) + return out + + def simulated_bonus_sample( *, target_logits: torch.Tensor, diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 284df9b3be..55d7e22c66 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -519,6 +519,9 @@ class Envs: # DSpark on HIP: with SGLANG_SIMULATE_ACC_LEN set, temperature-only sampling requests # run the greedy draft/accept path and only the bonus token is temperature-sampled. SGLANG_SIMULATE_ACC_GREEDY = EnvBool(True) + # DSpark on HIP: apply the simulated correct_len / bonus inside the folded verify epilogue + # (hip graph) instead of falling back to the eager accept + commit path. + SGLANG_DSPARK_SIMULATE_FOLD = EnvBool(True) SGLANG_SIMULATE_UNIFORM_EXPERTS = EnvBool(False) SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS = EnvBool(False) diff --git a/python/sglang/srt/managers/overlap_utils.py b/python/sglang/srt/managers/overlap_utils.py index faf3010aba..41d97577dd 100644 --- a/python/sglang/srt/managers/overlap_utils.py +++ b/python/sglang/srt/managers/overlap_utils.py @@ -541,7 +541,9 @@ class FutureMap: int(x) for x in fresh_cpu.tolist() ] - def resolve_seq_lens_cpu(self, batch: ScheduleBatch) -> None: + def resolve_seq_lens_cpu( + self, batch: ScheduleBatch, stream_ordered: bool = False + ) -> None: # Lazy pull from new_seq_lens_buf for spec_v2 (accept_lens not known to # schedule). The CPU mirror is gated by needs_cpu_seq_lens; backends that # opt out take the GPU-only path below. A private D2H stream overlaps the copy. @@ -552,7 +554,9 @@ class FutureMap: fi = draft_input.future_indices if fi is None: return - if self.publish_ready is not None: + if self.publish_ready is not None and stream_ordered: + self._publish_fresh = False + elif self.publish_ready is not None: if _DEBUG_ASSERT: # Consume-once: every event wait must be re-armed by a fresh # forward publish; a stale consume means a publish went missing. diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 0d0ec41b60..98c5f47bfc 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4572,12 +4572,22 @@ class Scheduler( if self.enable_overlap: # Self-gates on batch.spec_info.future_indices; non-spec_v2 # no-ops (ForwardBatch.init_new lazily computes the sum). - self.future_map.resolve_seq_lens_cpu(batch) + gpu_only_lens = ( + _is_hip + and self._confidence_budget_prepare is None + and not getattr(self.future_map, "needs_cpu_seq_lens", True) + ) + if not gpu_only_lens: + self.future_map.resolve_seq_lens_cpu(batch) if self._confidence_budget_prepare is not None: self._confidence_budget_prepare(batch, self.future_map) with self.forward_stream_ctx: self.forward_stream.wait_stream(self.schedule_stream) + if gpu_only_lens: + # publish_ready was recorded on forward_stream: stream order + # fences the gather, no host sync / event wait needed. + self.future_map.resolve_seq_lens_cpu(batch, stream_ordered=True) # resolve consumes SB staging (prefill_input_ids_cpu / # mix_running_indices). Run OUTSIDE isolation so the # snapshot captures the post-consume state — restoring diff --git a/python/sglang/srt/speculative/dspark_components/dspark_verify.py b/python/sglang/srt/speculative/dspark_components/dspark_verify.py index a6aaac3af4..273178cb92 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_verify.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_verify.py @@ -1,5 +1,6 @@ from __future__ import annotations +import random from typing import Optional import msgspec @@ -26,6 +27,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import ( ) from sglang.kernels.ops.speculative.dspark.simulated_bonus import ( simulated_bonus_sample, + simulated_bonus_sample_graph, ) from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.managers.schedule_batch import ScheduleBatch @@ -537,8 +539,18 @@ class DsparkVerifyEpilogue: commit_ctx: Optional[CommitInjectCtx] = None, fused_argmax: bool = False, simulated_correct_len: Optional[torch.Tensor] = None, + simulate_fold: bool = False, ) -> None: self.max_bs = int(max_bs) + # SGLANG_SIMULATE_ACC_LEN with a sampled bonus inside the graph too: the bonus + # is drawn at the simulated correct_len from these buffers, which + # begin_sim_bonus writes before replay (<= 0 temperature: argmax). + self.simulate_fold = bool(simulate_fold) and simulated_correct_len is not None + self.sim_temperatures_buf = torch.zeros( + (self.max_bs,), dtype=torch.float32, device=device + ) + self.sim_seed_buf = torch.zeros((1,), dtype=torch.int32, device=device) + self._sim_greedy_bs: Optional[int] = None self.stride = int(verify_num_draft_tokens) self._fused_argmax = bool(fused_argmax) self.gamma = self.stride - 1 @@ -617,6 +629,17 @@ class DsparkVerifyEpilogue: self.inject_gate_buf.fill_(int(armed)) self._static_step_state = state + def begin_sim_bonus(self, bs: int, temperatures: Optional[torch.Tensor]) -> None: + """The step's bonus temperatures (None: greedy) and seed, before replay.""" + if temperatures is None: + if self._sim_greedy_bs != bs: + self.sim_temperatures_buf.zero_() + self._sim_greedy_bs = bs + return + self._sim_greedy_bs = None + self.sim_temperatures_buf[:bs].copy_(temperatures.view(-1)) + self.sim_seed_buf.fill_(random.getrandbits(31)) + def _static_epilogue(self, out, forward_batch) -> None: bs = forward_batch.batch_size verify_lens = self.verify_lens_buf[:bs] @@ -751,6 +774,20 @@ class DsparkVerifyEpilogue: cutoff_verify_lens=cutoff_verify_lens, fused_argmax=self._fused_argmax, ) + if self.simulate_fold: + # Before the TP broadcast in build_accept_outs: every rank commits rank 0's draw. + bonus = ( + simulated_bonus_sample_graph( + target_logits=logits, + correct_len=self.simulated_correct_len.repeat(bs), + temperatures=self.sim_temperatures_buf[:bs], + seed=self.sim_seed_buf, + bs=bs, + rows_per_request=self.stride, + ) + .to(bonus.dtype) + .view_as(bonus) + ) accept = build_accept_outs( correct_len=correct_len, bonus=bonus, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 6bf618cbc0..67f1c4c6a4 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -365,6 +365,7 @@ class DSparkWorkerV2(BaseSpecWorker): tp_sync=self._tp_sync, fused_argmax=target_is_dsv41, simulated_correct_len=self._simulated_correct_len, + simulate_fold=_is_hip and envs.SGLANG_DSPARK_SIMULATE_FOLD.get(), commit_ctx=CommitInjectCtx( draft_model=self.draft_model, block_pos_offsets=self._block_pos_offsets, @@ -932,11 +933,24 @@ class DSparkWorkerV2(BaseSpecWorker): # draft proposal itself folded. and (sampling_info is None or sampling_info.is_all_greedy) and verify_logits_adjustments_are_noop(sampling_info) - # The graph accept is greedy, so a sampled simulated bonus is eager. - and simulate_bonus_sampling_info is None + # The graph accept is greedy: a sampled simulated bonus stays in the graph + # only when the epilogue draws it there (SGLANG_DSPARK_SIMULATE_FOLD). + and ( + simulate_bonus_sampling_info is None + or self._verify_executor.verify_epilogue.simulate_fold + ) and not batch.has_grammar ) self._verify_executor.begin_simulated_step() + if fold_eligible and self._verify_executor.verify_epilogue.simulate_fold: + self._verify_executor.verify_epilogue.begin_sim_bonus( + bs, + ( + None + if simulate_bonus_sampling_info is None + else simulate_bonus_sampling_info.temperatures[:bs] + ), + ) prepare_mamba_track_for_verify(batch) with self._observers.segment(InfoSegment.TARGET_VERIFY): if run_compact: