[Perf] [AMD] dspark: No per-step host sync in decode (1/4) (#44405)

Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
kk
2026-10-16 01:13:37 -07:00
committed by GitHub
co-authored by wunhuang
parent 5dd37af412
commit bdc4ec15ce
6 changed files with 166 additions and 5 deletions
@@ -60,6 +60,99 @@ def _simulated_bonus_select_kernel(
)
@triton.jit
def _simulated_bonus_row_partial_kernel(
logits_ptr,
logits_stride,
temperatures_ptr,
correct_len_ptr,
seed_ptr,
part_val_ptr,
part_idx_ptr,
vocab,
ROWS: tl.constexpr,
NSPLIT: tl.constexpr,
BLOCK: tl.constexpr,
):
req = tl.program_id(0)
split = tl.program_id(1)
row = req * ROWS + tl.load(correct_len_ptr + req)
t = tl.load(temperatures_ptr + req).to(tl.float32)
offs = split * BLOCK + tl.arange(0, BLOCK)
mask = offs < vocab
x = tl.load(
logits_ptr + row.to(tl.int64) * logits_stride + offs,
mask=mask,
other=float("-inf"),
).to(tl.float32)
seed = tl.load(seed_ptr)
u = tl.rand(seed, req * vocab + offs)
noisy = x / tl.maximum(t, 1e-5) - tl.log(-tl.log(u))
v = tl.where(mask, tl.where(t > 0.0, noisy, x), float("-inf"))
top = tl.max(v, axis=0)
idx = tl.min(tl.where(v == top, offs, vocab), axis=0)
tl.store(part_val_ptr + req * NSPLIT + split, top)
tl.store(part_idx_ptr + req * NSPLIT + split, idx)
@triton.jit
def _simulated_bonus_row_select_kernel(
part_val_ptr,
part_idx_ptr,
out_ptr,
vocab,
NSPLIT: tl.constexpr,
):
req = tl.program_id(0)
s = tl.arange(0, NSPLIT)
val = tl.load(part_val_ptr + req * NSPLIT + s)
idx = tl.load(part_idx_ptr + req * NSPLIT + s)
top = tl.max(val, axis=0)
tl.store(
out_ptr + req, tl.min(tl.where(val == top, idx, vocab), axis=0).to(tl.int64)
)
def simulated_bonus_sample_graph(
*,
target_logits: torch.Tensor,
correct_len: torch.Tensor,
temperatures: torch.Tensor,
seed: torch.Tensor,
bs: int,
rows_per_request: int,
) -> torch.Tensor:
"""Graph-safe variant of simulated_bonus_sample: seed ([1] int) and per-request
temperatures ([bs] float; <= 0 means argmax) are read from device buffers, and
only the row at ``correct_len`` is sampled. [bs] int64."""
assert target_logits.stride(1) == 1
device = target_logits.device
vocab = target_logits.shape[1]
block = 4096
nsplit = triton.next_power_of_2(triton.cdiv(vocab, block))
part_val = torch.empty((bs, nsplit), dtype=torch.float32, device=device)
part_idx = torch.empty((bs, nsplit), dtype=torch.int32, device=device)
_simulated_bonus_row_partial_kernel[(bs, nsplit)](
target_logits,
target_logits.stride(0),
temperatures,
correct_len.contiguous(),
seed,
part_val,
part_idx,
vocab,
ROWS=rows_per_request,
NSPLIT=nsplit,
BLOCK=block,
num_warps=8,
)
out = torch.empty((bs,), dtype=torch.int64, device=device)
_simulated_bonus_row_select_kernel[(bs,)](
part_val, part_idx, out, vocab, NSPLIT=nsplit
)
return out
def simulated_bonus_sample(
*,
target_logits: torch.Tensor,
+3
View File
@@ -519,6 +519,9 @@ class Envs:
# DSpark on HIP: with SGLANG_SIMULATE_ACC_LEN set, temperature-only sampling requests
# run the greedy draft/accept path and only the bonus token is temperature-sampled.
SGLANG_SIMULATE_ACC_GREEDY = EnvBool(True)
# DSpark on HIP: apply the simulated correct_len / bonus inside the folded verify epilogue
# (hip graph) instead of falling back to the eager accept + commit path.
SGLANG_DSPARK_SIMULATE_FOLD = EnvBool(True)
SGLANG_SIMULATE_UNIFORM_EXPERTS = EnvBool(False)
SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS = EnvBool(False)
+6 -2
View File
@@ -541,7 +541,9 @@ class FutureMap:
int(x) for x in fresh_cpu.tolist()
]
def resolve_seq_lens_cpu(self, batch: ScheduleBatch) -> None:
def resolve_seq_lens_cpu(
self, batch: ScheduleBatch, stream_ordered: bool = False
) -> None:
# Lazy pull from new_seq_lens_buf for spec_v2 (accept_lens not known to
# schedule). The CPU mirror is gated by needs_cpu_seq_lens; backends that
# opt out take the GPU-only path below. A private D2H stream overlaps the copy.
@@ -552,7 +554,9 @@ class FutureMap:
fi = draft_input.future_indices
if fi is None:
return
if self.publish_ready is not None:
if self.publish_ready is not None and stream_ordered:
self._publish_fresh = False
elif self.publish_ready is not None:
if _DEBUG_ASSERT:
# Consume-once: every event wait must be re-armed by a fresh
# forward publish; a stale consume means a publish went missing.
+11 -1
View File
@@ -4572,12 +4572,22 @@ class Scheduler(
if self.enable_overlap:
# Self-gates on batch.spec_info.future_indices; non-spec_v2
# no-ops (ForwardBatch.init_new lazily computes the sum).
self.future_map.resolve_seq_lens_cpu(batch)
gpu_only_lens = (
_is_hip
and self._confidence_budget_prepare is None
and not getattr(self.future_map, "needs_cpu_seq_lens", True)
)
if not gpu_only_lens:
self.future_map.resolve_seq_lens_cpu(batch)
if self._confidence_budget_prepare is not None:
self._confidence_budget_prepare(batch, self.future_map)
with self.forward_stream_ctx:
self.forward_stream.wait_stream(self.schedule_stream)
if gpu_only_lens:
# publish_ready was recorded on forward_stream: stream order
# fences the gather, no host sync / event wait needed.
self.future_map.resolve_seq_lens_cpu(batch, stream_ordered=True)
# resolve consumes SB staging (prefill_input_ids_cpu /
# mix_running_indices). Run OUTSIDE isolation so the
# snapshot captures the post-consume state — restoring
@@ -1,5 +1,6 @@
from __future__ import annotations
import random
from typing import Optional
import msgspec
@@ -26,6 +27,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
)
from sglang.kernels.ops.speculative.dspark.simulated_bonus import (
simulated_bonus_sample,
simulated_bonus_sample_graph,
)
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.schedule_batch import ScheduleBatch
@@ -537,8 +539,18 @@ class DsparkVerifyEpilogue:
commit_ctx: Optional[CommitInjectCtx] = None,
fused_argmax: bool = False,
simulated_correct_len: Optional[torch.Tensor] = None,
simulate_fold: bool = False,
) -> None:
self.max_bs = int(max_bs)
# SGLANG_SIMULATE_ACC_LEN with a sampled bonus inside the graph too: the bonus
# is drawn at the simulated correct_len from these buffers, which
# begin_sim_bonus writes before replay (<= 0 temperature: argmax).
self.simulate_fold = bool(simulate_fold) and simulated_correct_len is not None
self.sim_temperatures_buf = torch.zeros(
(self.max_bs,), dtype=torch.float32, device=device
)
self.sim_seed_buf = torch.zeros((1,), dtype=torch.int32, device=device)
self._sim_greedy_bs: Optional[int] = None
self.stride = int(verify_num_draft_tokens)
self._fused_argmax = bool(fused_argmax)
self.gamma = self.stride - 1
@@ -617,6 +629,17 @@ class DsparkVerifyEpilogue:
self.inject_gate_buf.fill_(int(armed))
self._static_step_state = state
def begin_sim_bonus(self, bs: int, temperatures: Optional[torch.Tensor]) -> None:
"""The step's bonus temperatures (None: greedy) and seed, before replay."""
if temperatures is None:
if self._sim_greedy_bs != bs:
self.sim_temperatures_buf.zero_()
self._sim_greedy_bs = bs
return
self._sim_greedy_bs = None
self.sim_temperatures_buf[:bs].copy_(temperatures.view(-1))
self.sim_seed_buf.fill_(random.getrandbits(31))
def _static_epilogue(self, out, forward_batch) -> None:
bs = forward_batch.batch_size
verify_lens = self.verify_lens_buf[:bs]
@@ -751,6 +774,20 @@ class DsparkVerifyEpilogue:
cutoff_verify_lens=cutoff_verify_lens,
fused_argmax=self._fused_argmax,
)
if self.simulate_fold:
# Before the TP broadcast in build_accept_outs: every rank commits rank 0's draw.
bonus = (
simulated_bonus_sample_graph(
target_logits=logits,
correct_len=self.simulated_correct_len.repeat(bs),
temperatures=self.sim_temperatures_buf[:bs],
seed=self.sim_seed_buf,
bs=bs,
rows_per_request=self.stride,
)
.to(bonus.dtype)
.view_as(bonus)
)
accept = build_accept_outs(
correct_len=correct_len,
bonus=bonus,
@@ -365,6 +365,7 @@ class DSparkWorkerV2(BaseSpecWorker):
tp_sync=self._tp_sync,
fused_argmax=target_is_dsv41,
simulated_correct_len=self._simulated_correct_len,
simulate_fold=_is_hip and envs.SGLANG_DSPARK_SIMULATE_FOLD.get(),
commit_ctx=CommitInjectCtx(
draft_model=self.draft_model,
block_pos_offsets=self._block_pos_offsets,
@@ -932,11 +933,24 @@ class DSparkWorkerV2(BaseSpecWorker):
# draft proposal itself folded.
and (sampling_info is None or sampling_info.is_all_greedy)
and verify_logits_adjustments_are_noop(sampling_info)
# The graph accept is greedy, so a sampled simulated bonus is eager.
and simulate_bonus_sampling_info is None
# The graph accept is greedy: a sampled simulated bonus stays in the graph
# only when the epilogue draws it there (SGLANG_DSPARK_SIMULATE_FOLD).
and (
simulate_bonus_sampling_info is None
or self._verify_executor.verify_epilogue.simulate_fold
)
and not batch.has_grammar
)
self._verify_executor.begin_simulated_step()
if fold_eligible and self._verify_executor.verify_epilogue.simulate_fold:
self._verify_executor.verify_epilogue.begin_sim_bonus(
bs,
(
None
if simulate_bonus_sampling_info is None
else simulate_bonus_sampling_info.temperatures[:bs]
),
)
prepare_mamba_track_for_verify(batch)
with self._observers.segment(InfoSegment.TARGET_VERIFY):
if run_compact: