[Perf] [AMD] dspark: No per-step host sync in decode (1/4) (#44405)
Co-authored-by: wunhuang <wunhuang@amd.com>
This commit is contained in:
@@ -60,6 +60,99 @@ def _simulated_bonus_select_kernel(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _simulated_bonus_row_partial_kernel(
|
||||
logits_ptr,
|
||||
logits_stride,
|
||||
temperatures_ptr,
|
||||
correct_len_ptr,
|
||||
seed_ptr,
|
||||
part_val_ptr,
|
||||
part_idx_ptr,
|
||||
vocab,
|
||||
ROWS: tl.constexpr,
|
||||
NSPLIT: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
req = tl.program_id(0)
|
||||
split = tl.program_id(1)
|
||||
row = req * ROWS + tl.load(correct_len_ptr + req)
|
||||
t = tl.load(temperatures_ptr + req).to(tl.float32)
|
||||
offs = split * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < vocab
|
||||
x = tl.load(
|
||||
logits_ptr + row.to(tl.int64) * logits_stride + offs,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
seed = tl.load(seed_ptr)
|
||||
u = tl.rand(seed, req * vocab + offs)
|
||||
noisy = x / tl.maximum(t, 1e-5) - tl.log(-tl.log(u))
|
||||
v = tl.where(mask, tl.where(t > 0.0, noisy, x), float("-inf"))
|
||||
top = tl.max(v, axis=0)
|
||||
idx = tl.min(tl.where(v == top, offs, vocab), axis=0)
|
||||
tl.store(part_val_ptr + req * NSPLIT + split, top)
|
||||
tl.store(part_idx_ptr + req * NSPLIT + split, idx)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _simulated_bonus_row_select_kernel(
|
||||
part_val_ptr,
|
||||
part_idx_ptr,
|
||||
out_ptr,
|
||||
vocab,
|
||||
NSPLIT: tl.constexpr,
|
||||
):
|
||||
req = tl.program_id(0)
|
||||
s = tl.arange(0, NSPLIT)
|
||||
val = tl.load(part_val_ptr + req * NSPLIT + s)
|
||||
idx = tl.load(part_idx_ptr + req * NSPLIT + s)
|
||||
top = tl.max(val, axis=0)
|
||||
tl.store(
|
||||
out_ptr + req, tl.min(tl.where(val == top, idx, vocab), axis=0).to(tl.int64)
|
||||
)
|
||||
|
||||
|
||||
def simulated_bonus_sample_graph(
|
||||
*,
|
||||
target_logits: torch.Tensor,
|
||||
correct_len: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
seed: torch.Tensor,
|
||||
bs: int,
|
||||
rows_per_request: int,
|
||||
) -> torch.Tensor:
|
||||
"""Graph-safe variant of simulated_bonus_sample: seed ([1] int) and per-request
|
||||
temperatures ([bs] float; <= 0 means argmax) are read from device buffers, and
|
||||
only the row at ``correct_len`` is sampled. [bs] int64."""
|
||||
assert target_logits.stride(1) == 1
|
||||
device = target_logits.device
|
||||
vocab = target_logits.shape[1]
|
||||
block = 4096
|
||||
nsplit = triton.next_power_of_2(triton.cdiv(vocab, block))
|
||||
part_val = torch.empty((bs, nsplit), dtype=torch.float32, device=device)
|
||||
part_idx = torch.empty((bs, nsplit), dtype=torch.int32, device=device)
|
||||
_simulated_bonus_row_partial_kernel[(bs, nsplit)](
|
||||
target_logits,
|
||||
target_logits.stride(0),
|
||||
temperatures,
|
||||
correct_len.contiguous(),
|
||||
seed,
|
||||
part_val,
|
||||
part_idx,
|
||||
vocab,
|
||||
ROWS=rows_per_request,
|
||||
NSPLIT=nsplit,
|
||||
BLOCK=block,
|
||||
num_warps=8,
|
||||
)
|
||||
out = torch.empty((bs,), dtype=torch.int64, device=device)
|
||||
_simulated_bonus_row_select_kernel[(bs,)](
|
||||
part_val, part_idx, out, vocab, NSPLIT=nsplit
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def simulated_bonus_sample(
|
||||
*,
|
||||
target_logits: torch.Tensor,
|
||||
|
||||
@@ -519,6 +519,9 @@ class Envs:
|
||||
# DSpark on HIP: with SGLANG_SIMULATE_ACC_LEN set, temperature-only sampling requests
|
||||
# run the greedy draft/accept path and only the bonus token is temperature-sampled.
|
||||
SGLANG_SIMULATE_ACC_GREEDY = EnvBool(True)
|
||||
# DSpark on HIP: apply the simulated correct_len / bonus inside the folded verify epilogue
|
||||
# (hip graph) instead of falling back to the eager accept + commit path.
|
||||
SGLANG_DSPARK_SIMULATE_FOLD = EnvBool(True)
|
||||
SGLANG_SIMULATE_UNIFORM_EXPERTS = EnvBool(False)
|
||||
SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS = EnvBool(False)
|
||||
|
||||
|
||||
@@ -541,7 +541,9 @@ class FutureMap:
|
||||
int(x) for x in fresh_cpu.tolist()
|
||||
]
|
||||
|
||||
def resolve_seq_lens_cpu(self, batch: ScheduleBatch) -> None:
|
||||
def resolve_seq_lens_cpu(
|
||||
self, batch: ScheduleBatch, stream_ordered: bool = False
|
||||
) -> None:
|
||||
# Lazy pull from new_seq_lens_buf for spec_v2 (accept_lens not known to
|
||||
# schedule). The CPU mirror is gated by needs_cpu_seq_lens; backends that
|
||||
# opt out take the GPU-only path below. A private D2H stream overlaps the copy.
|
||||
@@ -552,7 +554,9 @@ class FutureMap:
|
||||
fi = draft_input.future_indices
|
||||
if fi is None:
|
||||
return
|
||||
if self.publish_ready is not None:
|
||||
if self.publish_ready is not None and stream_ordered:
|
||||
self._publish_fresh = False
|
||||
elif self.publish_ready is not None:
|
||||
if _DEBUG_ASSERT:
|
||||
# Consume-once: every event wait must be re-armed by a fresh
|
||||
# forward publish; a stale consume means a publish went missing.
|
||||
|
||||
@@ -4572,12 +4572,22 @@ class Scheduler(
|
||||
if self.enable_overlap:
|
||||
# Self-gates on batch.spec_info.future_indices; non-spec_v2
|
||||
# no-ops (ForwardBatch.init_new lazily computes the sum).
|
||||
self.future_map.resolve_seq_lens_cpu(batch)
|
||||
gpu_only_lens = (
|
||||
_is_hip
|
||||
and self._confidence_budget_prepare is None
|
||||
and not getattr(self.future_map, "needs_cpu_seq_lens", True)
|
||||
)
|
||||
if not gpu_only_lens:
|
||||
self.future_map.resolve_seq_lens_cpu(batch)
|
||||
if self._confidence_budget_prepare is not None:
|
||||
self._confidence_budget_prepare(batch, self.future_map)
|
||||
|
||||
with self.forward_stream_ctx:
|
||||
self.forward_stream.wait_stream(self.schedule_stream)
|
||||
if gpu_only_lens:
|
||||
# publish_ready was recorded on forward_stream: stream order
|
||||
# fences the gather, no host sync / event wait needed.
|
||||
self.future_map.resolve_seq_lens_cpu(batch, stream_ordered=True)
|
||||
# resolve consumes SB staging (prefill_input_ids_cpu /
|
||||
# mix_running_indices). Run OUTSIDE isolation so the
|
||||
# snapshot captures the post-consume state — restoring
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
@@ -26,6 +27,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.simulated_bonus import (
|
||||
simulated_bonus_sample,
|
||||
simulated_bonus_sample_graph,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
@@ -537,8 +539,18 @@ class DsparkVerifyEpilogue:
|
||||
commit_ctx: Optional[CommitInjectCtx] = None,
|
||||
fused_argmax: bool = False,
|
||||
simulated_correct_len: Optional[torch.Tensor] = None,
|
||||
simulate_fold: bool = False,
|
||||
) -> None:
|
||||
self.max_bs = int(max_bs)
|
||||
# SGLANG_SIMULATE_ACC_LEN with a sampled bonus inside the graph too: the bonus
|
||||
# is drawn at the simulated correct_len from these buffers, which
|
||||
# begin_sim_bonus writes before replay (<= 0 temperature: argmax).
|
||||
self.simulate_fold = bool(simulate_fold) and simulated_correct_len is not None
|
||||
self.sim_temperatures_buf = torch.zeros(
|
||||
(self.max_bs,), dtype=torch.float32, device=device
|
||||
)
|
||||
self.sim_seed_buf = torch.zeros((1,), dtype=torch.int32, device=device)
|
||||
self._sim_greedy_bs: Optional[int] = None
|
||||
self.stride = int(verify_num_draft_tokens)
|
||||
self._fused_argmax = bool(fused_argmax)
|
||||
self.gamma = self.stride - 1
|
||||
@@ -617,6 +629,17 @@ class DsparkVerifyEpilogue:
|
||||
self.inject_gate_buf.fill_(int(armed))
|
||||
self._static_step_state = state
|
||||
|
||||
def begin_sim_bonus(self, bs: int, temperatures: Optional[torch.Tensor]) -> None:
|
||||
"""The step's bonus temperatures (None: greedy) and seed, before replay."""
|
||||
if temperatures is None:
|
||||
if self._sim_greedy_bs != bs:
|
||||
self.sim_temperatures_buf.zero_()
|
||||
self._sim_greedy_bs = bs
|
||||
return
|
||||
self._sim_greedy_bs = None
|
||||
self.sim_temperatures_buf[:bs].copy_(temperatures.view(-1))
|
||||
self.sim_seed_buf.fill_(random.getrandbits(31))
|
||||
|
||||
def _static_epilogue(self, out, forward_batch) -> None:
|
||||
bs = forward_batch.batch_size
|
||||
verify_lens = self.verify_lens_buf[:bs]
|
||||
@@ -751,6 +774,20 @@ class DsparkVerifyEpilogue:
|
||||
cutoff_verify_lens=cutoff_verify_lens,
|
||||
fused_argmax=self._fused_argmax,
|
||||
)
|
||||
if self.simulate_fold:
|
||||
# Before the TP broadcast in build_accept_outs: every rank commits rank 0's draw.
|
||||
bonus = (
|
||||
simulated_bonus_sample_graph(
|
||||
target_logits=logits,
|
||||
correct_len=self.simulated_correct_len.repeat(bs),
|
||||
temperatures=self.sim_temperatures_buf[:bs],
|
||||
seed=self.sim_seed_buf,
|
||||
bs=bs,
|
||||
rows_per_request=self.stride,
|
||||
)
|
||||
.to(bonus.dtype)
|
||||
.view_as(bonus)
|
||||
)
|
||||
accept = build_accept_outs(
|
||||
correct_len=correct_len,
|
||||
bonus=bonus,
|
||||
|
||||
@@ -365,6 +365,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
tp_sync=self._tp_sync,
|
||||
fused_argmax=target_is_dsv41,
|
||||
simulated_correct_len=self._simulated_correct_len,
|
||||
simulate_fold=_is_hip and envs.SGLANG_DSPARK_SIMULATE_FOLD.get(),
|
||||
commit_ctx=CommitInjectCtx(
|
||||
draft_model=self.draft_model,
|
||||
block_pos_offsets=self._block_pos_offsets,
|
||||
@@ -932,11 +933,24 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
# draft proposal itself folded.
|
||||
and (sampling_info is None or sampling_info.is_all_greedy)
|
||||
and verify_logits_adjustments_are_noop(sampling_info)
|
||||
# The graph accept is greedy, so a sampled simulated bonus is eager.
|
||||
and simulate_bonus_sampling_info is None
|
||||
# The graph accept is greedy: a sampled simulated bonus stays in the graph
|
||||
# only when the epilogue draws it there (SGLANG_DSPARK_SIMULATE_FOLD).
|
||||
and (
|
||||
simulate_bonus_sampling_info is None
|
||||
or self._verify_executor.verify_epilogue.simulate_fold
|
||||
)
|
||||
and not batch.has_grammar
|
||||
)
|
||||
self._verify_executor.begin_simulated_step()
|
||||
if fold_eligible and self._verify_executor.verify_epilogue.simulate_fold:
|
||||
self._verify_executor.verify_epilogue.begin_sim_bonus(
|
||||
bs,
|
||||
(
|
||||
None
|
||||
if simulate_bonus_sampling_info is None
|
||||
else simulate_bonus_sampling_info.temperatures[:bs]
|
||||
),
|
||||
)
|
||||
prepare_mamba_track_for_verify(batch)
|
||||
with self._observers.segment(InfoSegment.TARGET_VERIFY):
|
||||
if run_compact:
|
||||
|
||||
Reference in New Issue
Block a user