[DSpark] Add MarkovSampleStep, one op for a markov draft step's add and token pick (#43621)
This commit is contained in:
@@ -0,0 +1,446 @@
|
||||
"""One Markov draft step: add the step bias and pick one token per row, the
|
||||
argmax at temperature 0 and a Gumbel-max sample otherwise."""
|
||||
|
||||
import functools
|
||||
from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
from triton.language.extra import libdevice
|
||||
|
||||
from sglang.kernels.ops.speculative.dspark.dispatch import inputs_on_cuda
|
||||
|
||||
# Measured fastest for 1-64 rows, where the per-block Philox work sets latency.
|
||||
_BLOCK = 1024
|
||||
_NO_INDEX = tl.constexpr(2147483647)
|
||||
# 2**-32: maps a 32-bit Philox word to a uniform in (0, 1].
|
||||
_UNIFORM_SCALE = tl.constexpr(2.3283064365386963e-10)
|
||||
_PHILOX_ROUNDS = 10
|
||||
_ROUND_CODE = {torch.float32: 0, torch.bfloat16: 1, torch.float16: 2}
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _round_to_step_dtype(value, ROUND: tl.constexpr):
|
||||
if ROUND == 1:
|
||||
value = value.to(tl.bfloat16).to(tl.float32)
|
||||
if ROUND == 2:
|
||||
value = value.to(tl.float16).to(tl.float32)
|
||||
return value
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _markov_sample_partial_kernel(
|
||||
B,
|
||||
X,
|
||||
T,
|
||||
OFFSET,
|
||||
L,
|
||||
P,
|
||||
BS,
|
||||
XS,
|
||||
LS,
|
||||
WIDTH,
|
||||
PADDED,
|
||||
VOCAB_START,
|
||||
PARTS,
|
||||
KEY,
|
||||
STEP,
|
||||
BLOCK: tl.constexpr,
|
||||
ROUND: tl.constexpr,
|
||||
HAS_BIAS: tl.constexpr,
|
||||
STORE: tl.constexpr,
|
||||
SELECT: tl.constexpr,
|
||||
SAMPLE: tl.constexpr,
|
||||
PHILOX_ROUNDS: tl.constexpr,
|
||||
):
|
||||
row, part = tl.program_id(0), tl.program_id(1)
|
||||
i = part * BLOCK + tl.arange(0, BLOCK)
|
||||
valid = i < WIDTH
|
||||
in_shard = i < PADDED
|
||||
value = tl.load(X + row * XS + i, in_shard, 0).to(tl.float32)
|
||||
if HAS_BIAS:
|
||||
value += tl.load(B + row * BS + i, valid, 0).to(tl.float32)
|
||||
value = _round_to_step_dtype(value, ROUND)
|
||||
if STORE:
|
||||
tl.store(L + row * LS + i, value.to(L.dtype.element_ty), in_shard)
|
||||
if SELECT:
|
||||
_select_block(
|
||||
value,
|
||||
T,
|
||||
OFFSET,
|
||||
P,
|
||||
row,
|
||||
part,
|
||||
i,
|
||||
valid,
|
||||
PARTS,
|
||||
KEY,
|
||||
STEP,
|
||||
VOCAB_START,
|
||||
BLOCK,
|
||||
SAMPLE,
|
||||
PHILOX_ROUNDS,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _select_block(
|
||||
value,
|
||||
T,
|
||||
OFFSET,
|
||||
P,
|
||||
row,
|
||||
part,
|
||||
i,
|
||||
valid,
|
||||
PARTS,
|
||||
KEY,
|
||||
STEP,
|
||||
VOCAB_START,
|
||||
BLOCK: tl.constexpr,
|
||||
SAMPLE: tl.constexpr,
|
||||
PHILOX_ROUNDS: tl.constexpr,
|
||||
):
|
||||
key = value
|
||||
if SAMPLE:
|
||||
temperature = tl.load(T + row)
|
||||
else:
|
||||
temperature = 0.0
|
||||
if temperature != 0.0:
|
||||
# One Philox call yields four words; word k goes to token 4 * q + k.
|
||||
quad = (part * BLOCK + VOCAB_START) // 4 + tl.arange(0, BLOCK // 4)
|
||||
lane = tl.zeros([BLOCK // 4], tl.int64)
|
||||
w0, w1, w2, w3 = tl.philox(
|
||||
KEY,
|
||||
quad.to(tl.uint32),
|
||||
(lane + row).to(tl.uint32),
|
||||
(lane + STEP).to(tl.uint32),
|
||||
((lane + tl.load(OFFSET)) & 0xFFFFFFFF).to(tl.uint32),
|
||||
PHILOX_ROUNDS,
|
||||
)
|
||||
bits = tl.interleave(tl.interleave(w0, w2), tl.interleave(w1, w3))
|
||||
# Small words convert to FP32 exactly, so the winning small noise stays precise.
|
||||
uniform = (bits.to(tl.float32) + 0.5) * _UNIFORM_SCALE
|
||||
exp_noise = -libdevice.log1p(-uniform)
|
||||
key = value / temperature - tl.log(exp_noise)
|
||||
nan = valid & (key != key)
|
||||
has_nan = tl.sum(nan.to(tl.int32), 0) > 0
|
||||
maximum = tl.max(tl.where(valid & ~nan, key, -float("inf")), 0)
|
||||
wins = tl.where(has_nan, nan, valid & (key == maximum))
|
||||
idx = tl.min(tl.where(wins, i + VOCAB_START, _NO_INDEX), 0)
|
||||
maximum = tl.where(has_nan, float("nan"), maximum)
|
||||
tl.store(P + (row * PARTS + part) * 2, maximum)
|
||||
tl.store(P + (row * PARTS + part) * 2 + 1, idx.to(tl.float32, bitcast=True))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _markov_sample_finish_kernel(
|
||||
P,
|
||||
OUT,
|
||||
BS,
|
||||
PARTS: tl.constexpr,
|
||||
WORLD: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
i = tl.arange(0, BLOCK)
|
||||
valid = i < PARTS * WORLD
|
||||
rank, part = i // PARTS, i % PARTS
|
||||
offset = ((rank * BS + row) * PARTS + part) * 2
|
||||
value = tl.load(P + offset, valid, -float("inf"))
|
||||
idx = tl.load(P + offset + 1, valid, 0).to(tl.int32, bitcast=True)
|
||||
# The NVLink push transport changes +0 to -0 as its Lamport sentinel.
|
||||
# Indices are nonnegative int32 bits, so strip that sign bit to recover ID 0.
|
||||
idx &= _NO_INDEX
|
||||
valid &= idx != _NO_INDEX
|
||||
nan = valid & (value != value)
|
||||
has_nan = tl.sum(nan.to(tl.int32), 0) > 0
|
||||
maximum = tl.max(tl.where(valid & ~nan, value, -float("inf")), 0)
|
||||
wins = tl.where(has_nan, nan, valid & (value == maximum))
|
||||
result = tl.min(tl.where(wins, idx, _NO_INDEX), 0)
|
||||
tl.store(OUT + row, result.to(tl.int64))
|
||||
|
||||
|
||||
def _mul32(a: torch.Tensor, b: int) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# (high, low) 32-bit words of a * b; b splits in halves so int64 never overflows.
|
||||
product_lo = a * (b & 0xFFFF)
|
||||
product_hi = a * (b >> 16)
|
||||
high = (product_hi + (product_lo >> 16)) >> 16
|
||||
low = (((product_hi & 0xFFFF) << 16) + product_lo) & 0xFFFFFFFF
|
||||
return high, low
|
||||
|
||||
|
||||
def _philox(key: int, c0, c1, c2, c3) -> tuple[torch.Tensor, ...]:
|
||||
"""Philox4x32-10, the same function as ``tl.philox``."""
|
||||
k0, k1 = key & 0xFFFFFFFF, (key >> 32) & 0xFFFFFFFF
|
||||
for _ in range(_PHILOX_ROUNDS):
|
||||
hi_b, lo_b = _mul32(c2, 0xCD9E8D57)
|
||||
hi_a, lo_a = _mul32(c0, 0xD2511F53)
|
||||
c0, c1, c2, c3 = hi_b ^ c1 ^ k0, lo_b, hi_a ^ c3 ^ k1, lo_a
|
||||
k0 = (k0 + 0x9E3779B9) & 0xFFFFFFFF
|
||||
k1 = (k1 + 0xBB67AE85) & 0xFFFFFFFF
|
||||
return c0, c1, c2, c3
|
||||
|
||||
|
||||
def _launch_partial(
|
||||
base_logits,
|
||||
bias,
|
||||
*,
|
||||
width,
|
||||
store,
|
||||
temperatures,
|
||||
key,
|
||||
offset,
|
||||
step,
|
||||
step_dtype,
|
||||
has_sampled_rows,
|
||||
vocab_start=0,
|
||||
select=True,
|
||||
):
|
||||
# Returns the [rows, parts * 2] (value, index bits) candidates of each block.
|
||||
rows, padded = base_logits.shape
|
||||
# The padded width fixes the transport shape on every rank; WIDTH masks padding.
|
||||
parts = triton.cdiv(padded, _BLOCK)
|
||||
assert parts > 0
|
||||
partial = torch.empty(
|
||||
(rows, parts, 2), device=base_logits.device, dtype=torch.float32
|
||||
)
|
||||
_markov_sample_partial_kernel[(rows, parts)](
|
||||
base_logits if bias is None else bias,
|
||||
base_logits,
|
||||
temperatures,
|
||||
offset,
|
||||
base_logits if store is None else store,
|
||||
partial,
|
||||
0 if bias is None else bias.stride(0),
|
||||
base_logits.stride(0),
|
||||
0 if store is None else store.stride(0),
|
||||
width,
|
||||
padded,
|
||||
vocab_start,
|
||||
parts,
|
||||
key,
|
||||
step,
|
||||
_BLOCK,
|
||||
_ROUND_CODE[step_dtype],
|
||||
bias is not None,
|
||||
store is not None,
|
||||
select,
|
||||
has_sampled_rows is not False,
|
||||
_PHILOX_ROUNDS,
|
||||
num_warps=4,
|
||||
)
|
||||
return partial.view(rows, parts * 2)
|
||||
|
||||
|
||||
class MarkovSampleStep:
|
||||
"""Returns global token ids. ``bias`` covers the first ``width`` columns of
|
||||
the rank's padded shard ``base_logits``. Sharded, ``gather`` exchanges
|
||||
per-block candidates, or ``gather_logits`` the kept ``logits_out`` whole.
|
||||
``has_sampled_rows=False`` argmaxes every row."""
|
||||
|
||||
@classmethod
|
||||
def execute(cls, **kwargs) -> torch.Tensor:
|
||||
if inputs_on_cuda(**kwargs):
|
||||
return cls.triton(**kwargs)
|
||||
return cls.torch(**kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _check(
|
||||
base_logits,
|
||||
bias,
|
||||
temperatures,
|
||||
offset,
|
||||
step_dtype,
|
||||
vocab_start,
|
||||
gather,
|
||||
gather_logits,
|
||||
logits_out,
|
||||
):
|
||||
assert base_logits.ndim == bias.ndim == 2
|
||||
assert bias.shape[0] == base_logits.shape[0]
|
||||
assert bias.shape[1] <= base_logits.shape[1]
|
||||
assert bias.stride(1) == base_logits.stride(1) == 1
|
||||
assert temperatures.shape == (base_logits.shape[0],)
|
||||
assert temperatures.dtype == torch.float32 and temperatures.stride(0) == 1
|
||||
assert offset.dtype == torch.int64 and offset.numel() == 1
|
||||
assert step_dtype in _ROUND_CODE, step_dtype
|
||||
# Philox words go to tokens in quads of global ids.
|
||||
assert vocab_start % 4 == 0, vocab_start
|
||||
if logits_out is not None:
|
||||
assert logits_out.dtype == step_dtype and logits_out.stride(1) == 1
|
||||
if gather is None:
|
||||
assert logits_out.shape == base_logits.shape
|
||||
else:
|
||||
assert gather_logits is not None
|
||||
assert logits_out.shape[0] == base_logits.shape[0]
|
||||
|
||||
@classmethod
|
||||
def triton(
|
||||
cls,
|
||||
*,
|
||||
base_logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
key: int,
|
||||
offset: torch.Tensor,
|
||||
step: int,
|
||||
step_dtype: torch.dtype,
|
||||
vocab_start: int = 0,
|
||||
gather: Optional[Callable[[torch.Tensor], torch.Tensor]] = None,
|
||||
gather_logits: Optional[Callable[[torch.Tensor], torch.Tensor]] = None,
|
||||
logits_out: Optional[torch.Tensor] = None,
|
||||
has_sampled_rows: Optional[bool] = None,
|
||||
) -> torch.Tensor:
|
||||
cls._check(
|
||||
base_logits,
|
||||
bias,
|
||||
temperatures,
|
||||
offset,
|
||||
step_dtype,
|
||||
vocab_start,
|
||||
gather,
|
||||
gather_logits,
|
||||
logits_out,
|
||||
)
|
||||
rows, width = bias.shape
|
||||
launch = functools.partial(
|
||||
_launch_partial,
|
||||
temperatures=temperatures,
|
||||
key=key,
|
||||
offset=offset,
|
||||
step=step,
|
||||
step_dtype=step_dtype,
|
||||
has_sampled_rows=has_sampled_rows,
|
||||
)
|
||||
if gather is not None and logits_out is not None:
|
||||
local = torch.empty(
|
||||
base_logits.shape, device=base_logits.device, dtype=step_dtype
|
||||
)
|
||||
launch(base_logits, bias, width=width, store=local, select=False)
|
||||
full = gather_logits(local)[:, : logits_out.shape[1]]
|
||||
gathered = launch(full, None, width=full.shape[1], store=logits_out)
|
||||
else:
|
||||
partial = launch(
|
||||
base_logits,
|
||||
bias,
|
||||
width=width,
|
||||
store=logits_out,
|
||||
vocab_start=vocab_start,
|
||||
)
|
||||
gathered = partial if gather is None else gather(partial)
|
||||
parts = gathered.shape[1] // 2
|
||||
world = gathered.shape[0] // rows
|
||||
result = torch.empty(rows, device=base_logits.device, dtype=torch.int64)
|
||||
_markov_sample_finish_kernel[(rows,)](
|
||||
gathered,
|
||||
result,
|
||||
rows,
|
||||
parts,
|
||||
world,
|
||||
triton.next_power_of_2(parts * world),
|
||||
num_warps=4,
|
||||
)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def torch(
|
||||
cls,
|
||||
*,
|
||||
base_logits: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
temperatures: torch.Tensor,
|
||||
key: int,
|
||||
offset: torch.Tensor,
|
||||
step: int,
|
||||
step_dtype: torch.dtype,
|
||||
vocab_start: int = 0,
|
||||
gather: Optional[Callable[[torch.Tensor], torch.Tensor]] = None,
|
||||
gather_logits: Optional[Callable[[torch.Tensor], torch.Tensor]] = None,
|
||||
logits_out: Optional[torch.Tensor] = None,
|
||||
has_sampled_rows: Optional[bool] = None,
|
||||
) -> torch.Tensor:
|
||||
# None draws the noise for every row: reading the temperatures on the
|
||||
# host would break a graph capture.
|
||||
cls._check(
|
||||
base_logits,
|
||||
bias,
|
||||
temperatures,
|
||||
offset,
|
||||
step_dtype,
|
||||
vocab_start,
|
||||
gather,
|
||||
gather_logits,
|
||||
logits_out,
|
||||
)
|
||||
rows, width = bias.shape
|
||||
value = base_logits.to(torch.float32, copy=True)
|
||||
value[:, :width] += bias.float()
|
||||
value = value.to(step_dtype).float()
|
||||
if gather is not None and logits_out is not None:
|
||||
full = gather_logits(value.to(step_dtype))[:, : logits_out.shape[1]]
|
||||
return cls.torch(
|
||||
base_logits=full,
|
||||
bias=torch.zeros_like(full),
|
||||
temperatures=temperatures,
|
||||
key=key,
|
||||
offset=offset,
|
||||
step=step,
|
||||
step_dtype=step_dtype,
|
||||
logits_out=logits_out,
|
||||
has_sampled_rows=has_sampled_rows,
|
||||
)
|
||||
padded = base_logits.shape[1]
|
||||
device = base_logits.device
|
||||
if logits_out is not None:
|
||||
logits_out.copy_(value)
|
||||
|
||||
keys = value
|
||||
if has_sampled_rows is not False:
|
||||
quads = vocab_start // 4 + torch.arange(
|
||||
triton.cdiv(padded, 4), device=device, dtype=torch.int64
|
||||
)
|
||||
words = _philox(
|
||||
key,
|
||||
quads[None, :],
|
||||
torch.arange(rows, device=device, dtype=torch.int64)[:, None],
|
||||
torch.full((1, 1), step, device=device, dtype=torch.int64),
|
||||
offset.reshape(1, 1) & 0xFFFFFFFF,
|
||||
)
|
||||
words = torch.broadcast_tensors(*words)
|
||||
bits = torch.stack(words, dim=-1).reshape(rows, -1)[:, :padded]
|
||||
uniform = (bits.float() + 0.5) * _UNIFORM_SCALE.value
|
||||
exp_noise = -torch.log1p(-uniform)
|
||||
sampled = temperatures != 0
|
||||
safe_temperatures = torch.where(sampled, temperatures, 1.0)[:, None]
|
||||
noised = value / safe_temperatures - torch.log(exp_noise)
|
||||
keys = torch.where(sampled[:, None], noised, value)
|
||||
cols = torch.arange(padded, device=device)
|
||||
keys = torch.where(cols[None, :] < width, keys, -torch.inf)
|
||||
|
||||
if width == 0:
|
||||
local_values = torch.full((rows,), -torch.inf, device=device)
|
||||
local_indices = torch.full(
|
||||
(rows,), _NO_INDEX.value, dtype=torch.int32, device=device
|
||||
)
|
||||
else:
|
||||
local = keys.argmax(dim=1)
|
||||
local_values = keys.gather(1, local[:, None]).squeeze(1)
|
||||
local_indices = (local + vocab_start).to(torch.int32)
|
||||
partial = torch.stack(
|
||||
[local_values.float(), local_indices.view(torch.float32)], dim=1
|
||||
)
|
||||
gathered = partial if gather is None else gather(partial)
|
||||
gathered = gathered.view(-1, rows, 2)
|
||||
values = gathered[..., 0].transpose(0, 1)
|
||||
indices = gathered[..., 1].contiguous().view(torch.int32) & _NO_INDEX.value
|
||||
indices = indices.transpose(0, 1).long()
|
||||
# Lowest index among the NaNs if any, else among the maxima.
|
||||
valid = indices != _NO_INDEX.value
|
||||
nan = valid & values.isnan()
|
||||
has_nan = nan.any(dim=-1, keepdim=True)
|
||||
finite = torch.where(valid & ~nan, values, -torch.inf)
|
||||
maximum = finite.amax(dim=-1, keepdim=True)
|
||||
wins = torch.where(has_nan, nan, valid & (finite == maximum))
|
||||
return torch.where(wins, indices, _NO_INDEX.value).amin(dim=-1)
|
||||
@@ -1,109 +0,0 @@
|
||||
"""Exact greedy selection from TP-local base logits and rounded Markov bias."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sharded_greedy_partial_kernel(
|
||||
B,
|
||||
X,
|
||||
P,
|
||||
BS: tl.constexpr,
|
||||
XS: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
OFFSET: tl.constexpr,
|
||||
PARTS: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row, part = tl.program_id(0), tl.program_id(1)
|
||||
i = part * BLOCK + tl.arange(0, BLOCK)
|
||||
valid = i < WIDTH
|
||||
bias = tl.load(B + row * BS + i, valid, 0).to(tl.float32)
|
||||
base = tl.load(X + row * XS + i, valid, 0).to(tl.float32)
|
||||
value = base + bias
|
||||
nan = valid & (value != value)
|
||||
has_nan = tl.sum(nan.to(tl.int32), 0) > 0
|
||||
maximum = tl.max(tl.where(valid & ~nan, value, -float("inf")), 0)
|
||||
wins = tl.where(has_nan, nan, valid & (value == maximum))
|
||||
idx = tl.min(tl.where(wins, i + OFFSET, 2147483647), 0)
|
||||
maximum = tl.where(has_nan, float("nan"), maximum)
|
||||
tl.store(P + (row * PARTS + part) * 2, maximum)
|
||||
tl.store(P + (row * PARTS + part) * 2 + 1, idx.to(tl.float32, bitcast=True))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sharded_greedy_finish_kernel(
|
||||
P,
|
||||
OUT,
|
||||
BS: tl.constexpr,
|
||||
PARTS: tl.constexpr,
|
||||
WORLD: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
i = tl.arange(0, BLOCK)
|
||||
valid = i < PARTS * WORLD
|
||||
rank, part = i // PARTS, i % PARTS
|
||||
offset = ((rank * BS + row) * PARTS + part) * 2
|
||||
value = tl.load(P + offset, valid, -float("inf"))
|
||||
idx = tl.load(P + offset + 1, valid, 0).to(tl.int32, bitcast=True)
|
||||
# The NVLink push transport changes +0 to -0 as its Lamport sentinel.
|
||||
# Indices are nonnegative int32 bits, so strip that sign bit to recover ID 0.
|
||||
idx &= 2147483647
|
||||
valid &= idx != 2147483647
|
||||
nan = valid & (value != value)
|
||||
has_nan = tl.sum(nan.to(tl.int32), 0) > 0
|
||||
maximum = tl.max(tl.where(valid & ~nan, value, -float("inf")), 0)
|
||||
wins = tl.where(has_nan, nan, valid & (value == maximum))
|
||||
result = tl.min(tl.where(wins, idx, 2147483647), 0)
|
||||
tl.store(OUT + row, result.to(tl.int64))
|
||||
|
||||
|
||||
def sharded_greedy_step(bias, base_local, *, group, vocab_start, gather=None):
|
||||
"""Fused BuildStepLocal + vocab gather + argmax, without materializing logits.
|
||||
|
||||
Equivalent to argmax of rank-ordered ``build_step_local``/all_gather over the
|
||||
sharded vocab, excluding padding. The transport carries one (value,
|
||||
global-index-bits) pair per 4096-wide block of the shard per row; indices
|
||||
move as bits and are never converted numerically to float.
|
||||
"""
|
||||
assert bias.ndim == base_local.ndim == 2
|
||||
assert bias.shape[0] == base_local.shape[0]
|
||||
assert bias.shape[1] <= base_local.shape[1]
|
||||
assert bias.stride(1) == base_local.stride(1) == 1
|
||||
rows, width = bias.shape
|
||||
block = 4096
|
||||
parts = triton.cdiv(base_local.shape[1], block)
|
||||
assert parts > 0
|
||||
partial = torch.empty((rows, parts, 2), device=bias.device, dtype=torch.float32)
|
||||
_sharded_greedy_partial_kernel[(rows, parts)](
|
||||
bias,
|
||||
base_local,
|
||||
partial,
|
||||
bias.stride(0),
|
||||
base_local.stride(0),
|
||||
width,
|
||||
vocab_start,
|
||||
parts,
|
||||
block,
|
||||
num_warps=4,
|
||||
)
|
||||
# The padded partition width fixes the transport shape on all ranks;
|
||||
# WIDTH masks real entries, including a completely empty final shard.
|
||||
if gather is not None:
|
||||
gathered = gather(partial.view(rows, parts * 2))
|
||||
else:
|
||||
gathered = group.all_gather(partial, dim=0) if group.world_size > 1 else partial
|
||||
result = torch.empty(rows, device=bias.device, dtype=torch.int64)
|
||||
_sharded_greedy_finish_kernel[(rows,)](
|
||||
gathered,
|
||||
result,
|
||||
rows,
|
||||
parts,
|
||||
group.world_size,
|
||||
triton.next_power_of_2(parts * group.world_size),
|
||||
num_warps=4,
|
||||
)
|
||||
return result
|
||||
@@ -550,7 +550,8 @@ class Envs:
|
||||
SGLANG_DSPARK_OPT_MARKOV_W2_BF16 = EnvBool(True)
|
||||
SGLANG_DSPARK_OPT_MARKOV_W2_TP_SHARD = EnvBool(True)
|
||||
SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV = EnvBool(False)
|
||||
# With the TP-sharded markov_w2, gather each step's vocab-parallel logits over
|
||||
# With the TP-sharded markov_w2, run the draft's vocab all-gathers (each
|
||||
# step's per-shard token candidates, and the step logits sampling keeps) over
|
||||
# the NVLink push collective (CustomAllReduceV2's multicast plane) instead of
|
||||
# the NCCL ring. Only taken when the group's communicator has a multicast
|
||||
# plane; off, or no such plane, keeps the NCCL all-gather.
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_draft_model import (
|
||||
BuildStepLocal,
|
||||
CommitKvProj,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep
|
||||
from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config
|
||||
from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_gather
|
||||
from sglang.srt.environ import envs
|
||||
@@ -522,21 +523,25 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
|
||||
def sample_block_greedy_fused(self, base_logits, *, first_prev_tokens):
|
||||
if not self.supports_sharded_greedy or not base_logits.is_cuda:
|
||||
return None
|
||||
from sglang.kernels.ops.speculative.dspark.sharded_greedy import (
|
||||
sharded_greedy_step,
|
||||
)
|
||||
|
||||
shard = self._tp_shard
|
||||
prev = first_prev_tokens.long()
|
||||
# The greedy-only variant reads neither, so nothing fills them.
|
||||
greedy = torch.empty(prev.shape[0], dtype=torch.float32, device=prev.device)
|
||||
offset = torch.empty(1, dtype=torch.int64, device=prev.device)
|
||||
tokens = []
|
||||
for step in range(base_logits.shape[1]):
|
||||
bias, _ = self.step_bias(prev, hidden_states=None, state=None)
|
||||
prev = sharded_greedy_step(
|
||||
bias,
|
||||
base_logits[:, step],
|
||||
group=self._shard_group,
|
||||
prev = MarkovSampleStep.triton(
|
||||
base_logits=base_logits[:, step],
|
||||
bias=bias,
|
||||
temperatures=greedy,
|
||||
key=0,
|
||||
offset=offset,
|
||||
step=step,
|
||||
step_dtype=torch.float32,
|
||||
vocab_start=shard.org_vocab_start,
|
||||
gather=self._vocab_gather.gather_stacked,
|
||||
has_sampled_rows=False,
|
||||
)
|
||||
tokens.append(prev)
|
||||
return torch.stack(tokens, dim=1)
|
||||
|
||||
Reference in New Issue
Block a user