diff --git a/python/sglang/kernels/ops/speculative/dspark/markov_sample.py b/python/sglang/kernels/ops/speculative/dspark/markov_sample.py new file mode 100644 index 0000000000..5ed218f624 --- /dev/null +++ b/python/sglang/kernels/ops/speculative/dspark/markov_sample.py @@ -0,0 +1,446 @@ +"""One Markov draft step: add the step bias and pick one token per row, the +argmax at temperature 0 and a Gumbel-max sample otherwise.""" + +import functools +from typing import Callable, Optional + +import torch +import triton +import triton.language as tl +from triton.language.extra import libdevice + +from sglang.kernels.ops.speculative.dspark.dispatch import inputs_on_cuda + +# Measured fastest for 1-64 rows, where the per-block Philox work sets latency. +_BLOCK = 1024 +_NO_INDEX = tl.constexpr(2147483647) +# 2**-32: maps a 32-bit Philox word to a uniform in (0, 1]. +_UNIFORM_SCALE = tl.constexpr(2.3283064365386963e-10) +_PHILOX_ROUNDS = 10 +_ROUND_CODE = {torch.float32: 0, torch.bfloat16: 1, torch.float16: 2} + + +@triton.jit +def _round_to_step_dtype(value, ROUND: tl.constexpr): + if ROUND == 1: + value = value.to(tl.bfloat16).to(tl.float32) + if ROUND == 2: + value = value.to(tl.float16).to(tl.float32) + return value + + +@triton.jit +def _markov_sample_partial_kernel( + B, + X, + T, + OFFSET, + L, + P, + BS, + XS, + LS, + WIDTH, + PADDED, + VOCAB_START, + PARTS, + KEY, + STEP, + BLOCK: tl.constexpr, + ROUND: tl.constexpr, + HAS_BIAS: tl.constexpr, + STORE: tl.constexpr, + SELECT: tl.constexpr, + SAMPLE: tl.constexpr, + PHILOX_ROUNDS: tl.constexpr, +): + row, part = tl.program_id(0), tl.program_id(1) + i = part * BLOCK + tl.arange(0, BLOCK) + valid = i < WIDTH + in_shard = i < PADDED + value = tl.load(X + row * XS + i, in_shard, 0).to(tl.float32) + if HAS_BIAS: + value += tl.load(B + row * BS + i, valid, 0).to(tl.float32) + value = _round_to_step_dtype(value, ROUND) + if STORE: + tl.store(L + row * LS + i, value.to(L.dtype.element_ty), in_shard) + if SELECT: + _select_block( + value, + T, + OFFSET, + P, + row, + part, + i, + valid, + PARTS, + KEY, + STEP, + VOCAB_START, + BLOCK, + SAMPLE, + PHILOX_ROUNDS, + ) + + +@triton.jit +def _select_block( + value, + T, + OFFSET, + P, + row, + part, + i, + valid, + PARTS, + KEY, + STEP, + VOCAB_START, + BLOCK: tl.constexpr, + SAMPLE: tl.constexpr, + PHILOX_ROUNDS: tl.constexpr, +): + key = value + if SAMPLE: + temperature = tl.load(T + row) + else: + temperature = 0.0 + if temperature != 0.0: + # One Philox call yields four words; word k goes to token 4 * q + k. + quad = (part * BLOCK + VOCAB_START) // 4 + tl.arange(0, BLOCK // 4) + lane = tl.zeros([BLOCK // 4], tl.int64) + w0, w1, w2, w3 = tl.philox( + KEY, + quad.to(tl.uint32), + (lane + row).to(tl.uint32), + (lane + STEP).to(tl.uint32), + ((lane + tl.load(OFFSET)) & 0xFFFFFFFF).to(tl.uint32), + PHILOX_ROUNDS, + ) + bits = tl.interleave(tl.interleave(w0, w2), tl.interleave(w1, w3)) + # Small words convert to FP32 exactly, so the winning small noise stays precise. + uniform = (bits.to(tl.float32) + 0.5) * _UNIFORM_SCALE + exp_noise = -libdevice.log1p(-uniform) + key = value / temperature - tl.log(exp_noise) + nan = valid & (key != key) + has_nan = tl.sum(nan.to(tl.int32), 0) > 0 + maximum = tl.max(tl.where(valid & ~nan, key, -float("inf")), 0) + wins = tl.where(has_nan, nan, valid & (key == maximum)) + idx = tl.min(tl.where(wins, i + VOCAB_START, _NO_INDEX), 0) + maximum = tl.where(has_nan, float("nan"), maximum) + tl.store(P + (row * PARTS + part) * 2, maximum) + tl.store(P + (row * PARTS + part) * 2 + 1, idx.to(tl.float32, bitcast=True)) + + +@triton.jit +def _markov_sample_finish_kernel( + P, + OUT, + BS, + PARTS: tl.constexpr, + WORLD: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + i = tl.arange(0, BLOCK) + valid = i < PARTS * WORLD + rank, part = i // PARTS, i % PARTS + offset = ((rank * BS + row) * PARTS + part) * 2 + value = tl.load(P + offset, valid, -float("inf")) + idx = tl.load(P + offset + 1, valid, 0).to(tl.int32, bitcast=True) + # The NVLink push transport changes +0 to -0 as its Lamport sentinel. + # Indices are nonnegative int32 bits, so strip that sign bit to recover ID 0. + idx &= _NO_INDEX + valid &= idx != _NO_INDEX + nan = valid & (value != value) + has_nan = tl.sum(nan.to(tl.int32), 0) > 0 + maximum = tl.max(tl.where(valid & ~nan, value, -float("inf")), 0) + wins = tl.where(has_nan, nan, valid & (value == maximum)) + result = tl.min(tl.where(wins, idx, _NO_INDEX), 0) + tl.store(OUT + row, result.to(tl.int64)) + + +def _mul32(a: torch.Tensor, b: int) -> tuple[torch.Tensor, torch.Tensor]: + # (high, low) 32-bit words of a * b; b splits in halves so int64 never overflows. + product_lo = a * (b & 0xFFFF) + product_hi = a * (b >> 16) + high = (product_hi + (product_lo >> 16)) >> 16 + low = (((product_hi & 0xFFFF) << 16) + product_lo) & 0xFFFFFFFF + return high, low + + +def _philox(key: int, c0, c1, c2, c3) -> tuple[torch.Tensor, ...]: + """Philox4x32-10, the same function as ``tl.philox``.""" + k0, k1 = key & 0xFFFFFFFF, (key >> 32) & 0xFFFFFFFF + for _ in range(_PHILOX_ROUNDS): + hi_b, lo_b = _mul32(c2, 0xCD9E8D57) + hi_a, lo_a = _mul32(c0, 0xD2511F53) + c0, c1, c2, c3 = hi_b ^ c1 ^ k0, lo_b, hi_a ^ c3 ^ k1, lo_a + k0 = (k0 + 0x9E3779B9) & 0xFFFFFFFF + k1 = (k1 + 0xBB67AE85) & 0xFFFFFFFF + return c0, c1, c2, c3 + + +def _launch_partial( + base_logits, + bias, + *, + width, + store, + temperatures, + key, + offset, + step, + step_dtype, + has_sampled_rows, + vocab_start=0, + select=True, +): + # Returns the [rows, parts * 2] (value, index bits) candidates of each block. + rows, padded = base_logits.shape + # The padded width fixes the transport shape on every rank; WIDTH masks padding. + parts = triton.cdiv(padded, _BLOCK) + assert parts > 0 + partial = torch.empty( + (rows, parts, 2), device=base_logits.device, dtype=torch.float32 + ) + _markov_sample_partial_kernel[(rows, parts)]( + base_logits if bias is None else bias, + base_logits, + temperatures, + offset, + base_logits if store is None else store, + partial, + 0 if bias is None else bias.stride(0), + base_logits.stride(0), + 0 if store is None else store.stride(0), + width, + padded, + vocab_start, + parts, + key, + step, + _BLOCK, + _ROUND_CODE[step_dtype], + bias is not None, + store is not None, + select, + has_sampled_rows is not False, + _PHILOX_ROUNDS, + num_warps=4, + ) + return partial.view(rows, parts * 2) + + +class MarkovSampleStep: + """Returns global token ids. ``bias`` covers the first ``width`` columns of + the rank's padded shard ``base_logits``. Sharded, ``gather`` exchanges + per-block candidates, or ``gather_logits`` the kept ``logits_out`` whole. + ``has_sampled_rows=False`` argmaxes every row.""" + + @classmethod + def execute(cls, **kwargs) -> torch.Tensor: + if inputs_on_cuda(**kwargs): + return cls.triton(**kwargs) + return cls.torch(**kwargs) + + @staticmethod + def _check( + base_logits, + bias, + temperatures, + offset, + step_dtype, + vocab_start, + gather, + gather_logits, + logits_out, + ): + assert base_logits.ndim == bias.ndim == 2 + assert bias.shape[0] == base_logits.shape[0] + assert bias.shape[1] <= base_logits.shape[1] + assert bias.stride(1) == base_logits.stride(1) == 1 + assert temperatures.shape == (base_logits.shape[0],) + assert temperatures.dtype == torch.float32 and temperatures.stride(0) == 1 + assert offset.dtype == torch.int64 and offset.numel() == 1 + assert step_dtype in _ROUND_CODE, step_dtype + # Philox words go to tokens in quads of global ids. + assert vocab_start % 4 == 0, vocab_start + if logits_out is not None: + assert logits_out.dtype == step_dtype and logits_out.stride(1) == 1 + if gather is None: + assert logits_out.shape == base_logits.shape + else: + assert gather_logits is not None + assert logits_out.shape[0] == base_logits.shape[0] + + @classmethod + def triton( + cls, + *, + base_logits: torch.Tensor, + bias: torch.Tensor, + temperatures: torch.Tensor, + key: int, + offset: torch.Tensor, + step: int, + step_dtype: torch.dtype, + vocab_start: int = 0, + gather: Optional[Callable[[torch.Tensor], torch.Tensor]] = None, + gather_logits: Optional[Callable[[torch.Tensor], torch.Tensor]] = None, + logits_out: Optional[torch.Tensor] = None, + has_sampled_rows: Optional[bool] = None, + ) -> torch.Tensor: + cls._check( + base_logits, + bias, + temperatures, + offset, + step_dtype, + vocab_start, + gather, + gather_logits, + logits_out, + ) + rows, width = bias.shape + launch = functools.partial( + _launch_partial, + temperatures=temperatures, + key=key, + offset=offset, + step=step, + step_dtype=step_dtype, + has_sampled_rows=has_sampled_rows, + ) + if gather is not None and logits_out is not None: + local = torch.empty( + base_logits.shape, device=base_logits.device, dtype=step_dtype + ) + launch(base_logits, bias, width=width, store=local, select=False) + full = gather_logits(local)[:, : logits_out.shape[1]] + gathered = launch(full, None, width=full.shape[1], store=logits_out) + else: + partial = launch( + base_logits, + bias, + width=width, + store=logits_out, + vocab_start=vocab_start, + ) + gathered = partial if gather is None else gather(partial) + parts = gathered.shape[1] // 2 + world = gathered.shape[0] // rows + result = torch.empty(rows, device=base_logits.device, dtype=torch.int64) + _markov_sample_finish_kernel[(rows,)]( + gathered, + result, + rows, + parts, + world, + triton.next_power_of_2(parts * world), + num_warps=4, + ) + return result + + @classmethod + def torch( + cls, + *, + base_logits: torch.Tensor, + bias: torch.Tensor, + temperatures: torch.Tensor, + key: int, + offset: torch.Tensor, + step: int, + step_dtype: torch.dtype, + vocab_start: int = 0, + gather: Optional[Callable[[torch.Tensor], torch.Tensor]] = None, + gather_logits: Optional[Callable[[torch.Tensor], torch.Tensor]] = None, + logits_out: Optional[torch.Tensor] = None, + has_sampled_rows: Optional[bool] = None, + ) -> torch.Tensor: + # None draws the noise for every row: reading the temperatures on the + # host would break a graph capture. + cls._check( + base_logits, + bias, + temperatures, + offset, + step_dtype, + vocab_start, + gather, + gather_logits, + logits_out, + ) + rows, width = bias.shape + value = base_logits.to(torch.float32, copy=True) + value[:, :width] += bias.float() + value = value.to(step_dtype).float() + if gather is not None and logits_out is not None: + full = gather_logits(value.to(step_dtype))[:, : logits_out.shape[1]] + return cls.torch( + base_logits=full, + bias=torch.zeros_like(full), + temperatures=temperatures, + key=key, + offset=offset, + step=step, + step_dtype=step_dtype, + logits_out=logits_out, + has_sampled_rows=has_sampled_rows, + ) + padded = base_logits.shape[1] + device = base_logits.device + if logits_out is not None: + logits_out.copy_(value) + + keys = value + if has_sampled_rows is not False: + quads = vocab_start // 4 + torch.arange( + triton.cdiv(padded, 4), device=device, dtype=torch.int64 + ) + words = _philox( + key, + quads[None, :], + torch.arange(rows, device=device, dtype=torch.int64)[:, None], + torch.full((1, 1), step, device=device, dtype=torch.int64), + offset.reshape(1, 1) & 0xFFFFFFFF, + ) + words = torch.broadcast_tensors(*words) + bits = torch.stack(words, dim=-1).reshape(rows, -1)[:, :padded] + uniform = (bits.float() + 0.5) * _UNIFORM_SCALE.value + exp_noise = -torch.log1p(-uniform) + sampled = temperatures != 0 + safe_temperatures = torch.where(sampled, temperatures, 1.0)[:, None] + noised = value / safe_temperatures - torch.log(exp_noise) + keys = torch.where(sampled[:, None], noised, value) + cols = torch.arange(padded, device=device) + keys = torch.where(cols[None, :] < width, keys, -torch.inf) + + if width == 0: + local_values = torch.full((rows,), -torch.inf, device=device) + local_indices = torch.full( + (rows,), _NO_INDEX.value, dtype=torch.int32, device=device + ) + else: + local = keys.argmax(dim=1) + local_values = keys.gather(1, local[:, None]).squeeze(1) + local_indices = (local + vocab_start).to(torch.int32) + partial = torch.stack( + [local_values.float(), local_indices.view(torch.float32)], dim=1 + ) + gathered = partial if gather is None else gather(partial) + gathered = gathered.view(-1, rows, 2) + values = gathered[..., 0].transpose(0, 1) + indices = gathered[..., 1].contiguous().view(torch.int32) & _NO_INDEX.value + indices = indices.transpose(0, 1).long() + # Lowest index among the NaNs if any, else among the maxima. + valid = indices != _NO_INDEX.value + nan = valid & values.isnan() + has_nan = nan.any(dim=-1, keepdim=True) + finite = torch.where(valid & ~nan, values, -torch.inf) + maximum = finite.amax(dim=-1, keepdim=True) + wins = torch.where(has_nan, nan, valid & (finite == maximum)) + return torch.where(wins, indices, _NO_INDEX.value).amin(dim=-1) diff --git a/python/sglang/kernels/ops/speculative/dspark/sharded_greedy.py b/python/sglang/kernels/ops/speculative/dspark/sharded_greedy.py deleted file mode 100644 index b86bd4468b..0000000000 --- a/python/sglang/kernels/ops/speculative/dspark/sharded_greedy.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Exact greedy selection from TP-local base logits and rounded Markov bias.""" - -import torch -import triton -import triton.language as tl - - -@triton.jit -def _sharded_greedy_partial_kernel( - B, - X, - P, - BS: tl.constexpr, - XS: tl.constexpr, - WIDTH: tl.constexpr, - OFFSET: tl.constexpr, - PARTS: tl.constexpr, - BLOCK: tl.constexpr, -): - row, part = tl.program_id(0), tl.program_id(1) - i = part * BLOCK + tl.arange(0, BLOCK) - valid = i < WIDTH - bias = tl.load(B + row * BS + i, valid, 0).to(tl.float32) - base = tl.load(X + row * XS + i, valid, 0).to(tl.float32) - value = base + bias - nan = valid & (value != value) - has_nan = tl.sum(nan.to(tl.int32), 0) > 0 - maximum = tl.max(tl.where(valid & ~nan, value, -float("inf")), 0) - wins = tl.where(has_nan, nan, valid & (value == maximum)) - idx = tl.min(tl.where(wins, i + OFFSET, 2147483647), 0) - maximum = tl.where(has_nan, float("nan"), maximum) - tl.store(P + (row * PARTS + part) * 2, maximum) - tl.store(P + (row * PARTS + part) * 2 + 1, idx.to(tl.float32, bitcast=True)) - - -@triton.jit -def _sharded_greedy_finish_kernel( - P, - OUT, - BS: tl.constexpr, - PARTS: tl.constexpr, - WORLD: tl.constexpr, - BLOCK: tl.constexpr, -): - row = tl.program_id(0) - i = tl.arange(0, BLOCK) - valid = i < PARTS * WORLD - rank, part = i // PARTS, i % PARTS - offset = ((rank * BS + row) * PARTS + part) * 2 - value = tl.load(P + offset, valid, -float("inf")) - idx = tl.load(P + offset + 1, valid, 0).to(tl.int32, bitcast=True) - # The NVLink push transport changes +0 to -0 as its Lamport sentinel. - # Indices are nonnegative int32 bits, so strip that sign bit to recover ID 0. - idx &= 2147483647 - valid &= idx != 2147483647 - nan = valid & (value != value) - has_nan = tl.sum(nan.to(tl.int32), 0) > 0 - maximum = tl.max(tl.where(valid & ~nan, value, -float("inf")), 0) - wins = tl.where(has_nan, nan, valid & (value == maximum)) - result = tl.min(tl.where(wins, idx, 2147483647), 0) - tl.store(OUT + row, result.to(tl.int64)) - - -def sharded_greedy_step(bias, base_local, *, group, vocab_start, gather=None): - """Fused BuildStepLocal + vocab gather + argmax, without materializing logits. - - Equivalent to argmax of rank-ordered ``build_step_local``/all_gather over the - sharded vocab, excluding padding. The transport carries one (value, - global-index-bits) pair per 4096-wide block of the shard per row; indices - move as bits and are never converted numerically to float. - """ - assert bias.ndim == base_local.ndim == 2 - assert bias.shape[0] == base_local.shape[0] - assert bias.shape[1] <= base_local.shape[1] - assert bias.stride(1) == base_local.stride(1) == 1 - rows, width = bias.shape - block = 4096 - parts = triton.cdiv(base_local.shape[1], block) - assert parts > 0 - partial = torch.empty((rows, parts, 2), device=bias.device, dtype=torch.float32) - _sharded_greedy_partial_kernel[(rows, parts)]( - bias, - base_local, - partial, - bias.stride(0), - base_local.stride(0), - width, - vocab_start, - parts, - block, - num_warps=4, - ) - # The padded partition width fixes the transport shape on all ranks; - # WIDTH masks real entries, including a completely empty final shard. - if gather is not None: - gathered = gather(partial.view(rows, parts * 2)) - else: - gathered = group.all_gather(partial, dim=0) if group.world_size > 1 else partial - result = torch.empty(rows, device=bias.device, dtype=torch.int64) - _sharded_greedy_finish_kernel[(rows,)]( - gathered, - result, - rows, - parts, - group.world_size, - triton.next_power_of_2(parts * group.world_size), - num_warps=4, - ) - return result diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 3833f3b9cd..c28d839e56 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -550,7 +550,8 @@ class Envs: SGLANG_DSPARK_OPT_MARKOV_W2_BF16 = EnvBool(True) SGLANG_DSPARK_OPT_MARKOV_W2_TP_SHARD = EnvBool(True) SGLANG_DSPARK_OPT_FUSED_GREEDY_MARKOV = EnvBool(False) - # With the TP-sharded markov_w2, gather each step's vocab-parallel logits over + # With the TP-sharded markov_w2, run the draft's vocab all-gathers (each + # step's per-shard token candidates, and the step logits sampling keeps) over # the NVLink push collective (CustomAllReduceV2's multicast plane) instead of # the NCCL ring. Only taken when the group's communicator has a multicast # plane; off, or no such plane, keeps the NCCL all-gather. diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index a659dc75ad..6220e59768 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -17,6 +17,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_draft_model import ( BuildStepLocal, CommitKvProj, ) +from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_gather from sglang.srt.environ import envs @@ -522,21 +523,25 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead): def sample_block_greedy_fused(self, base_logits, *, first_prev_tokens): if not self.supports_sharded_greedy or not base_logits.is_cuda: return None - from sglang.kernels.ops.speculative.dspark.sharded_greedy import ( - sharded_greedy_step, - ) - shard = self._tp_shard prev = first_prev_tokens.long() + # The greedy-only variant reads neither, so nothing fills them. + greedy = torch.empty(prev.shape[0], dtype=torch.float32, device=prev.device) + offset = torch.empty(1, dtype=torch.int64, device=prev.device) tokens = [] for step in range(base_logits.shape[1]): bias, _ = self.step_bias(prev, hidden_states=None, state=None) - prev = sharded_greedy_step( - bias, - base_logits[:, step], - group=self._shard_group, + prev = MarkovSampleStep.triton( + base_logits=base_logits[:, step], + bias=bias, + temperatures=greedy, + key=0, + offset=offset, + step=step, + step_dtype=torch.float32, vocab_start=shard.org_vocab_start, gather=self._vocab_gather.gather_stacked, + has_sampled_rows=False, ) tokens.append(prev) return torch.stack(tokens, dim=1) diff --git a/test/registered/kernels/ops/speculative/test_dspark_markov_sample.py b/test/registered/kernels/ops/speculative/test_dspark_markov_sample.py new file mode 100644 index 0000000000..64687d67f7 --- /dev/null +++ b/test/registered/kernels/ops/speculative/test_dspark_markov_sample.py @@ -0,0 +1,283 @@ +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-small") + +import unittest + +import torch + +from sglang.kernels.ops.speculative.dspark.markov_sample import MarkovSampleStep +from sglang.test.test_utils import CustomTestCase + +DEVICE = torch.device("cuda") + + +def _offset(value: int) -> torch.Tensor: + return torch.tensor([value], dtype=torch.int64, device=DEVICE) + + +def _call(impl, *, base, bias, temps, step_dtype, offset=7, step=2, **kwargs): + return impl( + base_logits=base, + bias=bias, + temperatures=temps, + key=12345, + offset=_offset(offset), + step=step, + step_dtype=step_dtype, + **kwargs, + ) + + +class TestMarkovSampleStep(CustomTestCase): + def test_triton_matches_torch_reference(self): + g = torch.Generator(device=DEVICE).manual_seed(0) + rows, padded, width = 64, 9000, 8990 + for base_dtype, bias_dtype, step_dtype in [ + (torch.bfloat16, torch.bfloat16, torch.bfloat16), + (torch.bfloat16, torch.bfloat16, torch.float32), + (torch.float32, torch.float32, torch.float32), + ]: + base = torch.randn(rows, padded, generator=g, device=DEVICE) * 4 + base = base.to(base_dtype) + bias = torch.randn(rows, width, generator=g, device=DEVICE).to(bias_dtype) + # Half greedy rows, half sampled at mixed temperatures. + temps = torch.rand(rows, generator=g, device=DEVICE) * 1.5 + 0.1 + temps[::2] = 0.0 + out = {} + for name, impl in [ + ("triton", MarkovSampleStep.triton), + ("torch", MarkovSampleStep.torch), + ]: + logits = torch.empty(rows, padded, dtype=step_dtype, device=DEVICE) + tokens = _call( + impl, + base=base, + bias=bias, + temps=temps, + step_dtype=step_dtype, + vocab_start=100, + logits_out=logits, + ) + out[name] = (tokens, logits) + (t_tok, t_log), (r_tok, r_log) = out["triton"], out["torch"] + self.assertTrue(torch.equal(t_log, r_log)) + expected_log = base.float() + expected_log[:, :width] += bias.float() + self.assertTrue(torch.equal(t_log, expected_log.to(step_dtype))) + greedy = temps == 0 + expected_greedy = t_log.float()[:, :width].argmax(dim=1) + 100 + self.assertTrue(torch.equal(t_tok[greedy], expected_greedy[greedy])) + # Same Philox bits on both sides; only a last-ulp log difference on a + # near tie could split them. + agree = (t_tok[~greedy] == r_tok[~greedy]).float().mean().item() + self.assertGreaterEqual(agree, 0.95) + self.assertTrue(((t_tok >= 100) & (t_tok < 100 + width)).all()) + + def test_greedy_only_variant_matches_mixed_kernel(self): + g = torch.Generator(device=DEVICE).manual_seed(3) + base = torch.randn(16, 9000, generator=g, device=DEVICE).to(torch.bfloat16) + bias = torch.randn(16, 8990, generator=g, device=DEVICE).to(torch.bfloat16) + temps = torch.zeros(16, device=DEVICE) + kwargs = dict(base=base, bias=bias, temps=temps, step_dtype=torch.bfloat16) + mixed = _call(MarkovSampleStep.triton, **kwargs) + for impl in (MarkovSampleStep.triton, MarkovSampleStep.torch): + greedy_only = _call(impl, has_sampled_rows=False, **kwargs) + self.assertTrue(torch.equal(greedy_only, mixed)) + + def test_special_values_follow_argmax(self): + base = torch.zeros(4, 16, device=DEVICE) + base[0, 3] = float("nan") + base[0, 9] = float("nan") + base[1, :] = -float("inf") + base[2, 5] = 2.0 + base[2, 11] = 2.0 + base[3, 15] = 100.0 + bias = torch.zeros(4, 15, device=DEVICE) + temps = torch.zeros(4, device=DEVICE) + for impl in (MarkovSampleStep.triton, MarkovSampleStep.torch): + tokens = _call( + impl, base=base, bias=bias, temps=temps, step_dtype=torch.float32 + ) + # Lowest NaN, then lowest index of a tie; column 15 is padding. + self.assertEqual(tokens.tolist(), [3, 0, 5, 0]) + + def test_greedy_tie_across_blocks_takes_the_lower_index(self): + base = torch.zeros(2, 9000, device=DEVICE) + base[:, 1000] = base[:, 5000] = 5.0 + bias = torch.zeros(2, 9000, device=DEVICE) + for impl in (MarkovSampleStep.triton, MarkovSampleStep.torch): + tokens = _call( + impl, + base=base, + bias=bias, + temps=torch.zeros(2, device=DEVICE), + step_dtype=torch.float32, + ) + self.assertEqual(tokens.tolist(), [1000, 1000]) + + def test_strided_rows_match_contiguous(self): + # The draft passes one step of a [bs, gamma, vocab] block, rows strided. + g = torch.Generator(device=DEVICE).manual_seed(2) + block = torch.randn(8, 5, 6000, generator=g, device=DEVICE) + bias = torch.randn(8, 5990, generator=g, device=DEVICE) + temps = torch.rand(8, generator=g, device=DEVICE) + temps[:3] = 0.0 + logits_block = torch.empty(8, 5, 6000, device=DEVICE) + strided = _call( + MarkovSampleStep.triton, + base=block[:, 2], + bias=bias, + temps=temps, + step_dtype=torch.float32, + logits_out=logits_block[:, 2], + ) + logits = torch.empty(8, 6000, device=DEVICE) + contiguous = _call( + MarkovSampleStep.triton, + base=block[:, 2].contiguous(), + bias=bias, + temps=temps, + step_dtype=torch.float32, + logits_out=logits, + ) + self.assertTrue(torch.equal(strided, contiguous)) + self.assertTrue(torch.equal(logits_block[:, 2], logits)) + + def test_sampling_follows_tempered_softmax(self): + vocab, rows = 32, 1 << 16 + logits = torch.linspace(-3, 3, vocab, device=DEVICE) + base = logits.expand(rows, vocab).contiguous() + bias = torch.zeros(rows, vocab, device=DEVICE) + for temperature in (0.5, 1.0, 2.0): + temps = torch.full((rows,), temperature, device=DEVICE) + tokens = _call( + MarkovSampleStep.triton, + base=base, + bias=bias, + temps=temps, + step_dtype=torch.float32, + ) + freq = torch.bincount(tokens, minlength=vocab).float() / rows + expected = torch.softmax(logits / temperature, dim=0) + self.assertLess((freq - expected).abs().max().item(), 0.01) + + def test_noise_changes_with_offset_row_and_step(self): + rows, vocab = 256, 4096 + base = torch.zeros(rows, vocab, device=DEVICE) + bias = torch.zeros(rows, vocab, device=DEVICE) + temps = torch.ones(rows, device=DEVICE) + first = _call( + MarkovSampleStep.triton, + base=base, + bias=bias, + temps=temps, + step_dtype=torch.float32, + ) + self.assertGreater(first.unique().numel(), rows // 2) + for kwargs in ({"offset": 8}, {"step": 3}): + other = _call( + MarkovSampleStep.triton, + base=base, + bias=bias, + temps=temps, + step_dtype=torch.float32, + **kwargs, + ) + self.assertLess((other == first).float().mean().item(), 0.01) + + def test_sharded_vocab_matches_full_vocab(self): + # Every rank must pick the full vocab's token: the noise is keyed by + # global token id, so the shards draw what the full vocab draws. + g = torch.Generator(device=DEVICE).manual_seed(1) + rows, shard = 32, 5000 + base = torch.randn(rows, 3 * shard, generator=g, device=DEVICE) + bias = torch.randn(rows, 2 * shard - 7, generator=g, device=DEVICE) + temps = torch.rand(rows, generator=g, device=DEVICE) + temps[:8] = 0.0 + full = _call( + MarkovSampleStep.triton, + base=base[:, : bias.shape[1]].contiguous(), + bias=bias, + temps=temps, + step_dtype=torch.float32, + ) + # The last shard is empty: all its columns are padding. + shards = [ + (rank * shard, base[:, rank * shard : (rank + 1) * shard].contiguous()) + for rank in range(3) + ] + partials = {} + + def run(rank, gather): + start, shard_base = shards[rank] + return _call( + MarkovSampleStep.triton, + base=shard_base, + bias=bias[:, start : start + shard].contiguous(), + temps=temps, + step_dtype=torch.float32, + vocab_start=start, + gather=gather, + ) + + for rank in range(3): + run(rank, lambda partial, rank=rank: partials.setdefault(rank, partial)) + stacked = torch.cat([partials[rank] for rank in range(3)]) + for rank in range(3): + self.assertTrue(torch.equal(run(rank, lambda _: stacked), full)) + + def test_kept_sharded_logits_match_full_vocab(self): + # Kept logits are gathered whole and every rank picks from them: the + # same tokens and the same logits as the full vocab, both backends. + g = torch.Generator(device=DEVICE).manual_seed(2) + rows, shard = 32, 5000 + vocab = 2 * shard - 7 + base = torch.randn(rows, 3 * shard, generator=g, device=DEVICE) + bias = torch.randn(rows, vocab, generator=g, device=DEVICE) + temps = torch.rand(rows, generator=g, device=DEVICE) + temps[:8] = 0.0 + full_logits = torch.empty(rows, vocab, device=DEVICE) + full = _call( + MarkovSampleStep.triton, + base=base[:, :vocab].contiguous(), + bias=bias, + temps=temps, + step_dtype=torch.float32, + logits_out=full_logits, + ) + for impl in (MarkovSampleStep.triton, MarkovSampleStep.torch): + locals_ = {} + + def run(rank, gather_logits): + start = rank * shard + logits_out = torch.empty(rows, vocab, device=DEVICE) + tokens = _call( + impl, + base=base[:, start : start + shard].contiguous(), + bias=bias[:, start : start + shard].contiguous(), + temps=temps, + step_dtype=torch.float32, + vocab_start=start, + gather=lambda _: self.fail("candidates gathered"), + gather_logits=gather_logits, + logits_out=logits_out, + ) + return tokens, logits_out + + for rank in range(3): + run( + rank, + lambda local, rank=rank: locals_.setdefault( + rank, local.clone() + ).repeat(1, 3), + ) + stacked = torch.cat([locals_[rank] for rank in range(3)], dim=1) + for rank in range(3): + tokens, logits_out = run(rank, lambda _: stacked) + self.assertTrue(torch.equal(tokens, full)) + self.assertTrue(torch.equal(logits_out, full_logits)) + + +if __name__ == "__main__": + unittest.main()