728 lines
30 KiB
Python
728 lines
30 KiB
Python
"""Unit tests for MLX in-graph sampling (hardware_backend/mlx/sampling.py)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import unittest
|
|
from collections import Counter
|
|
|
|
from sglang.test.ci.ci_register import register_mlx_ci
|
|
from sglang.test.test_utils import CustomTestCase
|
|
|
|
register_mlx_ci(est_time=20, suite="stage-a-unit-test-mlx")
|
|
|
|
_HAS_MLX = importlib.util.find_spec("mlx") is not None
|
|
_SKIP_REASON = "requires mlx"
|
|
|
|
if _HAS_MLX:
|
|
import mlx.core as mx
|
|
|
|
from sglang.srt.hardware_backend.mlx.sampling import (
|
|
DEFAULT_SAMPLING_SEED,
|
|
GREEDY_PARAMS,
|
|
MAX_BOUNDED_TOP_K,
|
|
MlxLogprobSpec,
|
|
MlxSamplingParams,
|
|
_candidate_width,
|
|
_gumbel_noise,
|
|
_murmur_hash32,
|
|
all_greedy,
|
|
compute_logprobs,
|
|
sample_tokens,
|
|
sanitize_logits,
|
|
)
|
|
|
|
|
|
def _reference_murmur3(seed: int, pos: int, col: int) -> int:
|
|
"""Pure-Python MurmurHash3 mirroring the Triton kernel in
|
|
sglang/kernels/ops/sampling/murmur_hash.py: blocks seed_low,
|
|
seed_high, position, column; length-16 finalization; fmix32."""
|
|
|
|
def mix(h: int, k: int) -> int:
|
|
k = (k * 0xCC9E2D51) & 0xFFFFFFFF
|
|
k = ((k << 15) | (k >> 17)) & 0xFFFFFFFF
|
|
k = (k * 0x1B873593) & 0xFFFFFFFF
|
|
h ^= k
|
|
h = ((h << 13) | (h >> 19)) & 0xFFFFFFFF
|
|
return (h * 5 + 0xE6546B64) & 0xFFFFFFFF
|
|
|
|
seed &= 0xFFFFFFFFFFFFFFFF
|
|
h = mix(0, seed & 0xFFFFFFFF)
|
|
h = mix(h, (seed >> 32) & 0xFFFFFFFF)
|
|
h = mix(h, pos & 0xFFFFFFFF)
|
|
h = mix(h, col & 0xFFFFFFFF)
|
|
h ^= 16
|
|
h ^= h >> 16
|
|
h = (h * 0x85EBCA6B) & 0xFFFFFFFF
|
|
h ^= h >> 13
|
|
h = (h * 0xC2B2AE35) & 0xFFFFFFFF
|
|
h ^= h >> 16
|
|
return h
|
|
|
|
|
|
def _params(temperature=1.0, top_k=1 << 30, top_p=1.0, min_p=0.0, seed=None):
|
|
return MlxSamplingParams(
|
|
temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, seed=seed
|
|
)
|
|
|
|
|
|
@unittest.skipUnless(_HAS_MLX, _SKIP_REASON)
|
|
class TestMurmurHashPort(CustomTestCase):
|
|
def test_matches_pure_python_reference(self):
|
|
"""Guards the mx uint32 port of the CUDA murmur kernel: any drift in
|
|
wraparound/shift/block-order semantics changes seeded sampling."""
|
|
seeds = [0, 1, 42, 2**31, 2**63 + 12345]
|
|
positions = [0, 7, 1023, 2**31 - 1, 5]
|
|
vocab = 64
|
|
hashed = _murmur_hash32(seeds=seeds, positions=positions, vocab_size=vocab)
|
|
mx.eval(hashed)
|
|
for row, (seed, pos) in enumerate(zip(seeds, positions)):
|
|
for col in (0, 1, vocab // 2, vocab - 1):
|
|
self.assertEqual(
|
|
int(hashed[row, col].item()),
|
|
_reference_murmur3(seed, pos, col),
|
|
msg=f"mismatch at seed={seed} pos={pos} col={col}",
|
|
)
|
|
|
|
|
|
@unittest.skipUnless(_HAS_MLX, _SKIP_REASON)
|
|
class TestSampleTokens(CustomTestCase):
|
|
VOCAB = 32
|
|
|
|
def _logits(self, batch_size: int, key_int: int = 0) -> mx.array:
|
|
return (
|
|
mx.random.normal(shape=(batch_size, self.VOCAB), key=mx.random.key(key_int))
|
|
* 3.0
|
|
)
|
|
|
|
def _draw(self, logits, params, positions=None, n=200, key_start=100):
|
|
"""Sample n times with distinct keys, return per-row token Counters."""
|
|
batch_size = logits.shape[0]
|
|
positions = positions if positions is not None else [5] * batch_size
|
|
counters = [Counter() for _ in range(batch_size)]
|
|
for i in range(n):
|
|
toks = sample_tokens(
|
|
last_logits=logits,
|
|
params=params,
|
|
positions=positions,
|
|
key=mx.random.key(key_start + i),
|
|
)
|
|
mx.eval(toks)
|
|
for row, t in enumerate(toks.tolist()):
|
|
counters[row][int(t)] += 1
|
|
return counters
|
|
|
|
@staticmethod
|
|
def _reference_support(probs, top_k, top_p, min_p):
|
|
"""Independent replica of the mask, in pure Python, on sorted probs."""
|
|
order = sorted(range(len(probs)), key=lambda i: (-probs[i], i))
|
|
keep, cum = [], 0.0
|
|
for rank, idx in enumerate(order):
|
|
p = probs[idx]
|
|
masked = (
|
|
rank >= min(top_k, len(probs))
|
|
or cum > top_p
|
|
or p < probs[order[0]] * min_p
|
|
)
|
|
cum += p
|
|
if not masked:
|
|
keep.append(idx)
|
|
return set(keep)
|
|
|
|
def _probs(self, logits):
|
|
probs = mx.softmax(logits.astype(mx.float32), axis=-1)
|
|
mx.eval(probs)
|
|
return probs[0].tolist()
|
|
|
|
def test_greedy_rows_match_argmax_in_mixed_batch(self):
|
|
"""A greedy row must return exactly argmax even when other rows in
|
|
the batch sample — guards the where() row-select and the sglang
|
|
greedy convention (top_k == 1)."""
|
|
logits = self._logits(3)
|
|
expected = mx.argmax(logits, axis=-1).tolist()
|
|
params = [_params(top_k=1), _params(temperature=0.7), _params(top_k=1)]
|
|
counters = self._draw(logits, params, n=25)
|
|
self.assertEqual(set(counters[0]), {expected[0]})
|
|
self.assertEqual(set(counters[2]), {expected[2]})
|
|
|
|
def test_filter_supports_match_reference(self):
|
|
"""The sampled support must stay inside the independently computed
|
|
mask for each filter and for their combination — guards the rank
|
|
mask, the nucleus exclusion, the min_p threshold, and the
|
|
sorted->vocab index map."""
|
|
cases = [
|
|
("top_k=2", dict(top_k=2)),
|
|
("top_p=0.6", dict(top_p=0.6)),
|
|
("min_p=0.3", dict(min_p=0.3)),
|
|
("top_k=8,top_p=0.7,min_p=0.05", dict(top_k=8, top_p=0.7, min_p=0.05)),
|
|
]
|
|
for label, kwargs in cases:
|
|
with self.subTest(label):
|
|
logits = self._logits(1, key_int=3)
|
|
support = self._reference_support(
|
|
self._probs(logits),
|
|
kwargs.get("top_k", 1 << 30),
|
|
kwargs.get("top_p", 1.0),
|
|
kwargs.get("min_p", 0.0),
|
|
)
|
|
drawn = set(self._draw(logits, [_params(**kwargs)], n=400)[0])
|
|
self.assertTrue(drawn <= support, f"{label}: extra {drawn - support}")
|
|
self.assertTrue(drawn, f"{label}: nothing sampled")
|
|
|
|
def test_seeded_row_is_deterministic_and_key_independent(self):
|
|
"""A row with sampling_seed must produce the same token regardless
|
|
of the RNG key or batch composition — the murmur-gumbel path only
|
|
depends on (seed, position, logits)."""
|
|
logits = self._logits(2, key_int=5)
|
|
seeded = _params(temperature=1.0, seed=1234)
|
|
tok_solo = sample_tokens(
|
|
last_logits=logits[:1],
|
|
params=[seeded],
|
|
positions=[9],
|
|
key=mx.random.key(0),
|
|
)
|
|
tok_other_key = sample_tokens(
|
|
last_logits=logits[:1],
|
|
params=[seeded],
|
|
positions=[9],
|
|
key=mx.random.key(999),
|
|
)
|
|
tok_in_batch = sample_tokens(
|
|
last_logits=logits,
|
|
params=[seeded, _params(temperature=0.8)],
|
|
positions=[9, 3],
|
|
key=mx.random.key(7),
|
|
)
|
|
mx.eval(tok_solo, tok_other_key, tok_in_batch)
|
|
self.assertEqual(tok_solo.tolist(), tok_other_key.tolist())
|
|
self.assertEqual(int(tok_in_batch[0].item()), int(tok_solo[0].item()))
|
|
|
|
def test_seeded_row_unaffected_by_batchmate_filtering(self):
|
|
"""A seeded row's token must not change when a batchmate triggers
|
|
the top-k/top-p sort path — guards the vocab-id-space noise
|
|
contract (regression: noise was applied in sorted-rank space when
|
|
any row filtered, so batch composition changed seeded tokens)."""
|
|
# Near-uniform seeded row: the Gumbel noise decides the token, so
|
|
# a change of noise index space is guaranteed to show up.
|
|
seeded_logits = (
|
|
mx.random.normal(shape=(1, self.VOCAB), key=mx.random.key(8)) * 0.05
|
|
)
|
|
mate_logits = (
|
|
mx.random.normal(shape=(1, self.VOCAB), key=mx.random.key(9)) * 3.0
|
|
)
|
|
logits = mx.concatenate([seeded_logits, mate_logits], axis=0)
|
|
seeded = _params(seed=4321)
|
|
solo = sample_tokens(
|
|
last_logits=logits[:1],
|
|
params=[seeded],
|
|
positions=[6],
|
|
key=mx.random.key(0),
|
|
)
|
|
with_filtering_mate = sample_tokens(
|
|
last_logits=logits,
|
|
params=[seeded, _params(temperature=1.2, top_k=2)],
|
|
positions=[6, 11],
|
|
key=mx.random.key(55),
|
|
)
|
|
mx.eval(solo, with_filtering_mate)
|
|
self.assertEqual(int(with_filtering_mate[0].item()), int(solo[0].item()))
|
|
|
|
def test_bounded_top_k_picks_the_same_token_as_the_full_vocab_chain(self):
|
|
"""The bounded top-K chain is an optimization, not a policy change:
|
|
a seeded row must pick the same token whether or not the batch is
|
|
eligible for it. A batchmate without a finite top_k pushes the
|
|
whole batch back onto the full-vocab chain, so the same seeded row
|
|
is sampled both ways here."""
|
|
logits = self._logits(2, key_int=11)
|
|
seeded = _params(top_k=4, seed=2024)
|
|
bounded = sample_tokens(
|
|
last_logits=logits[:1],
|
|
params=[seeded],
|
|
positions=[6],
|
|
key=mx.random.key(0),
|
|
)
|
|
# min_p alone leaves top_k at TOP_K_ALL, so this batch falls back.
|
|
full_vocab = sample_tokens(
|
|
last_logits=logits,
|
|
params=[seeded, _params(min_p=0.1)],
|
|
positions=[6, 2],
|
|
key=mx.random.key(3),
|
|
)
|
|
mx.eval(bounded, full_vocab)
|
|
self.assertEqual(int(full_vocab[0].item()), int(bounded[0].item()))
|
|
|
|
def test_candidate_width_gates_the_bounded_chain(self):
|
|
"""Only a batch whose widest top_k fits inside both the bound and
|
|
the vocabulary may shrink the chain; anything else must return the
|
|
full vocab size (which selects the scatter-back path)."""
|
|
vocab = 4096
|
|
for label, params, expected in [
|
|
("under the bound", [_params(top_k=64)], 64),
|
|
("at the bound", [_params(top_k=MAX_BOUNDED_TOP_K)], MAX_BOUNDED_TOP_K),
|
|
("past the bound", [_params(top_k=MAX_BOUNDED_TOP_K + 1)], vocab),
|
|
("no top_k (TOP_K_ALL)", [_params()], vocab),
|
|
("top_k == vocab", [_params(top_k=vocab)], vocab),
|
|
("widest row wins", [_params(top_k=4), _params(top_k=64)], 64),
|
|
("one unbounded row", [_params(top_k=4), _params(top_p=0.9)], vocab),
|
|
]:
|
|
with self.subTest(label):
|
|
self.assertEqual(_candidate_width(params, vocab), expected)
|
|
|
|
def test_seeded_row_varies_with_position(self):
|
|
"""Positions feed the hash, so a fixed seed must not freeze the
|
|
distribution across steps: over many positions the sampled tokens
|
|
must not all collapse to one value (vocab of near-uniform probs)."""
|
|
logits = mx.zeros((1, self.VOCAB)) # uniform distribution
|
|
seeded = [_params(seed=77)]
|
|
toks = set()
|
|
for pos in range(40):
|
|
t = sample_tokens(
|
|
last_logits=logits,
|
|
params=seeded,
|
|
positions=[pos],
|
|
key=mx.random.key(0),
|
|
)
|
|
mx.eval(t)
|
|
toks.add(int(t[0].item()))
|
|
self.assertGreater(len(toks), 5, toks)
|
|
|
|
def test_seeded_noise_is_finite(self):
|
|
"""The uniform draw is clamped to [2**-32, 1 - 2**-24] before the
|
|
double log: uint32(0xFFFFFFFF) rounds UP to 2**32 in float32, so an
|
|
unclamped u can exceed 1 and make log(-log u) NaN — and u == 1 gives
|
|
+inf, which would deterministically force that token."""
|
|
# Sanity-check the hazard the clamp exists for.
|
|
u_max = mx.array([0xFFFFFFFF], dtype=mx.uint32).astype(mx.float32) / float(
|
|
0xFFFFFFFF
|
|
)
|
|
mx.eval(u_max)
|
|
self.assertGreaterEqual(float(u_max.item()), 1.0)
|
|
self.assertFalse(bool(mx.isfinite(-mx.log(-mx.log(u_max))).item()))
|
|
|
|
noise = _gumbel_noise(
|
|
params=[_params(seed=1), _params(seed=2**63 - 1)],
|
|
positions=[0, 4096],
|
|
shape=(2, 1 << 16),
|
|
key=mx.random.key(0),
|
|
)
|
|
mx.eval(noise)
|
|
self.assertTrue(bool(mx.all(mx.isfinite(noise)).item()))
|
|
|
|
def test_seed_with_min_p_is_supported(self):
|
|
"""seed + min_p is well defined here (the pytorch backend asserts on
|
|
the combination): Gumbel-max over unnormalized masked weights is
|
|
invariant to the missing renormalization, which is exactly the TODO
|
|
at layers/sampler.py's multinomial_with_seed path."""
|
|
logits = self._logits(1, key_int=4)
|
|
support = self._reference_support(self._probs(logits), 1 << 30, 1.0, 0.3)
|
|
tok = sample_tokens(
|
|
last_logits=logits,
|
|
params=[_params(seed=1234, min_p=0.3)],
|
|
positions=[9],
|
|
key=mx.random.key(0),
|
|
)
|
|
mx.eval(tok)
|
|
self.assertIn(int(tok[0].item()), support)
|
|
|
|
def test_temperature_sharpens_distribution(self):
|
|
"""Lower temperature must concentrate mass on the argmax token —
|
|
guards the per-row temperature division (e.g. broadcasting bugs
|
|
that apply one row's temperature to all rows)."""
|
|
logits = self._logits(2, key_int=6)
|
|
expected0 = int(mx.argmax(logits[0]).item())
|
|
params = [_params(temperature=0.05), _params(temperature=5.0)]
|
|
counters = self._draw(logits, params, n=200)
|
|
self.assertGreater(counters[0][expected0] / 200.0, 0.95)
|
|
self.assertGreater(len(counters[1]), 5, "high temp should spread mass")
|
|
|
|
def test_greedy_helpers(self):
|
|
self.assertTrue(all_greedy([GREEDY_PARAMS, _params(top_k=1)]))
|
|
self.assertFalse(all_greedy([GREEDY_PARAMS, _params(temperature=0.9)]))
|
|
|
|
|
|
@unittest.skipUnless(_HAS_MLX, _SKIP_REASON)
|
|
class TestSanitizeAndLogprobs(CustomTestCase):
|
|
VOCAB = 16
|
|
|
|
def test_sanitize_matches_nan_to_num_semantics(self):
|
|
"""Guards the port of sanitize_nan_logits' exact replacement values
|
|
(+-1e30, not dtype extremes — temperature division would overflow
|
|
dtype extremes back to inf and softmax them to NaN)."""
|
|
import struct
|
|
|
|
def f32(v):
|
|
return struct.unpack("f", struct.pack("f", v))[0]
|
|
|
|
x = mx.array([[1.0, float("nan"), float("inf"), -float("inf")]])
|
|
out = sanitize_logits(x)
|
|
mx.eval(out)
|
|
self.assertEqual(out.tolist(), [[1.0, f32(-1e30), f32(1e30), f32(-1e30)]])
|
|
|
|
def test_logprobs_match_reference_and_row_shapes(self):
|
|
"""compute_logprobs must equal log_softmax(logits/temp) per row and
|
|
cut top-k / token-ids to each row's requested shape — guards the
|
|
per-row temperature broadcast and the spec row alignment."""
|
|
import math
|
|
|
|
logits = mx.random.normal(shape=(2, self.VOCAB), key=mx.random.key(11))
|
|
params = [_params(temperature=0.5), _params(temperature=2.0)]
|
|
tokens = mx.array([3, 7], dtype=mx.uint32)
|
|
spec = MlxLogprobSpec(top_ks=(2, 0), token_ids=(None, (1, 4)))
|
|
lp = compute_logprobs(logits, params, tokens, spec)
|
|
mx.eval(*[a for a in [lp.chosen, lp.top_val, lp.top_idx] if a is not None])
|
|
|
|
raw = logits.tolist()
|
|
for row, temp in ((0, 0.5), (1, 2.0)):
|
|
scaled = [v / temp for v in raw[row]]
|
|
m = max(scaled)
|
|
lse = m + math.log(sum(math.exp(v - m) for v in scaled))
|
|
ref = [v - lse for v in scaled]
|
|
chosen_token = int(tokens[row].item())
|
|
self.assertAlmostEqual(
|
|
float(lp.chosen[row].item()), ref[chosen_token], places=4
|
|
)
|
|
if row == 0:
|
|
expect_top = sorted(ref, reverse=True)[:2]
|
|
got = lp.top_val[row].tolist()[:2]
|
|
for a, b in zip(got, expect_top):
|
|
self.assertAlmostEqual(a, b, places=4)
|
|
if row == 1:
|
|
got = lp.token_ids_val[1].tolist()
|
|
self.assertAlmostEqual(got[0], ref[1], places=4)
|
|
self.assertAlmostEqual(got[1], ref[4], places=4)
|
|
self.assertIsNone(lp.token_ids_val[0])
|
|
|
|
|
|
@unittest.skipUnless(_HAS_MLX, _SKIP_REASON)
|
|
class TestRunnerSelectTokens(CustomTestCase):
|
|
"""_select_tokens_with_logprobs lifecycle on a bare runner (object.__new__)."""
|
|
|
|
class _FakeCache:
|
|
def __init__(self, offset):
|
|
self.offset = offset
|
|
|
|
class _FakeLayout:
|
|
has_auxiliary_state = False
|
|
first_attention_layer_index = 0
|
|
|
|
def _runner(self, enable_sampling):
|
|
from sglang.srt.hardware_backend.mlx.model_runner import MlxModelRunner
|
|
|
|
runner = object.__new__(MlxModelRunner)
|
|
runner._enable_sampling = enable_sampling
|
|
runner._cache_layout = self._FakeLayout()
|
|
runner._req_sampling = {}
|
|
runner._rng_key = mx.random.key(0) if enable_sampling else None
|
|
return runner
|
|
|
|
def test_disabled_and_greedy_paths_consume_no_rng(self):
|
|
"""Flag-off and all-greedy batches must return exact argmax and
|
|
leave the RNG key untouched — guards the byte-exact greedy
|
|
contract that the e2e temp=0 test relies on."""
|
|
logits = mx.random.normal(shape=(2, 16), key=mx.random.key(1))
|
|
expected = mx.argmax(logits, axis=-1).tolist()
|
|
caches = [[self._FakeCache(4)], [self._FakeCache(9)]]
|
|
|
|
disabled = self._runner(enable_sampling=False)
|
|
toks = disabled._select_tokens_with_logprobs(logits, ["a", "b"], caches)[0]
|
|
self.assertEqual(toks.tolist(), expected)
|
|
|
|
enabled = self._runner(enable_sampling=True)
|
|
enabled._req_sampling = {"a": GREEDY_PARAMS, "b": _params(top_k=1)}
|
|
key_before = enabled._rng_key
|
|
toks = enabled._select_tokens_with_logprobs(logits, ["a", "b"], caches)[0]
|
|
self.assertEqual(toks.tolist(), expected)
|
|
self.assertIs(enabled._rng_key, key_before)
|
|
|
|
def test_discarded_chunk_without_trunk_stays_greedy_and_consumes_no_rng(self):
|
|
"""A needs_logits=False chunk on a model without a headless trunk
|
|
must not sample: consuming RNG for a discarded token would make
|
|
the final output depend on prefill chunking."""
|
|
|
|
def full_model_only(input_ids, cache=None):
|
|
return mx.zeros((1, input_ids.shape[1], 16))
|
|
|
|
runner = self._runner(enable_sampling=True)
|
|
runner.model = full_model_only # no .model attr -> no trunk
|
|
runner._req_sampling = {"a": _params(temperature=1.0)}
|
|
key_before = runner._rng_key
|
|
tok, lazy_logprobs = runner._forward_lazy_token(
|
|
mx.array([[3, 4]], dtype=mx.int32),
|
|
[self._FakeCache(2)],
|
|
needs_logits=False,
|
|
req_id="a",
|
|
)
|
|
self.assertIsNone(lazy_logprobs)
|
|
mx.eval(tok)
|
|
self.assertEqual(tok.tolist(), [0]) # argmax of zeros
|
|
self.assertIs(runner._rng_key, key_before)
|
|
|
|
def test_logit_edits_gate_greedy_and_sampled_and_logprobs(self):
|
|
"""An additive -inf edit row must exclude a token from greedy argmax,
|
|
from sampling, AND from the reported logprob distribution — guards
|
|
the edits-before-selection ordering (a regression that samples raw
|
|
logits would pass every other test on near-uniform inputs)."""
|
|
runner = self._runner(enable_sampling=True)
|
|
runner._req_sampling = {"g": GREEDY_PARAMS, "s": _params(temperature=1.0)}
|
|
caches = [[self._FakeCache(4)], [self._FakeCache(4)]]
|
|
logits = mx.zeros((2, 8))
|
|
logits = mx.put_along_axis(
|
|
logits,
|
|
mx.array([[7], [7]], dtype=mx.uint32),
|
|
mx.array([[5.0], [5.0]]),
|
|
axis=-1,
|
|
) # token 7 dominates both rows
|
|
edits = mx.zeros((2, 8))
|
|
edits = mx.put_along_axis(
|
|
edits,
|
|
mx.array([[7], [7]], dtype=mx.uint32),
|
|
mx.array([[-float("inf")], [-float("inf")]]),
|
|
axis=-1,
|
|
) # ...but is masked out for both
|
|
spec = MlxLogprobSpec(top_ks=(1, 1), token_ids=(None, None))
|
|
for _ in range(10):
|
|
tokens, lp = runner._select_tokens_with_logprobs(
|
|
logits, ["g", "s"], caches, edits, spec
|
|
)
|
|
mx.eval(tokens, lp.chosen, lp.top_val)
|
|
self.assertNotIn(7, tokens.tolist())
|
|
self.assertNotIn(7, [row[0] for row in lp.top_idx.tolist()])
|
|
|
|
def test_seed_is_gated_on_deterministic_inference(self):
|
|
"""Upstream seed contract: SamplingBatchInfo only populates
|
|
sampling_seed under --enable-deterministic-inference, and then seeds
|
|
every row (default 42). A per-request seed outside that flag is
|
|
ignored by every other backend, so it is ignored here too."""
|
|
from types import SimpleNamespace
|
|
|
|
def make_req(sampling_seed):
|
|
return SimpleNamespace(
|
|
sampling_params=SimpleNamespace(
|
|
temperature=0.8,
|
|
top_k=1 << 30,
|
|
top_p=1.0,
|
|
min_p=0.0,
|
|
sampling_seed=sampling_seed,
|
|
frequency_penalty=0.0,
|
|
presence_penalty=0.0,
|
|
repetition_penalty=1.0,
|
|
)
|
|
)
|
|
|
|
self.assertIsNone(MlxSamplingParams.from_req(make_req(7)).seed)
|
|
self.assertIsNone(
|
|
MlxSamplingParams.from_req(make_req(None), deterministic_seeding=False).seed
|
|
)
|
|
self.assertEqual(
|
|
MlxSamplingParams.from_req(make_req(7), deterministic_seeding=True).seed, 7
|
|
)
|
|
self.assertEqual(
|
|
MlxSamplingParams.from_req(make_req(None), deterministic_seeding=True).seed,
|
|
DEFAULT_SAMPLING_SEED,
|
|
)
|
|
|
|
def test_chained_decode_keeps_logit_bias(self):
|
|
"""A chained decode step must keep applying the batch's static
|
|
logit_bias rows — regression: the chained path passed edits=None,
|
|
silently dropping the bias after the first (fresh) step."""
|
|
runner = self._runner(enable_sampling=True)
|
|
runner._req_sampling = {"a": GREEDY_PARAMS}
|
|
runner._req_caches = {"a": [self._FakeCache(3)]}
|
|
runner._req_token_ids = {"a": [1]}
|
|
# token 2 dominates; the edit row bans it -> argmax must fall to 1
|
|
logits = mx.array([[0.0, 3.0, 5.0, 0.0]])
|
|
runner._decode_with_batched_attention = lambda caches, x, rids: logits
|
|
edits = mx.array([[0.0, 0.0, -float("inf"), 0.0]])
|
|
|
|
fresh = runner.decode_batch_start(["a"], edit_rows=edits)
|
|
chained = runner.decode_batch_start_chained(fresh)
|
|
mx.eval(fresh.lazy_tokens, chained.lazy_tokens)
|
|
self.assertEqual(fresh.lazy_tokens.tolist(), [1])
|
|
self.assertEqual(chained.lazy_tokens.tolist(), [1])
|
|
|
|
def test_logits_hook_bridge_roundtrip(self):
|
|
"""The custom-logit-processor hook must see materialized float32
|
|
logits and its in-place edits must re-enter the graph — guards the
|
|
mx->numpy->mx bridge (a copy-semantics change would drop edits)."""
|
|
runner = self._runner(enable_sampling=True)
|
|
logits = mx.zeros((1, 8), dtype=mx.bfloat16)
|
|
|
|
def hook(arr):
|
|
assert arr.dtype.name == "float32"
|
|
arr[0, 5] = 99.0
|
|
return arr
|
|
|
|
edited = runner._run_logits_hook(logits, hook)
|
|
mx.eval(edited)
|
|
self.assertEqual(int(mx.argmax(edited, axis=-1)[0].item()), 5)
|
|
|
|
def test_sampling_path_advances_rng_key(self):
|
|
"""Consecutive sampling builds must consume distinct keys, or every
|
|
chained decode step would draw identical noise."""
|
|
logits = mx.zeros((1, 16))
|
|
runner = self._runner(enable_sampling=True)
|
|
runner._req_sampling = {"a": _params(temperature=1.0)}
|
|
caches = [[self._FakeCache(3)]]
|
|
toks = set()
|
|
for _ in range(20):
|
|
t = runner._select_tokens_with_logprobs(logits, ["a"], caches)[0]
|
|
mx.eval(t)
|
|
toks.add(int(t[0].item()))
|
|
self.assertGreater(len(toks), 3, toks)
|
|
|
|
|
|
@unittest.skipUnless(_HAS_MLX, _SKIP_REASON)
|
|
class TestWorkerSamplingExtras(CustomTestCase):
|
|
"""Worker-side builders: logit-edit rows, logprob specs, output assembly."""
|
|
|
|
VOCAB = 8
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
from sglang.srt.runtime_context import get_context
|
|
|
|
# The worker reads --mlx-enable-sampling off the device config bag,
|
|
# which fails closed before a publish.
|
|
cls._config = get_context().override_server_args(mlx_enable_sampling=True)
|
|
cls._config.install()
|
|
cls.addClassCleanup(cls._config.restore)
|
|
|
|
@staticmethod
|
|
def _worker():
|
|
from sglang.srt.hardware_backend.mlx.tp_worker import MlxTpModelWorker
|
|
|
|
return MlxTpModelWorker.__new__(MlxTpModelWorker)
|
|
|
|
def _batch(self, sinfo, n=2, return_logprob=False, has_grammar=False):
|
|
from types import SimpleNamespace
|
|
|
|
return SimpleNamespace(
|
|
reqs=[
|
|
SimpleNamespace(
|
|
rid=f"r{i}", return_logprob=return_logprob, grammar=None
|
|
)
|
|
for i in range(n)
|
|
],
|
|
sampling_info=sinfo,
|
|
return_logprob=return_logprob,
|
|
top_logprobs_nums=None,
|
|
token_ids_logprobs=None,
|
|
has_grammar=has_grammar,
|
|
)
|
|
|
|
def test_edit_rows_combine_grammar_mask_and_bias(self):
|
|
"""The grammar mask must be applied through the backend's own
|
|
apply_vocab_mask on a zeros base and summed with logit_bias —
|
|
guards the backend-agnostic zeros-trick, the combine order, and the
|
|
ForwardBatch.init_new grammars-population mirror (regression: the
|
|
MLX paths never build a ForwardBatch, so sinfo.grammars stayed None
|
|
and live grammar objects produced no mask at all)."""
|
|
from types import SimpleNamespace
|
|
|
|
import torch
|
|
|
|
class FakeGrammar:
|
|
def apply_vocab_mask(self, logits, vocab_mask):
|
|
logits[0, 3] = -float("inf") # row 0 forbids token 3
|
|
|
|
sinfo = SimpleNamespace(
|
|
grammars=None, # not yet populated, as on the real MLX path
|
|
logit_bias=torch.zeros(2, self.VOCAB).index_put_(
|
|
(torch.tensor([1]), torch.tensor([5])), torch.tensor([2.5])
|
|
),
|
|
vocab_size=self.VOCAB,
|
|
grammar_mask=None,
|
|
)
|
|
|
|
def update_mask():
|
|
sinfo.grammar_mask = SimpleNamespace(
|
|
grammar=FakeGrammar(), vocab_mask=torch.zeros(2, 1)
|
|
)
|
|
|
|
sinfo.update_regex_vocab_mask = update_mask
|
|
|
|
batch = self._batch(sinfo, has_grammar=True)
|
|
batch.reqs[0].grammar = object()
|
|
rows = self._worker()._build_logit_edit_rows(batch)
|
|
self.assertEqual(
|
|
[g is not None for g in sinfo.grammars],
|
|
[True, False],
|
|
"worker must mirror ForwardBatch.init_new's grammars population",
|
|
)
|
|
mx.eval(rows["r0"], rows["r1"])
|
|
self.assertEqual(rows["r0"].tolist()[3], -float("inf"))
|
|
self.assertEqual(rows["r1"].tolist()[5], 2.5)
|
|
self.assertIsNone(sinfo.grammar_mask, "mask must be released after use")
|
|
|
|
def test_edit_rows_none_when_nothing_to_edit(self):
|
|
from types import SimpleNamespace
|
|
|
|
sinfo = SimpleNamespace(grammars=None, logit_bias=None, vocab_size=self.VOCAB)
|
|
self.assertIsNone(self._worker()._build_logit_edit_rows(self._batch(sinfo)))
|
|
|
|
def test_logprob_spec_subset_alignment(self):
|
|
"""Spec rows must align to the rid subset order, not batch order —
|
|
guards mixed-batch decode sub-batches."""
|
|
from sglang.srt.hardware_backend.mlx.tp_worker import MlxTpModelWorker
|
|
|
|
rows = {"a": (3, None), "c": (0, (7, 9))}
|
|
spec = MlxTpModelWorker._logprob_spec_for(rows, ["c", "b", "a"])
|
|
self.assertEqual(spec.top_ks, (0, 0, 3))
|
|
self.assertEqual(spec.token_ids, ((7, 9), None, None))
|
|
self.assertIsNone(MlxTpModelWorker._logprob_spec_for(rows, ["x"]))
|
|
|
|
@unittest.skipUnless(
|
|
importlib.util.find_spec("xgrammar") is not None, "requires xgrammar"
|
|
)
|
|
def test_xgrammar_wrapper_supports_cpu_logits(self):
|
|
"""The MLX edit-row builder feeds CPU logits to the grammar
|
|
backend's apply_vocab_mask — regression: the xgrammar wrapper
|
|
raised 'Unsupported device: cpu' (its dispatch stopped at
|
|
cuda/xpu/musa/npu), so every grammar request crashed the worker."""
|
|
import math
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from sglang.srt.constrained.xgrammar_backend import XGrammarGrammar
|
|
|
|
logits = torch.zeros(1, 40)
|
|
blocks = math.ceil(40 / 32)
|
|
bitmask = torch.full((1, blocks), -1, dtype=torch.int32)
|
|
bitmask[0, 0] = int(np.int32(np.uint32(0xFFFFFFFF & ~(1 << 7))))
|
|
XGrammarGrammar.apply_vocab_mask(None, logits, bitmask)
|
|
self.assertEqual(logits[0, 7].item(), -float("inf"))
|
|
self.assertEqual(logits[0, 6].item(), 0.0)
|
|
|
|
def test_assemble_logprob_output_matches_scheduler_contract(self):
|
|
"""Field shapes must survive the scheduler's move_logprobs_to_cpu
|
|
(`.tolist()` on the batch tensor and on every per-row val/idx entry)
|
|
and add_logprob_return_values indexing — guards the external
|
|
LogitsProcessorOutput consumption contract, including rows without
|
|
logprob requests getting empty-but-tolistable fills."""
|
|
from types import SimpleNamespace
|
|
|
|
from sglang.srt.hardware_backend.mlx.tp_worker import MlxTpModelWorker
|
|
|
|
step_rows = {"a": (-1.5, [-0.1, -0.2], [4, 2], [-3.0], [9])}
|
|
reqs = [SimpleNamespace(rid="a"), SimpleNamespace(rid="b")]
|
|
out = MlxTpModelWorker._assemble_logprob_output(step_rows, reqs)
|
|
|
|
self.assertEqual(out.next_token_logprobs.tolist(), [-1.5, 0.0])
|
|
self.assertEqual(
|
|
[v.tolist() for v in out.next_token_top_logprobs_val],
|
|
[[-0.10000000149011612, -0.20000000298023224], []],
|
|
)
|
|
self.assertEqual(
|
|
[v.tolist() for v in out.next_token_top_logprobs_idx], [[4, 2], []]
|
|
)
|
|
self.assertEqual(
|
|
[v.tolist() for v in out.next_token_token_ids_logprobs_val],
|
|
[[-3.0], []],
|
|
)
|
|
self.assertEqual(out.next_token_token_ids_logprobs_idx, [[9], []])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|