Files
sglang/test/registered/radix_cache/test_mamba_cache_kl.py
T

250 lines
9.4 KiB
Python

"""KL regression tests for Mamba prefix-cache correctness.
Keep each scenario in its own class so server configurations and cache-pressure
requirements remain explicit.
"""
import unittest
import numpy as np
import requests
from transformers import AutoConfig, AutoTokenizer
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
register_cuda_ci(est_time=240, stage="extra-a", runner_config="1-gpu-large")
class MambaCacheKLMixin:
ignore_eos = False
def _generate(self, input_ids, max_new_tokens, *, score=False):
payload = {
"input_ids": input_ids,
"sampling_params": {
"temperature": 0,
"max_new_tokens": max_new_tokens,
"ignore_eos": self.ignore_eos,
},
"return_logprob": True,
"logprob_start_len": -1,
}
if score:
payload["token_ids_logprob"] = self.vocab_ids
response = requests.post(self.base_url + "/generate", json=payload, timeout=120)
response.raise_for_status()
result = response.json()
self.assertEqual(result["meta_info"]["completion_tokens"], max_new_tokens)
return result
def _flush_cache(self):
response = requests.post(self.base_url + "/flush_cache", timeout=30)
response.raise_for_status()
def _distribution(self, result):
rows = result["meta_info"]["output_token_ids_logprobs"]
self.assertEqual(len(rows), 1)
self.assertEqual([row[1] for row in rows[0]], self.vocab_ids)
logprobs = np.asarray([row[0] for row in rows[0]], dtype=np.float64)
self.assertTrue(np.isfinite(logprobs).all())
self.assertAlmostEqual(float(np.exp(logprobs).sum()), 1.0, places=4)
# Remove log-softmax rounding error before computing the exact KL.
return logprobs - np.logaddexp.reduce(logprobs)
class TestMambaLazyInplaceKL(MambaCacheKLMixin, DefaultServerBase):
"""Guard lazy Mamba checkpoint reuse when the optional track slot is exhausted.
Keep overlap scheduling and decode CUDA graphs enabled. A four-slot pool holds
two cached prefixes, the request's live state, and its keep checkpoint. The
first decode must track in place; delayed prefill processing must not publish
that buffer under the old checkpoint length.
Compare full-vocabulary next-token distributions at the SAME 255-token prompt,
before and after a 32-token decode seeds the cache. Comparing later free-running
tokens would confound cache corruption with different generated histories.
GB300 / Qwen3.5-0.8B: reverting the checkpoint invalidation gives KL=0.649448
and a stale 192-token cache hit; with the fix KL=0 and the safe hit is 128 tokens.
"""
ignore_eos = True
model = "Qwen/Qwen3.5-0.8B"
server_env = {
"SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "0",
"SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL": "0",
}
other_args = [
"--skip-tokenizer-init",
"--mamba-radix-cache-strategy",
"extra_buffer_lazy",
"--mamba-track-interval",
"128",
"--max-mamba-cache-size",
"4",
"--max-running-requests",
"1",
"--max-total-tokens",
"8192",
"--mem-fraction-static",
"0.3",
"--cuda-graph-max-bs-decode",
"1",
"--cuda-graph-backend-prefill",
"disabled",
"--chunked-prefill-size",
"2048",
"--attention-backend",
"triton",
"--linear-attn-backend",
"triton",
"--mamba-backend",
"triton",
"--random-seed",
"42",
]
def _warm_prefixes(self):
self._flush_cache()
self._generate([44] * 128, 1)
self._generate([42] * 128, 1)
def test_lazy_donation_with_four_slots_and_one_request(self):
"""A prefix-hit request must finish without a fifth Mamba slot under overlap."""
self._flush_cache()
self._generate([42] * 128, 1)
# One cached prefix, live state, and keep checkpoint occupy three slots;
# the boundary decode takes the fourth before prefill donation runs.
result = self._generate([42] * 128 + [43] * 127, 32)
self.assertEqual(result["meta_info"]["cached_tokens"], 128)
logprobs = result["meta_info"]["output_token_logprobs"]
self.assertEqual(len(logprobs), 32)
self.assertTrue(np.isfinite([row[0] for row in logprobs]).all())
def test_reuse_distribution_after_inplace_tracking(self):
config = AutoConfig.from_pretrained(self.model)
self.vocab_ids = list(range(config.get_text_config().vocab_size))
prompt = [42] * 128 + [43] * 127
# A single output token does not launch the boundary-crossing decode.
self._warm_prefixes()
reference = self._generate(prompt, 1, score=True)
self.assertEqual(reference["meta_info"]["cached_tokens"], 128)
reference_logprobs = self._distribution(reference)
self._warm_prefixes()
seed = self._generate(prompt, 32)
self.assertEqual(seed["meta_info"]["cached_tokens"], 128)
reused = self._generate(prompt, 1, score=True)
self.assertGreaterEqual(reused["meta_info"]["cached_tokens"], 128)
reused_logprobs = self._distribution(reused)
kl = float(
np.sum(np.exp(reference_logprobs) * (reference_logprobs - reused_logprobs))
)
print(
f"lazy in-place checkpoint KL(reference || reuse)={kl:.9g}, "
f"cached_tokens={reused['meta_info']['cached_tokens']}",
flush=True,
)
self.assertGreaterEqual(kl, -1e-12)
# Leave room for numerical noise while staying far below the broken run.
self.assertLess(kl, 1e-4, "Cache reuse changed the next-token distribution")
class TestMambaSpecTruncationKL(MambaCacheKLMixin, DefaultServerBase):
"""NEXTN can advance the no_buffer state past max_new_tokens. Compare warm
and cold distributions for identical token IDs to detect a misaligned key.
Different prefill splits introduce numerical noise, so KL need not be zero.
"""
model = "Qwen/Qwen3.5-0.8B"
revision = "2fc06364715b967f1860aea9cf38778875588b17"
other_args = [
"--revision",
revision,
"--mamba-radix-cache-strategy",
"no_buffer",
"--page-size",
"1",
"--disable-overlap-schedule",
"--disable-cuda-graph",
"--mem-fraction-static",
"0.25",
"--max-total-tokens",
"8192",
"--max-running-requests",
"8",
"--max-mamba-cache-size",
"128",
"--attention-backend",
"triton",
"--linear-attn-decode-backend",
"triton",
"--linear-attn-prefill-backend",
"triton",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
]
def test_reuse_distribution_after_length_truncation(self):
tokenizer = AutoTokenizer.from_pretrained(self.model, revision=self.revision)
config = AutoConfig.from_pretrained(self.model, revision=self.revision)
self.vocab_ids = list(range(config.get_text_config().vocab_size))
newline = tokenizer.encode("\n", add_special_tokens=False)
cases = (
(
"count",
"Continue the sequence with numbers separated by spaces: 1 2 3 4 5 6 7 8 9 10",
9,
),
("story", "Write a story about a cat who likes fish.", 5),
)
for name, text, limit in cases:
with self.subTest(case=name):
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": text}],
tokenize=True,
return_dict=False,
add_generation_prompt=True,
enable_thinking=False,
)
self._flush_cache()
seed = self._generate(prompt, limit)
self.assertGreater(seed["meta_info"]["spec_verify_ct"], 0)
self.assertGreater(seed["meta_info"]["spec_num_correct_drafts"], 0)
self.assertEqual(seed["meta_info"]["finish_reason"]["type"], "length")
probe = prompt + seed["output_ids"] + newline
warm = self._generate(probe, 1, score=True)
self.assertGreater(warm["meta_info"]["cached_tokens"], 0)
self._flush_cache()
cold = self._generate(probe, 1, score=True)
self.assertEqual(cold["meta_info"]["cached_tokens"], 0)
cold_logprobs = self._distribution(cold)
warm_logprobs = self._distribution(warm)
kl = float(
np.sum(np.exp(cold_logprobs) * (cold_logprobs - warm_logprobs))
)
print(
f"no_buffer NEXTN {name} KL(cold || warm)={kl:.9g}, "
f"cached_tokens={warm['meta_info']['cached_tokens']}, "
f"prompt_tokens={len(prompt)}",
flush=True,
)
self.assertGreaterEqual(kl, -1e-12)
self.assertLess(
kl, 5e-3, "Truncated speculative state corrupted cache reuse"
)
if __name__ == "__main__":
unittest.main()