diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 7499375032..5df48b9e2c 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -3485,6 +3485,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): if new_slot is not None: pool.set_mamba_ping_pong_slot(req, other_idx, new_slot[0]) req.kv.mamba_next_track_idx = other_idx + else: + # The in-flight write invalidates the keep slot's old checkpoint. + req.kv.mamba_last_track_seqlen = None def mamba_lazy_spec_prepare(self, mamba_track_interval: int, max_draft_tokens: int): """Lazy-mode spec counterpart of mamba_lazy_prealloc_at_boundary. @@ -3521,6 +3524,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): if new_slot is not None: pool.set_mamba_ping_pong_slot(req, other_idx, new_slot[0]) has_pending = True + else: + req.kv.mamba_last_track_seqlen = None # On failure the verify scatters in place into the keep slot. track_positions.append( other_idx if has_pending else req.kv.mamba_next_track_idx diff --git a/test/registered/radix_cache/test_mamba_cache_kl.py b/test/registered/radix_cache/test_mamba_cache_kl.py new file mode 100644 index 0000000000..511c80b0fe --- /dev/null +++ b/test/registered/radix_cache/test_mamba_cache_kl.py @@ -0,0 +1,137 @@ +"""KL regression tests for Mamba prefix-cache correctness. + +Keep each scenario in its own class so server configurations and cache-pressure +requirements remain explicit. +""" + +import unittest + +import numpy as np +import requests +from transformers import AutoConfig + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.server_fixtures.default_fixture import DefaultServerBase + +register_cuda_ci(est_time=120, stage="extra-a", runner_config="1-gpu-large") + + +class TestMambaLazyInplaceKL(DefaultServerBase): + """Guard lazy Mamba checkpoint reuse when the optional track slot is exhausted. + + Keep overlap scheduling and decode CUDA graphs enabled. A four-slot pool holds + two cached prefixes, the request's live state, and its keep checkpoint. The + first decode must track in place; delayed prefill processing must not publish + that buffer under the old checkpoint length. + + Compare full-vocabulary next-token distributions at the SAME 255-token prompt, + before and after a 32-token decode seeds the cache. Comparing later free-running + tokens would confound cache corruption with different generated histories. + + GB300 / Qwen3.5-0.8B: reverting the checkpoint invalidation gives KL=0.649448 + and a stale 192-token cache hit; with the fix KL=0 and the safe hit is 128 tokens. + """ + + model = "Qwen/Qwen3.5-0.8B" + server_env = { + "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "0", + "SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL": "0", + } + other_args = [ + "--skip-tokenizer-init", + "--mamba-radix-cache-strategy", + "extra_buffer_lazy", + "--mamba-track-interval", + "128", + "--max-mamba-cache-size", + "4", + "--max-running-requests", + "1", + "--max-total-tokens", + "8192", + "--mem-fraction-static", + "0.3", + "--cuda-graph-max-bs-decode", + "1", + "--cuda-graph-backend-prefill", + "disabled", + "--chunked-prefill-size", + "2048", + "--attention-backend", + "triton", + "--linear-attn-backend", + "triton", + "--mamba-backend", + "triton", + "--random-seed", + "42", + ] + + def _generate(self, input_ids, max_new_tokens, *, score=False): + payload = { + "input_ids": input_ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + "return_logprob": True, + "logprob_start_len": -1, + } + if score: + payload["token_ids_logprob"] = self.vocab_ids + response = requests.post(self.base_url + "/generate", json=payload, timeout=120) + response.raise_for_status() + result = response.json() + self.assertEqual(result["meta_info"]["completion_tokens"], max_new_tokens) + return result + + def _warm_prefixes(self): + response = requests.post(self.base_url + "/flush_cache", timeout=30) + response.raise_for_status() + self._generate([44] * 128, 1) + self._generate([42] * 128, 1) + + def _distribution(self, result): + rows = result["meta_info"]["output_token_ids_logprobs"] + self.assertEqual(len(rows), 1) + self.assertEqual([row[1] for row in rows[0]], self.vocab_ids) + logprobs = np.asarray([row[0] for row in rows[0]], dtype=np.float64) + self.assertTrue(np.isfinite(logprobs).all()) + self.assertAlmostEqual(float(np.exp(logprobs).sum()), 1.0, places=4) + # Remove log-softmax rounding error before computing the exact KL. + return logprobs - np.logaddexp.reduce(logprobs) + + def test_reuse_distribution_after_inplace_tracking(self): + config = AutoConfig.from_pretrained(self.model) + self.vocab_ids = list(range(config.get_text_config().vocab_size)) + prompt = [42] * 128 + [43] * 127 + + # A single output token does not launch the boundary-crossing decode. + self._warm_prefixes() + reference = self._generate(prompt, 1, score=True) + self.assertEqual(reference["meta_info"]["cached_tokens"], 128) + reference_logprobs = self._distribution(reference) + + self._warm_prefixes() + seed = self._generate(prompt, 32) + self.assertEqual(seed["meta_info"]["cached_tokens"], 128) + reused = self._generate(prompt, 1, score=True) + self.assertGreaterEqual(reused["meta_info"]["cached_tokens"], 128) + reused_logprobs = self._distribution(reused) + + kl = float( + np.sum(np.exp(reference_logprobs) * (reference_logprobs - reused_logprobs)) + ) + print( + f"lazy in-place checkpoint KL(reference || reuse)={kl:.9g}, " + f"cached_tokens={reused['meta_info']['cached_tokens']}", + flush=True, + ) + self.assertGreaterEqual(kl, -1e-12) + # Leave room for numerical noise while staying far below the broken run. + self.assertLess(kl, 1e-4, "Cache reuse changed the next-token distribution") + + +if __name__ == "__main__": + unittest.main()