diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index 928e3cb410..acec238bac 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -1422,6 +1422,9 @@ class SchedulerBatchResultProcessor: self.decode_offload_manager.offload_kv_cache(req) if req.finished(): + if not self._mamba_can_cache_finished_req(req, result, i): + req.skip_radix_cache_insert = True + # isinstance narrowing: create_worker may also return plain # TpModelWorker-based drafts, which carry no spec-worker hooks. if isinstance(self.draft_worker, BaseSpecWorker): @@ -1455,6 +1458,30 @@ class SchedulerBatchResultProcessor: self._maybe_collect_customized_info(i, req, logits_output) + def _mamba_can_cache_finished_req( + self, req: Req, result: GenerationBatchResult, i: int + ) -> bool: + if ( + result.num_correct_drafts_per_req_cpu is None + or req.kv.mamba_ping_pong_track_buffer is not None + or not self.tree_cache.supports_mamba() + ): + return True + + # no_buffer donates the live state, which cannot be shortened with the key. + if req.owned_kv_len() != req.kv.kv_committed_len: + return False + + retained = result.grammar_retained_tokens + if retained is None or retained[i] is None: + return True + + num_verified = ( + result.num_correct_drafts_per_req_cpu[i] + + result.num_non_draft_tokens_per_req + ) + return len(retained[i]) == num_verified + def _maybe_update_reasoning_tokens( self, req: Req, diff --git a/test/registered/radix_cache/test_mamba_cache_kl.py b/test/registered/radix_cache/test_mamba_cache_kl.py index 511c80b0fe..b0543c14cf 100644 --- a/test/registered/radix_cache/test_mamba_cache_kl.py +++ b/test/registered/radix_cache/test_mamba_cache_kl.py @@ -8,15 +8,52 @@ import unittest import numpy as np import requests -from transformers import AutoConfig +from transformers import AutoConfig, AutoTokenizer from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.server_fixtures.default_fixture import DefaultServerBase -register_cuda_ci(est_time=120, stage="extra-a", runner_config="1-gpu-large") +register_cuda_ci(est_time=240, stage="extra-a", runner_config="1-gpu-large") -class TestMambaLazyInplaceKL(DefaultServerBase): +class MambaCacheKLMixin: + ignore_eos = False + + def _generate(self, input_ids, max_new_tokens, *, score=False): + payload = { + "input_ids": input_ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": self.ignore_eos, + }, + "return_logprob": True, + "logprob_start_len": -1, + } + if score: + payload["token_ids_logprob"] = self.vocab_ids + response = requests.post(self.base_url + "/generate", json=payload, timeout=120) + response.raise_for_status() + result = response.json() + self.assertEqual(result["meta_info"]["completion_tokens"], max_new_tokens) + return result + + def _flush_cache(self): + response = requests.post(self.base_url + "/flush_cache", timeout=30) + response.raise_for_status() + + def _distribution(self, result): + rows = result["meta_info"]["output_token_ids_logprobs"] + self.assertEqual(len(rows), 1) + self.assertEqual([row[1] for row in rows[0]], self.vocab_ids) + logprobs = np.asarray([row[0] for row in rows[0]], dtype=np.float64) + self.assertTrue(np.isfinite(logprobs).all()) + self.assertAlmostEqual(float(np.exp(logprobs).sum()), 1.0, places=4) + # Remove log-softmax rounding error before computing the exact KL. + return logprobs - np.logaddexp.reduce(logprobs) + + +class TestMambaLazyInplaceKL(MambaCacheKLMixin, DefaultServerBase): """Guard lazy Mamba checkpoint reuse when the optional track slot is exhausted. Keep overlap scheduling and decode CUDA graphs enabled. A four-slot pool holds @@ -32,6 +69,7 @@ class TestMambaLazyInplaceKL(DefaultServerBase): and a stale 192-token cache hit; with the fix KL=0 and the safe hit is 128 tokens. """ + ignore_eos = True model = "Qwen/Qwen3.5-0.8B" server_env = { "SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "0", @@ -67,41 +105,11 @@ class TestMambaLazyInplaceKL(DefaultServerBase): "42", ] - def _generate(self, input_ids, max_new_tokens, *, score=False): - payload = { - "input_ids": input_ids, - "sampling_params": { - "temperature": 0, - "max_new_tokens": max_new_tokens, - "ignore_eos": True, - }, - "return_logprob": True, - "logprob_start_len": -1, - } - if score: - payload["token_ids_logprob"] = self.vocab_ids - response = requests.post(self.base_url + "/generate", json=payload, timeout=120) - response.raise_for_status() - result = response.json() - self.assertEqual(result["meta_info"]["completion_tokens"], max_new_tokens) - return result - def _warm_prefixes(self): - response = requests.post(self.base_url + "/flush_cache", timeout=30) - response.raise_for_status() + self._flush_cache() self._generate([44] * 128, 1) self._generate([42] * 128, 1) - def _distribution(self, result): - rows = result["meta_info"]["output_token_ids_logprobs"] - self.assertEqual(len(rows), 1) - self.assertEqual([row[1] for row in rows[0]], self.vocab_ids) - logprobs = np.asarray([row[0] for row in rows[0]], dtype=np.float64) - self.assertTrue(np.isfinite(logprobs).all()) - self.assertAlmostEqual(float(np.exp(logprobs).sum()), 1.0, places=4) - # Remove log-softmax rounding error before computing the exact KL. - return logprobs - np.logaddexp.reduce(logprobs) - def test_reuse_distribution_after_inplace_tracking(self): config = AutoConfig.from_pretrained(self.model) self.vocab_ids = list(range(config.get_text_config().vocab_size)) @@ -133,5 +141,96 @@ class TestMambaLazyInplaceKL(DefaultServerBase): self.assertLess(kl, 1e-4, "Cache reuse changed the next-token distribution") +class TestMambaSpecTruncationKL(MambaCacheKLMixin, DefaultServerBase): + """NEXTN can advance the no_buffer state past max_new_tokens. Compare warm + and cold distributions for identical token IDs to detect a misaligned key. + Different prefill splits introduce numerical noise, so KL need not be zero. + """ + + model = "Qwen/Qwen3.5-0.8B" + revision = "2fc06364715b967f1860aea9cf38778875588b17" + other_args = [ + "--revision", + revision, + "--mamba-radix-cache-strategy", + "no_buffer", + "--page-size", + "1", + "--disable-overlap-schedule", + "--disable-cuda-graph", + "--mem-fraction-static", + "0.25", + "--max-total-tokens", + "8192", + "--max-running-requests", + "8", + "--max-mamba-cache-size", + "128", + "--attention-backend", + "triton", + "--linear-attn-decode-backend", + "triton", + "--linear-attn-prefill-backend", + "triton", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + ] + + def test_reuse_distribution_after_length_truncation(self): + tokenizer = AutoTokenizer.from_pretrained(self.model, revision=self.revision) + config = AutoConfig.from_pretrained(self.model, revision=self.revision) + self.vocab_ids = list(range(config.get_text_config().vocab_size)) + newline = tokenizer.encode("\n", add_special_tokens=False) + cases = ( + ( + "count", + "Continue the sequence with numbers separated by spaces: 1 2 3 4 5 6 7 8 9 10", + 9, + ), + ("story", "Write a story about a cat who likes fish.", 5), + ) + for name, text, limit in cases: + with self.subTest(case=name): + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": text}], + tokenize=True, + return_dict=False, + add_generation_prompt=True, + enable_thinking=False, + ) + self._flush_cache() + seed = self._generate(prompt, limit) + self.assertGreater(seed["meta_info"]["spec_verify_ct"], 0) + self.assertGreater(seed["meta_info"]["spec_num_correct_drafts"], 0) + self.assertEqual(seed["meta_info"]["finish_reason"]["type"], "length") + probe = prompt + seed["output_ids"] + newline + warm = self._generate(probe, 1, score=True) + self.assertGreater(warm["meta_info"]["cached_tokens"], 0) + self._flush_cache() + cold = self._generate(probe, 1, score=True) + self.assertEqual(cold["meta_info"]["cached_tokens"], 0) + cold_logprobs = self._distribution(cold) + warm_logprobs = self._distribution(warm) + kl = float( + np.sum(np.exp(cold_logprobs) * (cold_logprobs - warm_logprobs)) + ) + print( + f"no_buffer NEXTN {name} KL(cold || warm)={kl:.9g}, " + f"cached_tokens={warm['meta_info']['cached_tokens']}, " + f"prompt_tokens={len(prompt)}", + flush=True, + ) + self.assertGreaterEqual(kl, -1e-12) + self.assertLess( + kl, 5e-3, "Truncated speculative state corrupted cache reuse" + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_batch_result_processor_spec_grammar.py b/test/registered/unit/managers/test_batch_result_processor_spec_grammar.py index 6fc1a88c42..237827fa81 100644 --- a/test/registered/unit/managers/test_batch_result_processor_spec_grammar.py +++ b/test/registered/unit/managers/test_batch_result_processor_spec_grammar.py @@ -5,7 +5,9 @@ token, so the over-drafted suffix is never committed to KV nor emitted. """ import unittest +from dataclasses import replace from types import SimpleNamespace +from unittest.mock import MagicMock import torch @@ -15,6 +17,8 @@ from sglang.srt.managers.scheduler_components.batch_result_processor import ( SchedulerBatchResultProcessor, ) from sglang.srt.managers.utils import GenerationBatchResult +from sglang.srt.mem_cache.common import release_kv_cache +from sglang.srt.runtime_context import get_context from sglang.srt.sampling.sampling_params import ( REQUEST_REASONING_END_TOKEN_IDS_KEY, SamplingParams, @@ -251,5 +255,82 @@ class TestReasoningTokenAccounting(CustomTestCase): self.assertFalse(req._is_reasoning_over) +class TestFinishedMambaSpecCheckpoint(CustomTestCase): + def test_finished_checkpoint_boundaries(self): + processor = replace( + _make_processor(), tree_cache=SimpleNamespace(supports_mamba=lambda: True) + ) + cases = ( + ("grammar", [4], 1, 1, 0, False), + ("zero_commit", [], 0, 1, 0, False), + ("eos_or_stop", None, 1, 1, 0, False), + ("length", None, 2, 1, 0, False), + ("aligned", None, 3, 1, 0, True), + ("grammar_aligned", [4, 5, 6], 3, 1, 0, True), + ("two_non_draft", [4, 5, 6], 3, 2, 0, False), + ("dedicated_slot", [4], 1, 1, 1, True), + ("ping_pong_slots", [4], 1, 1, 2, True), + ) + for name, retained, finish_len, non_draft, slots, expected in cases: + with ( + self.subTest(case=name), + get_context().override_server_args(strip_thinking_cache=False), + ): + req = _make_req(1) + req.output_ids = [4, 5, 6] + req.finished_len = finish_len + req.kv.kv_committed_len = 3 + ( + len(retained) if retained is not None else 2 + non_draft + ) + req.kv.mamba_ping_pong_track_buffer = ( + torch.arange(slots) if slots else None + ) + result = _make_result(3, [3], [4, 5, 6]) + result.num_correct_drafts_per_req_cpu = [2] + result.num_non_draft_tokens_per_req = non_draft + result.grammar_retained_tokens = [retained] + self.assertEqual( + processor._mamba_can_cache_finished_req(req, result, 0), expected + ) + + def test_skip_checkpoint_survives_deferred_release(self): + with get_context().override_server_args( + speculative_algorithm="EAGLE", + mamba_radix_cache_strategy="no_buffer", + disable_overlap_schedule=True, + disaggregation_decode_enable_offload_kvcache=True, + ): + req = _make_req(1) + req.output_ids = [3] + req.kv.kv_committed_len = 3 + req.kv.kv_allocated_len = 6 + req.kv.req_pool_idx = 0 + result = _make_result(3, [3], [4, 5, 6]) + batch = _FakeBatch([req]) + batch.mamba_track_mask_cpu = None + tree = MagicMock() + tree.supports_mamba.return_value = True + tree.claim_kv_row.return_value = False + tree.token_to_kv_pool_allocator.page_size = 1 + processor = replace( + _make_processor(), + tree_cache=tree, + decode_offload_manager=SimpleNamespace( + offload_kv_cache=lambda req: True + ), + ) + tokens = processor._resolve_spec_v2_tokens(result, batch)[0] + req.output_ids.extend(tokens) + req.update_finish_state(len(tokens)) + self.assertTrue(req.finished()) + + processor._handle_finish_state_updated_req(req, batch, result, 0, None) + self.assertTrue(req.skip_radix_cache_insert) + self.assertFalse(req.kv.is_kv_released) + release_kv_cache(req, tree, checkpoint=True) + tree.checkpoint.assert_not_called() + self.assertTrue(req.kv.is_kv_released) + + if __name__ == "__main__": unittest.main()