From b20c375c10443e4f4a5656689a04d514194364fd Mon Sep 17 00:00:00 2001 From: Ke Bao Date: Tue, 11 Aug 2026 21:47:01 +0800 Subject: [PATCH] Fix flaky decode cache-hit check in Inkling test (#34405) --- python/sglang/test/kl_test_utils.py | 6 +++++- .../models_e2e/test_inkling_small_nvfp4.py | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/python/sglang/test/kl_test_utils.py b/python/sglang/test/kl_test_utils.py index b6c5b3751..dd5ae682c 100644 --- a/python/sglang/test/kl_test_utils.py +++ b/python/sglang/test/kl_test_utils.py @@ -308,6 +308,7 @@ def test_input_output_logprobs_match_decode_cache_hit_helper( max_samples=None, max_new_tokens=8192, trust_remote_code=False, + min_cache_hit_ratio=0.5, ): server_info = requests.get(base_url + "/server_info").json() if server_info["disable_radix_cache"]: @@ -363,7 +364,10 @@ def test_input_output_logprobs_match_decode_cache_hit_helper( output_logprobs.append(_extract_output_logprobs(result)) if not os.environ.get("SGLANG_TEST_SKIP_CACHE_HIT_ASSERT"): - assert len(new_input_ids) > 0.5 * len( + # Page-aligned SWA retention decides which prompts hit at all, so the default + # only screens out a vacuous run. A caller whose checkpoint interval makes + # every prompt hit raises this to pin that down. + assert len(new_input_ids) > min_cache_hit_ratio * len( second_turn_input_ids ), f"Too few decode cache hits: {len(new_input_ids)}/{len(second_turn_input_ids)}" diff --git a/test/registered/models_e2e/test_inkling_small_nvfp4.py b/test/registered/models_e2e/test_inkling_small_nvfp4.py index 40998b8a6..c32fb6932 100644 --- a/test/registered/models_e2e/test_inkling_small_nvfp4.py +++ b/test/registered/models_e2e/test_inkling_small_nvfp4.py @@ -58,6 +58,12 @@ KL_DIV_THRESHOLD = 1e-9 # checkpoint or a mis-restored prefix would surface. KL_MAX_NEW_TOKENS = 1024 +# Equal to the page size below. Out-of-window SWA slots are freed a page at a +# time, so only a checkpoint sitting on a page boundary still has a full window +# of SWA data below it -- at the default 256 half the sequence lengths land off +# that boundary and lose their decode prefix entirely. +KL_TRACK_INTERVAL = 128 + class TestInklingSmallNvfp4(CustomTestCase): @classmethod @@ -162,6 +168,8 @@ class TestInklingSmallNvfp4Deterministic(CustomTestCase): "0.1", "--mem-fraction-static", "0.85", + "--mamba-track-interval", + str(KL_TRACK_INTERVAL), "--enable-deterministic-inference", ], env={**os.environ, "SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}, @@ -172,7 +180,7 @@ class TestInklingSmallNvfp4Deterministic(CustomTestCase): if getattr(cls, "process", None) is not None: kill_process_tree(cls.process.pid) - def _run(self, helper): + def _run(self, helper, **kwargs): helper( self.base_url, {self.model: {"kl_div": KL_DIV_THRESHOLD}}, @@ -180,6 +188,7 @@ class TestInklingSmallNvfp4Deterministic(CustomTestCase): max_samples=32, max_new_tokens=KL_MAX_NEW_TOKENS, trust_remote_code=True, + **kwargs, ) def test_input_output_logprobs_match(self): @@ -189,7 +198,9 @@ class TestInklingSmallNvfp4Deterministic(CustomTestCase): self._run(assert_logprobs_match_prefill_cache_hit) def test_input_output_logprobs_match_decode_cache_hit(self): - self._run(assert_logprobs_match_decode_cache_hit) + # 0.99 is every prompt: the interval above makes the reuse unconditional, so + # a single miss is a state-reuse regression rather than a geometry coincidence. + self._run(assert_logprobs_match_decode_cache_hit, min_cache_hit_ratio=0.99) if __name__ == "__main__":