From 55cc90b533cdf4a956bd9e32460c1e1a5829c218 Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Sun, 27 Sep 2026 20:23:07 -0700 Subject: [PATCH] [CI] Real-model Kimi-Linear PD parity at page, DCP virtual-page, chunk and cached-prefix boundaries (#41378) --- python/sglang/test/kits/pd_parity_kit.py | 86 +++++++++++++++---- .../test_disaggregation_kimi_linear.py | 78 ++++++++++++++--- 2 files changed, 133 insertions(+), 31 deletions(-) diff --git a/python/sglang/test/kits/pd_parity_kit.py b/python/sglang/test/kits/pd_parity_kit.py index 19b5ecb20d..acf62808cb 100644 --- a/python/sglang/test/kits/pd_parity_kit.py +++ b/python/sglang/test/kits/pd_parity_kit.py @@ -1,4 +1,5 @@ import time +from typing import Optional import requests @@ -14,16 +15,22 @@ class PDLogprobParityMixin: # Mix in before the PD server fixture, which owns the P/D launches. reference_parallel_args = [] baseline_args = [] + parity_prompts = [[1] + [100 + i % 1000 for i in range(256)]] + parity_max_new_tokens = 4 + parity_logprob_delta = 0.05 + # Prompts longer than this are resent after flushing only the decode side, + # so prefill serves them from its radix cache; None skips the resend. + parity_cached_prefix_min_prompt_tokens: Optional[int] = None - @staticmethod - def generate(base_url): + @classmethod + def _generate(cls, *, base_url, input_ids): response = requests.post( base_url + "/generate", json={ - "input_ids": [1] + [100 + i % 1000 for i in range(256)], + "input_ids": input_ids, "sampling_params": { "temperature": 0, - "max_new_tokens": 4, + "max_new_tokens": cls.parity_max_new_tokens, "ignore_eos": True, }, "return_logprob": True, @@ -34,7 +41,31 @@ class PDLogprobParityMixin: response.raise_for_status() return response.json()["meta_info"] - def test_logprob_parity(self): + @staticmethod + def _flush_cache(base_url): + response = requests.post( + base_url + "/flush_cache", params={"timeout": 30}, timeout=120 + ) + response.raise_for_status() + + def _assert_logprob_parity(self, *, reference, actual, label): + reference_logprobs = reference["output_token_logprobs"] + actual_logprobs = actual["output_token_logprobs"] + self.assertEqual( + [item[1] for item in reference_logprobs], + [item[1] for item in actual_logprobs], + label, + ) + self.assertEqual(len(reference_logprobs), self.parity_max_new_tokens, label) + for reference_item, actual_item in zip(reference_logprobs, actual_logprobs): + self.assertAlmostEqual( + reference_item[0], + actual_item[0], + delta=self.parity_logprob_delta, + msg=label, + ) + + def _collect_references(self): baseline = popen_launch_server( self.model, self.lb_url, @@ -45,25 +76,42 @@ class PDLogprobParityMixin: env=self.extra_prefill_env, ) try: - reference = self.generate(self.lb_url) + references = [] + for prompt in self.parity_prompts: + self._flush_cache(self.lb_url) + references.append( + self._generate(base_url=self.lb_url, input_ids=prompt) + ) finally: kill_process_tree(baseline.pid, wait_timeout=60) time.sleep(5) + return references - self.launch_all() - disaggregated = self.generate(self.lb_url) - - reference_logprobs = reference["output_token_logprobs"] - disaggregated_logprobs = disaggregated["output_token_logprobs"] - self.assertEqual( - [item[1] for item in reference_logprobs], - [item[1] for item in disaggregated_logprobs], + def _check_prompt_parity(self, *, prompt, reference): + self._flush_cache(self.prefill_url) + self._flush_cache(self.decode_url) + actual = self._generate(base_url=self.lb_url, input_ids=prompt) + self._assert_logprob_parity( + reference=reference, actual=actual, label=f"prompt_tokens={len(prompt)}" ) - self.assertEqual(len(reference_logprobs), 4) - for reference_item, disaggregated_item in zip( - reference_logprobs, disaggregated_logprobs - ): - self.assertAlmostEqual(reference_item[0], disaggregated_item[0], delta=0.05) + min_tokens = self.parity_cached_prefix_min_prompt_tokens + if min_tokens is None or len(prompt) <= min_tokens: + return + self._flush_cache(self.decode_url) + cached = self._generate(base_url=self.lb_url, input_ids=prompt) + self.assertGreater(cached["cached_tokens"], 0) + self._assert_logprob_parity( + reference=reference, + actual=cached, + label=f"cached prompt_tokens={len(prompt)}", + ) + + def test_logprob_parity(self): + references = self._collect_references() + self.launch_all() + for prompt, reference in zip(self.parity_prompts, references): + with self.subTest(prompt_tokens=len(prompt)): + self._check_prompt_parity(prompt=prompt, reference=reference) assert_process_healthy(self, "load balancer", self.process_lb, self.lb_url) assert_process_healthy(self, "prefill", self.process_prefill, self.prefill_url) diff --git a/test/registered/disaggregation/test_disaggregation_kimi_linear.py b/test/registered/disaggregation/test_disaggregation_kimi_linear.py index db6f37df72..44a9e5b275 100644 --- a/test/registered/disaggregation/test_disaggregation_kimi_linear.py +++ b/test/registered/disaggregation/test_disaggregation_kimi_linear.py @@ -1,16 +1,27 @@ import unittest +from transformers import AutoTokenizer + from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.pd_parity_kit import PDLogprobParityMixin from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) -register_cuda_ci(est_time=259, stage="base-c", runner_config="4-gpu-h100") +register_cuda_ci(est_time=400, stage="extra-b", runner_config="4-gpu-h100") -KIMI_LINEAR_MODEL = "yujiepan/kimi-linear-tiny-random" +KIMI_LINEAR_MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct" SERVER_ENV = {"SGLANG_BATCH_INVARIANT_OPS_ENABLE_MM_DEEPGEMM": "0"} +PAGE_SIZE = 16 +CHUNKED_PREFILL_SIZE = 64 +DCP_SIZE = 2 +# Prefill checkpoints the linear-attn state every 64 tokens, so shorter +# prompts never get a radix hit. +LINEAR_ATTN_CHECKPOINT_TOKENS = 64 + +# Deterministic so a cached-prefix prefill, which runs only the suffix, stays +# bit-exact with the reference's full prefill. DETERMINISTIC_ARGS = [ "--skip-tokenizer-init", "--random-seed", @@ -26,28 +37,71 @@ DETERMINISTIC_ARGS = [ "disabled", ] +DCP_ARGS = DETERMINISTIC_ARGS + [ + "--attention-backend", + "flashinfer", + "--page-size", + str(PAGE_SIZE), + "--chunked-prefill-size", + str(CHUNKED_PREFILL_SIZE), +] -class TestKimiLinearHeterogeneousTPDisaggregation( - PDLogprobParityMixin, PDDisaggregationServerBase -): + +def _boundary_prompts(tokenizer): + # Straddle the physical page, the DCP virtual page, and the prefill chunk. + filler = tokenizer.encode( + "Archive note: the weather was mild, the office lights were on, " + "and no unusual event was reported. ", + add_special_tokens=False, + ) + question = tokenizer.encode( + "\nSummarize the archive in one sentence:", add_special_tokens=False + ) + virtual_page_size = PAGE_SIZE * DCP_SIZE + lengths = sorted( + { + boundary + delta + for boundary in (PAGE_SIZE, virtual_page_size, CHUNKED_PREFILL_SIZE) + for delta in (-1, 0, 1) + } + | {2 * CHUNKED_PREFILL_SIZE + 1, 4 * CHUNKED_PREFILL_SIZE + 1} + ) + body = filler * (max(lengths) // len(filler) + 1) + return [body[: length - len(question)] + question for length in lengths] + + +# Every arm runs the same layout as its monolithic reference, so PD must match +# it token for token. +class TestKimiLinearTPDisaggregation(PDLogprobParityMixin, PDDisaggregationServerBase): model = KIMI_LINEAR_MODEL extra_prefill_env = SERVER_ENV extra_decode_env = SERVER_ENV prefill_tp_size = 2 - decode_tp_size = 1 + decode_tp_size = 2 decode_base_gpu_id = 2 reference_parallel_args = ["--tp-size", "2"] baseline_args = DETERMINISTIC_ARGS extra_prefill_args = DETERMINISTIC_ARGS extra_decode_args = DETERMINISTIC_ARGS + parity_max_new_tokens = 8 + # Measured 0.0 on every prompt; a corrupted DCP relayout moved it by >= 0.05. + parity_logprob_delta = 1e-3 + parity_cached_prefix_min_prompt_tokens = LINEAR_ATTN_CHECKPOINT_TOKENS + + @classmethod + def setUpClass(cls): + super().setUpClass() + tokenizer = AutoTokenizer.from_pretrained(cls.model, trust_remote_code=True) + cls.parity_prompts = _boundary_prompts(tokenizer) -class TestKimiLinearPipelineDisaggregation(TestKimiLinearHeterogeneousTPDisaggregation): - prefill_tp_size = 1 - decode_tp_size = 1 - decode_base_gpu_id = 2 - reference_parallel_args = ["--tp-size", "1", "--pp-size", "2"] - extra_prefill_args = DETERMINISTIC_ARGS + ["--pp-size", "2"] +class TestKimiLinearDCPDisaggregation(TestKimiLinearTPDisaggregation): + reference_parallel_args = ["--tp-size", "2", "--dcp-size", str(DCP_SIZE)] + baseline_args = DCP_ARGS + extra_prefill_args = DCP_ARGS + extra_decode_args = DCP_ARGS + ["--dcp-size", str(DCP_SIZE)] + # Deterministic flashinfer runs without the radix cache. + parity_cached_prefix_min_prompt_tokens = None if __name__ == "__main__":