From d59c3448b085173aeebee9b60e400b7e3854db93 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:03:27 -0700 Subject: [PATCH] [Fix] kimi-k3: Preserve pipeline graph bank inputs and bound chunk profiling (#43604) --- .../dynamic_chunk_sizer.py | 62 +++++---- python/sglang/srt/models/kimi_k3.py | 2 + .../test_dynamic_chunk_sizer.py | 127 ++++++++++++++++++ .../models/test_kimi_k3_pipeline_layers.py | 125 +++++++++++++++++ 4 files changed, 288 insertions(+), 28 deletions(-) create mode 100644 test/registered/unit/managers/scheduler_components/test_dynamic_chunk_sizer.py create mode 100644 test/registered/unit/models/test_kimi_k3_pipeline_layers.py diff --git a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py index c6ce743e0c..858ada8748 100644 --- a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py +++ b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py @@ -140,11 +140,16 @@ class DynamicChunkSizer: model_runner = self.model_runner model_config = model_runner.model_config input_ids_list: List[array[int]] = [] + # Each sample is a complete request. Allocating past its mapping row + # would leave KV slots that release_kv_cache cannot address. + max_profile_tokens = min( + int(self.chunked_prefill_size * 1.25), + model_config.context_len, + self.req_to_token_pool.max_context_len, + ) + step = max(1, max_profile_tokens // 128) for i in range(128): - chunk_size = int( - self.chunked_prefill_size * 1.25 - - i * (self.chunked_prefill_size * 1.25 // 128) - ) + chunk_size = max_profile_tokens - i * step if chunk_size <= 0: break input_ids = array( @@ -221,34 +226,35 @@ class DynamicChunkSizer: start = time.perf_counter() batch.prepare_for_extend() - # Resolve deferred H2D: prepare_for_extend now leaves input_ids=None - if batch.input_ids is None and batch.prefill_input_ids_cpu is not None: - batch.input_ids = batch.prefill_input_ids_cpu.to( - self.device, non_blocking=True + try: + # Resolve deferred H2D: prepare_for_extend now leaves input_ids=None + if batch.input_ids is None and batch.prefill_input_ids_cpu is not None: + batch.input_ids = batch.prefill_input_ids_cpu.to( + self.device, non_blocking=True + ) + batch.prefill_input_ids_cpu = None + + forward_batch = ForwardBatch.init_new( + batch, + model_runner, + return_hidden_states_before_norm=False, ) - batch.prefill_input_ids_cpu = None + set_is_extend_in_batch(batch.forward_mode.is_extend()) - forward_batch = ForwardBatch.init_new( - batch, - model_runner, - return_hidden_states_before_norm=False, - ) - set_is_extend_in_batch(batch.forward_mode.is_extend()) + _ = model_runner.forward( + forward_batch=forward_batch, pp_proxy_tensors=pp_proxy + ) - _ = model_runner.forward( - forward_batch=forward_batch, pp_proxy_tensors=pp_proxy - ) + # Synchronize after forward to ensure GPU operations complete + device_module.synchronize() - # Synchronize after forward to ensure GPU operations complete - device_module.synchronize() - - latency_seconds = time.perf_counter() - start - latency_ms = latency_seconds * 1e3 # Convert to milliseconds - seq_lens.append(len(input_ids)) - latencies.append(latency_ms) - - # Release KV and Mamba cache - if req.kv.holds_kv: + latency_seconds = time.perf_counter() - start + latency_ms = latency_seconds * 1e3 # Convert to milliseconds + seq_lens.append(len(input_ids)) + latencies.append(latency_ms) + finally: + # Profiling failures disable the predictor; they must still + # return the synthetic request's KV and recurrent state. release_kv_cache(req, self.tree_cache, checkpoint=False) logger.info( diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 1979a2e76d..9158dd950b 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -3005,6 +3005,8 @@ class KimiK3LinearForCausalLM(nn.Module): self.model = KimiK3LinearModel( config, quant_config, prefix=maybe_prefix(prefix, "model") ) + self.start_layer = self.model.start_layer + self.end_layer = self.model.end_layer self.pp_group = get_parallel().pp_group if self.pp_group.is_last_rank: self.lm_head = ParallelLMHead( diff --git a/test/registered/unit/managers/scheduler_components/test_dynamic_chunk_sizer.py b/test/registered/unit/managers/scheduler_components/test_dynamic_chunk_sizer.py new file mode 100644 index 0000000000..eb2f3877fa --- /dev/null +++ b/test/registered/unit/managers/scheduler_components/test_dynamic_chunk_sizer.py @@ -0,0 +1,127 @@ +"""Synthetic chunk profiling must stay within request bounds and release on failure.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.managers.scheduler_components import dynamic_chunk_sizer as module +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestDynamicChunkSizerProfiling(CustomTestCase): + def _profile(self, *, context, mapping, prefill, base, fail_forward=False): + prepared = [] + live = set() + runner = SimpleNamespace( + model_config=SimpleNamespace( + context_len=context, hidden_size=2, dtype=torch.float32 + ), + get_pp_proxy_topk_size=lambda: None, + get_pp_proxy_residual_num_blocks=lambda: None, + ) + + def make_req(**kwargs): + return SimpleNamespace( + rid=kwargs["rid"], + full_untruncated_fill_ids=kwargs["origin_input_ids"], + init_next_round_input=Mock(), + last_node=None, + kv=SimpleNamespace(holds_kv=False), + ) + + def make_batch(reqs, *_args): + req = reqs[0] + + def prepare(): + prepared.append(req.extend_end) + live.add(req.rid) + req.kv.holds_kv = True + + return SimpleNamespace( + prepare_for_extend=prepare, + input_ids=torch.zeros(1, dtype=torch.int64), + forward_mode=SimpleNamespace(is_extend=lambda: True), + ) + + def release(req, _cache, *, checkpoint): + self.assertFalse(checkpoint) + live.remove(req.rid) + req.kv.holds_kv = False + + runner.forward = Mock( + side_effect=RuntimeError("synthetic forward failed") + if fail_forward + else None + ) + with ( + patch.object( + module, "get_parallel", return_value=SimpleNamespace(pp_rank=0) + ), + patch.object(module, "Req", side_effect=make_req), + patch.object(module.ScheduleBatch, "init_new", side_effect=make_batch), + patch.object( + module.ForwardBatch, "init_new", return_value=SimpleNamespace() + ), + patch.object(module, "release_kv_cache", side_effect=release), + patch.object(module, "is_dp_attention_enabled", return_value=False), + patch.object(module, "set_is_extend_in_batch"), + patch.object(module, "get_device_module", return_value=Mock()), + patch.object(module, "tqdm", side_effect=lambda items, **_kwargs: items), + ): + sizer = module.DynamicChunkSizer( + model_runner=runner, + model_config=runner.model_config, + tree_cache=Mock(), + req_to_token_pool=SimpleNamespace(max_context_len=mapping), + token_to_kv_pool_allocator=Mock(), + spec_algorithm=Mock(), + chunked_prefill_size=base, + max_prefill_tokens=prefill, + page_size=1, + device="cpu", + ) + if fail_forward: + with self.assertRaisesRegex(RuntimeError, "synthetic forward failed"): + sizer._profile_prefill_latency() + self.assertEqual(len(prepared), 1) + else: + lengths, latencies = sizer._profile_prefill_latency() + self.assertEqual(lengths, prepared) + self.assertEqual(len(latencies), len(lengths)) + self.assertGreater(len(set(lengths)), 8) + self.assertGreater(min(lengths), 0) + self.assertLessEqual(max(lengths), min(context, mapping)) + self.assertEqual( + live, set(), "profiling left a prepared request holding KV" + ) + return prepared + + def test_samples_fit_request_capacity_and_preserve_probe_range(self): + for context, mapping, prefill, base, expected_max in ( + (8192, 8196, 16384, 16384, 8192), + (8192, 4096, 16384, 16384, 4096), + (8192, 8196, 2048, 16384, 8192), + (8192, 8196, 16384, 512, 640), + (32768, 32772, 16384, 16384, 20480), + ): + with self.subTest( + context=context, mapping=mapping, prefill=prefill, base=base + ): + lengths = self._profile( + context=context, mapping=mapping, prefill=prefill, base=base + ) + self.assertEqual(max(lengths), expected_max) + + def test_forward_failure_releases_prepared_request(self): + self._profile( + context=8192, mapping=8196, prefill=16384, base=512, fail_forward=True + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_kimi_k3_pipeline_layers.py b/test/registered/unit/models/test_kimi_k3_pipeline_layers.py new file mode 100644 index 0000000000..ec8d04db18 --- /dev/null +++ b/test/registered/unit/models/test_kimi_k3_pipeline_layers.py @@ -0,0 +1,125 @@ +"""K3 wrappers must expose the local layer range to runner buffer sizing.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.model_executor.model_runner_components import misc_utils +from sglang.srt.model_executor.model_runner_components.layer_setup import ( + resolve_layer_indices, +) +from sglang.srt.model_executor.runner_utils.buffers import allocate_pp_proxy_tensors +from sglang.srt.models import kimi_k3 +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestKimiK3PipelineLayers(CustomTestCase): + def _build_text_model(self, start, end): + with ( + patch.object( + kimi_k3, + "KimiK3LinearModel", + return_value=SimpleNamespace(start_layer=start, end_layer=end), + ), + patch.object( + kimi_k3, + "get_parallel", + return_value=SimpleNamespace( + pp_group=SimpleNamespace(is_last_rank=False) + ), + ), + patch.object(kimi_k3, "LogitsProcessor"), + ): + return kimi_k3.KimiK3LinearForCausalLM( + SimpleNamespace(vocab_size=16, hidden_size=8) + ) + + def _assert_runner_bank(self, model, arch, start, end): + text_config = SimpleNamespace(attn_res_block_size=4) + model_config = SimpleNamespace( + hf_config=SimpleNamespace(architectures=[arch]), + hf_text_config=text_config, + num_hidden_layers=6, + num_attention_layers=6, + num_nextn_predict_layers=None, + swa_attention_layer_ids=None, + ) + info = resolve_layer_indices( + model=model, model_config=model_config, is_draft_worker=False + ) + self.assertEqual((info.start_layer, info.end_layer), (start, end)) + self.assertEqual(info.num_effective_layers, end - start) + with patch.object( + misc_utils, + "get_parallel", + return_value=SimpleNamespace(pp_size=2, pp_rank=1), + ): + blocks = misc_utils.resolve_pp_proxy_residual_num_blocks( + model_config=model_config, start_layer=info.start_layer + ) + self.assertEqual(blocks, (start + 3) // 4) + buffers = allocate_pp_proxy_tensors( + max_num_tokens=3, + max_hidden_tokens=3, + hidden_size=8, + dtype=torch.float32, + pp_proxy_residual_num_blocks=blocks, + ) + self.assertEqual(buffers["attn_res_bank"].shape, (3, blocks, 8)) + + def test_text_model_sizes_inherited_bank_at_both_cuts(self): + for start in (3, 4): + with self.subTest(start=start): + self._assert_runner_bank( + self._build_text_model(start, 6), + "KimiK3LinearForCausalLM", + start, + 6, + ) + + def test_multimodal_model_exposes_language_layer_range(self): + for start in (3, 4): + with ( + self.subTest(start=start), + patch.object(kimi_k3, "KimiK3VisionTower"), + patch.object(kimi_k3, "KimiK3MultiModalProjector"), + patch.object( + kimi_k3, + "KimiK3LinearForCausalLM", + return_value=self._build_text_model(start, 6), + ), + ): + model = kimi_k3.KimiK3ForConditionalGeneration( + SimpleNamespace( + encoder_only=False, + vision_config=SimpleNamespace(), + text_config=SimpleNamespace(), + ) + ) + self._assert_runner_bank( + model, "KimiK3ForConditionalGeneration", start, 6 + ) + + def test_encoder_only_keeps_existing_layer_range(self): + with ( + patch.object(kimi_k3, "KimiK3VisionTower"), + patch.object(kimi_k3, "KimiK3MultiModalProjector"), + ): + model = kimi_k3.KimiK3ForConditionalGeneration( + SimpleNamespace( + encoder_only=True, + vision_config=SimpleNamespace(), + text_config=SimpleNamespace(num_hidden_layers=6), + ) + ) + self.assertIsNone(model.language_model) + self.assertEqual((model.start_layer, model.end_layer), (0, 6)) + + +if __name__ == "__main__": + unittest.main()