[Fix] kimi-k3: Preserve pipeline graph bank inputs and bound chunk profiling (#43604)

This commit is contained in:
Cheng Wan
2026-10-15 02:03:27 -07:00
committed by GitHub
parent 71a17dbc24
commit d59c3448b0
4 changed files with 288 additions and 28 deletions
@@ -140,11 +140,16 @@ class DynamicChunkSizer:
model_runner = self.model_runner
model_config = model_runner.model_config
input_ids_list: List[array[int]] = []
# Each sample is a complete request. Allocating past its mapping row
# would leave KV slots that release_kv_cache cannot address.
max_profile_tokens = min(
int(self.chunked_prefill_size * 1.25),
model_config.context_len,
self.req_to_token_pool.max_context_len,
)
step = max(1, max_profile_tokens // 128)
for i in range(128):
chunk_size = int(
self.chunked_prefill_size * 1.25
- i * (self.chunked_prefill_size * 1.25 // 128)
)
chunk_size = max_profile_tokens - i * step
if chunk_size <= 0:
break
input_ids = array(
@@ -221,34 +226,35 @@ class DynamicChunkSizer:
start = time.perf_counter()
batch.prepare_for_extend()
# Resolve deferred H2D: prepare_for_extend now leaves input_ids=None
if batch.input_ids is None and batch.prefill_input_ids_cpu is not None:
batch.input_ids = batch.prefill_input_ids_cpu.to(
self.device, non_blocking=True
try:
# Resolve deferred H2D: prepare_for_extend now leaves input_ids=None
if batch.input_ids is None and batch.prefill_input_ids_cpu is not None:
batch.input_ids = batch.prefill_input_ids_cpu.to(
self.device, non_blocking=True
)
batch.prefill_input_ids_cpu = None
forward_batch = ForwardBatch.init_new(
batch,
model_runner,
return_hidden_states_before_norm=False,
)
batch.prefill_input_ids_cpu = None
set_is_extend_in_batch(batch.forward_mode.is_extend())
forward_batch = ForwardBatch.init_new(
batch,
model_runner,
return_hidden_states_before_norm=False,
)
set_is_extend_in_batch(batch.forward_mode.is_extend())
_ = model_runner.forward(
forward_batch=forward_batch, pp_proxy_tensors=pp_proxy
)
_ = model_runner.forward(
forward_batch=forward_batch, pp_proxy_tensors=pp_proxy
)
# Synchronize after forward to ensure GPU operations complete
device_module.synchronize()
# Synchronize after forward to ensure GPU operations complete
device_module.synchronize()
latency_seconds = time.perf_counter() - start
latency_ms = latency_seconds * 1e3 # Convert to milliseconds
seq_lens.append(len(input_ids))
latencies.append(latency_ms)
# Release KV and Mamba cache
if req.kv.holds_kv:
latency_seconds = time.perf_counter() - start
latency_ms = latency_seconds * 1e3 # Convert to milliseconds
seq_lens.append(len(input_ids))
latencies.append(latency_ms)
finally:
# Profiling failures disable the predictor; they must still
# return the synthetic request's KV and recurrent state.
release_kv_cache(req, self.tree_cache, checkpoint=False)
logger.info(
+2
View File
@@ -3005,6 +3005,8 @@ class KimiK3LinearForCausalLM(nn.Module):
self.model = KimiK3LinearModel(
config, quant_config, prefix=maybe_prefix(prefix, "model")
)
self.start_layer = self.model.start_layer
self.end_layer = self.model.end_layer
self.pp_group = get_parallel().pp_group
if self.pp_group.is_last_rank:
self.lm_head = ParallelLMHead(
@@ -0,0 +1,127 @@
"""Synthetic chunk profiling must stay within request bounds and release on failure."""
import unittest
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
from sglang.srt.managers.scheduler_components import dynamic_chunk_sizer as module
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class TestDynamicChunkSizerProfiling(CustomTestCase):
def _profile(self, *, context, mapping, prefill, base, fail_forward=False):
prepared = []
live = set()
runner = SimpleNamespace(
model_config=SimpleNamespace(
context_len=context, hidden_size=2, dtype=torch.float32
),
get_pp_proxy_topk_size=lambda: None,
get_pp_proxy_residual_num_blocks=lambda: None,
)
def make_req(**kwargs):
return SimpleNamespace(
rid=kwargs["rid"],
full_untruncated_fill_ids=kwargs["origin_input_ids"],
init_next_round_input=Mock(),
last_node=None,
kv=SimpleNamespace(holds_kv=False),
)
def make_batch(reqs, *_args):
req = reqs[0]
def prepare():
prepared.append(req.extend_end)
live.add(req.rid)
req.kv.holds_kv = True
return SimpleNamespace(
prepare_for_extend=prepare,
input_ids=torch.zeros(1, dtype=torch.int64),
forward_mode=SimpleNamespace(is_extend=lambda: True),
)
def release(req, _cache, *, checkpoint):
self.assertFalse(checkpoint)
live.remove(req.rid)
req.kv.holds_kv = False
runner.forward = Mock(
side_effect=RuntimeError("synthetic forward failed")
if fail_forward
else None
)
with (
patch.object(
module, "get_parallel", return_value=SimpleNamespace(pp_rank=0)
),
patch.object(module, "Req", side_effect=make_req),
patch.object(module.ScheduleBatch, "init_new", side_effect=make_batch),
patch.object(
module.ForwardBatch, "init_new", return_value=SimpleNamespace()
),
patch.object(module, "release_kv_cache", side_effect=release),
patch.object(module, "is_dp_attention_enabled", return_value=False),
patch.object(module, "set_is_extend_in_batch"),
patch.object(module, "get_device_module", return_value=Mock()),
patch.object(module, "tqdm", side_effect=lambda items, **_kwargs: items),
):
sizer = module.DynamicChunkSizer(
model_runner=runner,
model_config=runner.model_config,
tree_cache=Mock(),
req_to_token_pool=SimpleNamespace(max_context_len=mapping),
token_to_kv_pool_allocator=Mock(),
spec_algorithm=Mock(),
chunked_prefill_size=base,
max_prefill_tokens=prefill,
page_size=1,
device="cpu",
)
if fail_forward:
with self.assertRaisesRegex(RuntimeError, "synthetic forward failed"):
sizer._profile_prefill_latency()
self.assertEqual(len(prepared), 1)
else:
lengths, latencies = sizer._profile_prefill_latency()
self.assertEqual(lengths, prepared)
self.assertEqual(len(latencies), len(lengths))
self.assertGreater(len(set(lengths)), 8)
self.assertGreater(min(lengths), 0)
self.assertLessEqual(max(lengths), min(context, mapping))
self.assertEqual(
live, set(), "profiling left a prepared request holding KV"
)
return prepared
def test_samples_fit_request_capacity_and_preserve_probe_range(self):
for context, mapping, prefill, base, expected_max in (
(8192, 8196, 16384, 16384, 8192),
(8192, 4096, 16384, 16384, 4096),
(8192, 8196, 2048, 16384, 8192),
(8192, 8196, 16384, 512, 640),
(32768, 32772, 16384, 16384, 20480),
):
with self.subTest(
context=context, mapping=mapping, prefill=prefill, base=base
):
lengths = self._profile(
context=context, mapping=mapping, prefill=prefill, base=base
)
self.assertEqual(max(lengths), expected_max)
def test_forward_failure_releases_prepared_request(self):
self._profile(
context=8192, mapping=8196, prefill=16384, base=512, fail_forward=True
)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,125 @@
"""K3 wrappers must expose the local layer range to runner buffer sizing."""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.model_executor.model_runner_components import misc_utils
from sglang.srt.model_executor.model_runner_components.layer_setup import (
resolve_layer_indices,
)
from sglang.srt.model_executor.runner_utils.buffers import allocate_pp_proxy_tensors
from sglang.srt.models import kimi_k3
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class TestKimiK3PipelineLayers(CustomTestCase):
def _build_text_model(self, start, end):
with (
patch.object(
kimi_k3,
"KimiK3LinearModel",
return_value=SimpleNamespace(start_layer=start, end_layer=end),
),
patch.object(
kimi_k3,
"get_parallel",
return_value=SimpleNamespace(
pp_group=SimpleNamespace(is_last_rank=False)
),
),
patch.object(kimi_k3, "LogitsProcessor"),
):
return kimi_k3.KimiK3LinearForCausalLM(
SimpleNamespace(vocab_size=16, hidden_size=8)
)
def _assert_runner_bank(self, model, arch, start, end):
text_config = SimpleNamespace(attn_res_block_size=4)
model_config = SimpleNamespace(
hf_config=SimpleNamespace(architectures=[arch]),
hf_text_config=text_config,
num_hidden_layers=6,
num_attention_layers=6,
num_nextn_predict_layers=None,
swa_attention_layer_ids=None,
)
info = resolve_layer_indices(
model=model, model_config=model_config, is_draft_worker=False
)
self.assertEqual((info.start_layer, info.end_layer), (start, end))
self.assertEqual(info.num_effective_layers, end - start)
with patch.object(
misc_utils,
"get_parallel",
return_value=SimpleNamespace(pp_size=2, pp_rank=1),
):
blocks = misc_utils.resolve_pp_proxy_residual_num_blocks(
model_config=model_config, start_layer=info.start_layer
)
self.assertEqual(blocks, (start + 3) // 4)
buffers = allocate_pp_proxy_tensors(
max_num_tokens=3,
max_hidden_tokens=3,
hidden_size=8,
dtype=torch.float32,
pp_proxy_residual_num_blocks=blocks,
)
self.assertEqual(buffers["attn_res_bank"].shape, (3, blocks, 8))
def test_text_model_sizes_inherited_bank_at_both_cuts(self):
for start in (3, 4):
with self.subTest(start=start):
self._assert_runner_bank(
self._build_text_model(start, 6),
"KimiK3LinearForCausalLM",
start,
6,
)
def test_multimodal_model_exposes_language_layer_range(self):
for start in (3, 4):
with (
self.subTest(start=start),
patch.object(kimi_k3, "KimiK3VisionTower"),
patch.object(kimi_k3, "KimiK3MultiModalProjector"),
patch.object(
kimi_k3,
"KimiK3LinearForCausalLM",
return_value=self._build_text_model(start, 6),
),
):
model = kimi_k3.KimiK3ForConditionalGeneration(
SimpleNamespace(
encoder_only=False,
vision_config=SimpleNamespace(),
text_config=SimpleNamespace(),
)
)
self._assert_runner_bank(
model, "KimiK3ForConditionalGeneration", start, 6
)
def test_encoder_only_keeps_existing_layer_range(self):
with (
patch.object(kimi_k3, "KimiK3VisionTower"),
patch.object(kimi_k3, "KimiK3MultiModalProjector"),
):
model = kimi_k3.KimiK3ForConditionalGeneration(
SimpleNamespace(
encoder_only=True,
vision_config=SimpleNamespace(),
text_config=SimpleNamespace(num_hidden_layers=6),
)
)
self.assertIsNone(model.language_model)
self.assertEqual((model.start_layer, model.end_layer), (0, 6))
if __name__ == "__main__":
unittest.main()