[Fix] kimi-k3: Preserve pipeline graph bank inputs and bound chunk profiling (#43604)
This commit is contained in:
@@ -140,11 +140,16 @@ class DynamicChunkSizer:
|
||||
model_runner = self.model_runner
|
||||
model_config = model_runner.model_config
|
||||
input_ids_list: List[array[int]] = []
|
||||
# Each sample is a complete request. Allocating past its mapping row
|
||||
# would leave KV slots that release_kv_cache cannot address.
|
||||
max_profile_tokens = min(
|
||||
int(self.chunked_prefill_size * 1.25),
|
||||
model_config.context_len,
|
||||
self.req_to_token_pool.max_context_len,
|
||||
)
|
||||
step = max(1, max_profile_tokens // 128)
|
||||
for i in range(128):
|
||||
chunk_size = int(
|
||||
self.chunked_prefill_size * 1.25
|
||||
- i * (self.chunked_prefill_size * 1.25 // 128)
|
||||
)
|
||||
chunk_size = max_profile_tokens - i * step
|
||||
if chunk_size <= 0:
|
||||
break
|
||||
input_ids = array(
|
||||
@@ -221,34 +226,35 @@ class DynamicChunkSizer:
|
||||
start = time.perf_counter()
|
||||
batch.prepare_for_extend()
|
||||
|
||||
# Resolve deferred H2D: prepare_for_extend now leaves input_ids=None
|
||||
if batch.input_ids is None and batch.prefill_input_ids_cpu is not None:
|
||||
batch.input_ids = batch.prefill_input_ids_cpu.to(
|
||||
self.device, non_blocking=True
|
||||
try:
|
||||
# Resolve deferred H2D: prepare_for_extend now leaves input_ids=None
|
||||
if batch.input_ids is None and batch.prefill_input_ids_cpu is not None:
|
||||
batch.input_ids = batch.prefill_input_ids_cpu.to(
|
||||
self.device, non_blocking=True
|
||||
)
|
||||
batch.prefill_input_ids_cpu = None
|
||||
|
||||
forward_batch = ForwardBatch.init_new(
|
||||
batch,
|
||||
model_runner,
|
||||
return_hidden_states_before_norm=False,
|
||||
)
|
||||
batch.prefill_input_ids_cpu = None
|
||||
set_is_extend_in_batch(batch.forward_mode.is_extend())
|
||||
|
||||
forward_batch = ForwardBatch.init_new(
|
||||
batch,
|
||||
model_runner,
|
||||
return_hidden_states_before_norm=False,
|
||||
)
|
||||
set_is_extend_in_batch(batch.forward_mode.is_extend())
|
||||
_ = model_runner.forward(
|
||||
forward_batch=forward_batch, pp_proxy_tensors=pp_proxy
|
||||
)
|
||||
|
||||
_ = model_runner.forward(
|
||||
forward_batch=forward_batch, pp_proxy_tensors=pp_proxy
|
||||
)
|
||||
# Synchronize after forward to ensure GPU operations complete
|
||||
device_module.synchronize()
|
||||
|
||||
# Synchronize after forward to ensure GPU operations complete
|
||||
device_module.synchronize()
|
||||
|
||||
latency_seconds = time.perf_counter() - start
|
||||
latency_ms = latency_seconds * 1e3 # Convert to milliseconds
|
||||
seq_lens.append(len(input_ids))
|
||||
latencies.append(latency_ms)
|
||||
|
||||
# Release KV and Mamba cache
|
||||
if req.kv.holds_kv:
|
||||
latency_seconds = time.perf_counter() - start
|
||||
latency_ms = latency_seconds * 1e3 # Convert to milliseconds
|
||||
seq_lens.append(len(input_ids))
|
||||
latencies.append(latency_ms)
|
||||
finally:
|
||||
# Profiling failures disable the predictor; they must still
|
||||
# return the synthetic request's KV and recurrent state.
|
||||
release_kv_cache(req, self.tree_cache, checkpoint=False)
|
||||
|
||||
logger.info(
|
||||
|
||||
@@ -3005,6 +3005,8 @@ class KimiK3LinearForCausalLM(nn.Module):
|
||||
self.model = KimiK3LinearModel(
|
||||
config, quant_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.start_layer = self.model.start_layer
|
||||
self.end_layer = self.model.end_layer
|
||||
self.pp_group = get_parallel().pp_group
|
||||
if self.pp_group.is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Synthetic chunk profiling must stay within request bounds and release on failure."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.scheduler_components import dynamic_chunk_sizer as module
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestDynamicChunkSizerProfiling(CustomTestCase):
|
||||
def _profile(self, *, context, mapping, prefill, base, fail_forward=False):
|
||||
prepared = []
|
||||
live = set()
|
||||
runner = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
context_len=context, hidden_size=2, dtype=torch.float32
|
||||
),
|
||||
get_pp_proxy_topk_size=lambda: None,
|
||||
get_pp_proxy_residual_num_blocks=lambda: None,
|
||||
)
|
||||
|
||||
def make_req(**kwargs):
|
||||
return SimpleNamespace(
|
||||
rid=kwargs["rid"],
|
||||
full_untruncated_fill_ids=kwargs["origin_input_ids"],
|
||||
init_next_round_input=Mock(),
|
||||
last_node=None,
|
||||
kv=SimpleNamespace(holds_kv=False),
|
||||
)
|
||||
|
||||
def make_batch(reqs, *_args):
|
||||
req = reqs[0]
|
||||
|
||||
def prepare():
|
||||
prepared.append(req.extend_end)
|
||||
live.add(req.rid)
|
||||
req.kv.holds_kv = True
|
||||
|
||||
return SimpleNamespace(
|
||||
prepare_for_extend=prepare,
|
||||
input_ids=torch.zeros(1, dtype=torch.int64),
|
||||
forward_mode=SimpleNamespace(is_extend=lambda: True),
|
||||
)
|
||||
|
||||
def release(req, _cache, *, checkpoint):
|
||||
self.assertFalse(checkpoint)
|
||||
live.remove(req.rid)
|
||||
req.kv.holds_kv = False
|
||||
|
||||
runner.forward = Mock(
|
||||
side_effect=RuntimeError("synthetic forward failed")
|
||||
if fail_forward
|
||||
else None
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
module, "get_parallel", return_value=SimpleNamespace(pp_rank=0)
|
||||
),
|
||||
patch.object(module, "Req", side_effect=make_req),
|
||||
patch.object(module.ScheduleBatch, "init_new", side_effect=make_batch),
|
||||
patch.object(
|
||||
module.ForwardBatch, "init_new", return_value=SimpleNamespace()
|
||||
),
|
||||
patch.object(module, "release_kv_cache", side_effect=release),
|
||||
patch.object(module, "is_dp_attention_enabled", return_value=False),
|
||||
patch.object(module, "set_is_extend_in_batch"),
|
||||
patch.object(module, "get_device_module", return_value=Mock()),
|
||||
patch.object(module, "tqdm", side_effect=lambda items, **_kwargs: items),
|
||||
):
|
||||
sizer = module.DynamicChunkSizer(
|
||||
model_runner=runner,
|
||||
model_config=runner.model_config,
|
||||
tree_cache=Mock(),
|
||||
req_to_token_pool=SimpleNamespace(max_context_len=mapping),
|
||||
token_to_kv_pool_allocator=Mock(),
|
||||
spec_algorithm=Mock(),
|
||||
chunked_prefill_size=base,
|
||||
max_prefill_tokens=prefill,
|
||||
page_size=1,
|
||||
device="cpu",
|
||||
)
|
||||
if fail_forward:
|
||||
with self.assertRaisesRegex(RuntimeError, "synthetic forward failed"):
|
||||
sizer._profile_prefill_latency()
|
||||
self.assertEqual(len(prepared), 1)
|
||||
else:
|
||||
lengths, latencies = sizer._profile_prefill_latency()
|
||||
self.assertEqual(lengths, prepared)
|
||||
self.assertEqual(len(latencies), len(lengths))
|
||||
self.assertGreater(len(set(lengths)), 8)
|
||||
self.assertGreater(min(lengths), 0)
|
||||
self.assertLessEqual(max(lengths), min(context, mapping))
|
||||
self.assertEqual(
|
||||
live, set(), "profiling left a prepared request holding KV"
|
||||
)
|
||||
return prepared
|
||||
|
||||
def test_samples_fit_request_capacity_and_preserve_probe_range(self):
|
||||
for context, mapping, prefill, base, expected_max in (
|
||||
(8192, 8196, 16384, 16384, 8192),
|
||||
(8192, 4096, 16384, 16384, 4096),
|
||||
(8192, 8196, 2048, 16384, 8192),
|
||||
(8192, 8196, 16384, 512, 640),
|
||||
(32768, 32772, 16384, 16384, 20480),
|
||||
):
|
||||
with self.subTest(
|
||||
context=context, mapping=mapping, prefill=prefill, base=base
|
||||
):
|
||||
lengths = self._profile(
|
||||
context=context, mapping=mapping, prefill=prefill, base=base
|
||||
)
|
||||
self.assertEqual(max(lengths), expected_max)
|
||||
|
||||
def test_forward_failure_releases_prepared_request(self):
|
||||
self._profile(
|
||||
context=8192, mapping=8196, prefill=16384, base=512, fail_forward=True
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,125 @@
|
||||
"""K3 wrappers must expose the local layer range to runner buffer sizing."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.model_executor.model_runner_components import misc_utils
|
||||
from sglang.srt.model_executor.model_runner_components.layer_setup import (
|
||||
resolve_layer_indices,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils.buffers import allocate_pp_proxy_tensors
|
||||
from sglang.srt.models import kimi_k3
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestKimiK3PipelineLayers(CustomTestCase):
|
||||
def _build_text_model(self, start, end):
|
||||
with (
|
||||
patch.object(
|
||||
kimi_k3,
|
||||
"KimiK3LinearModel",
|
||||
return_value=SimpleNamespace(start_layer=start, end_layer=end),
|
||||
),
|
||||
patch.object(
|
||||
kimi_k3,
|
||||
"get_parallel",
|
||||
return_value=SimpleNamespace(
|
||||
pp_group=SimpleNamespace(is_last_rank=False)
|
||||
),
|
||||
),
|
||||
patch.object(kimi_k3, "LogitsProcessor"),
|
||||
):
|
||||
return kimi_k3.KimiK3LinearForCausalLM(
|
||||
SimpleNamespace(vocab_size=16, hidden_size=8)
|
||||
)
|
||||
|
||||
def _assert_runner_bank(self, model, arch, start, end):
|
||||
text_config = SimpleNamespace(attn_res_block_size=4)
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(architectures=[arch]),
|
||||
hf_text_config=text_config,
|
||||
num_hidden_layers=6,
|
||||
num_attention_layers=6,
|
||||
num_nextn_predict_layers=None,
|
||||
swa_attention_layer_ids=None,
|
||||
)
|
||||
info = resolve_layer_indices(
|
||||
model=model, model_config=model_config, is_draft_worker=False
|
||||
)
|
||||
self.assertEqual((info.start_layer, info.end_layer), (start, end))
|
||||
self.assertEqual(info.num_effective_layers, end - start)
|
||||
with patch.object(
|
||||
misc_utils,
|
||||
"get_parallel",
|
||||
return_value=SimpleNamespace(pp_size=2, pp_rank=1),
|
||||
):
|
||||
blocks = misc_utils.resolve_pp_proxy_residual_num_blocks(
|
||||
model_config=model_config, start_layer=info.start_layer
|
||||
)
|
||||
self.assertEqual(blocks, (start + 3) // 4)
|
||||
buffers = allocate_pp_proxy_tensors(
|
||||
max_num_tokens=3,
|
||||
max_hidden_tokens=3,
|
||||
hidden_size=8,
|
||||
dtype=torch.float32,
|
||||
pp_proxy_residual_num_blocks=blocks,
|
||||
)
|
||||
self.assertEqual(buffers["attn_res_bank"].shape, (3, blocks, 8))
|
||||
|
||||
def test_text_model_sizes_inherited_bank_at_both_cuts(self):
|
||||
for start in (3, 4):
|
||||
with self.subTest(start=start):
|
||||
self._assert_runner_bank(
|
||||
self._build_text_model(start, 6),
|
||||
"KimiK3LinearForCausalLM",
|
||||
start,
|
||||
6,
|
||||
)
|
||||
|
||||
def test_multimodal_model_exposes_language_layer_range(self):
|
||||
for start in (3, 4):
|
||||
with (
|
||||
self.subTest(start=start),
|
||||
patch.object(kimi_k3, "KimiK3VisionTower"),
|
||||
patch.object(kimi_k3, "KimiK3MultiModalProjector"),
|
||||
patch.object(
|
||||
kimi_k3,
|
||||
"KimiK3LinearForCausalLM",
|
||||
return_value=self._build_text_model(start, 6),
|
||||
),
|
||||
):
|
||||
model = kimi_k3.KimiK3ForConditionalGeneration(
|
||||
SimpleNamespace(
|
||||
encoder_only=False,
|
||||
vision_config=SimpleNamespace(),
|
||||
text_config=SimpleNamespace(),
|
||||
)
|
||||
)
|
||||
self._assert_runner_bank(
|
||||
model, "KimiK3ForConditionalGeneration", start, 6
|
||||
)
|
||||
|
||||
def test_encoder_only_keeps_existing_layer_range(self):
|
||||
with (
|
||||
patch.object(kimi_k3, "KimiK3VisionTower"),
|
||||
patch.object(kimi_k3, "KimiK3MultiModalProjector"),
|
||||
):
|
||||
model = kimi_k3.KimiK3ForConditionalGeneration(
|
||||
SimpleNamespace(
|
||||
encoder_only=True,
|
||||
vision_config=SimpleNamespace(),
|
||||
text_config=SimpleNamespace(num_hidden_layers=6),
|
||||
)
|
||||
)
|
||||
self.assertIsNone(model.language_model)
|
||||
self.assertEqual((model.start_layer, model.end_layer), (0, 6))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user