[Fix] kimi-k3: Preserve pipeline graph bank inputs and bound chunk profiling (#43604)

This commit is contained in:
Cheng Wan
2026-10-15 02:03:27 -07:00
committed by GitHub
parent 71a17dbc24
commit d59c3448b0
4 changed files with 288 additions and 28 deletions
@@ -140,11 +140,16 @@ class DynamicChunkSizer:
model_runner = self.model_runner
model_config = model_runner.model_config
input_ids_list: List[array[int]] = []
# Each sample is a complete request. Allocating past its mapping row
# would leave KV slots that release_kv_cache cannot address.
max_profile_tokens = min(
int(self.chunked_prefill_size * 1.25),
model_config.context_len,
self.req_to_token_pool.max_context_len,
)
step = max(1, max_profile_tokens // 128)
for i in range(128):
chunk_size = int(
self.chunked_prefill_size * 1.25
- i * (self.chunked_prefill_size * 1.25 // 128)
)
chunk_size = max_profile_tokens - i * step
if chunk_size <= 0:
break
input_ids = array(
@@ -221,34 +226,35 @@ class DynamicChunkSizer:
start = time.perf_counter()
batch.prepare_for_extend()
# Resolve deferred H2D: prepare_for_extend now leaves input_ids=None
if batch.input_ids is None and batch.prefill_input_ids_cpu is not None:
batch.input_ids = batch.prefill_input_ids_cpu.to(
self.device, non_blocking=True
try:
# Resolve deferred H2D: prepare_for_extend now leaves input_ids=None
if batch.input_ids is None and batch.prefill_input_ids_cpu is not None:
batch.input_ids = batch.prefill_input_ids_cpu.to(
self.device, non_blocking=True
)
batch.prefill_input_ids_cpu = None
forward_batch = ForwardBatch.init_new(
batch,
model_runner,
return_hidden_states_before_norm=False,
)
batch.prefill_input_ids_cpu = None
set_is_extend_in_batch(batch.forward_mode.is_extend())
forward_batch = ForwardBatch.init_new(
batch,
model_runner,
return_hidden_states_before_norm=False,
)
set_is_extend_in_batch(batch.forward_mode.is_extend())
_ = model_runner.forward(
forward_batch=forward_batch, pp_proxy_tensors=pp_proxy
)
_ = model_runner.forward(
forward_batch=forward_batch, pp_proxy_tensors=pp_proxy
)
# Synchronize after forward to ensure GPU operations complete
device_module.synchronize()
# Synchronize after forward to ensure GPU operations complete
device_module.synchronize()
latency_seconds = time.perf_counter() - start
latency_ms = latency_seconds * 1e3 # Convert to milliseconds
seq_lens.append(len(input_ids))
latencies.append(latency_ms)
# Release KV and Mamba cache
if req.kv.holds_kv:
latency_seconds = time.perf_counter() - start
latency_ms = latency_seconds * 1e3 # Convert to milliseconds
seq_lens.append(len(input_ids))
latencies.append(latency_ms)
finally:
# Profiling failures disable the predictor; they must still
# return the synthetic request's KV and recurrent state.
release_kv_cache(req, self.tree_cache, checkpoint=False)
logger.info(
+2
View File
@@ -3005,6 +3005,8 @@ class KimiK3LinearForCausalLM(nn.Module):
self.model = KimiK3LinearModel(
config, quant_config, prefix=maybe_prefix(prefix, "model")
)
self.start_layer = self.model.start_layer
self.end_layer = self.model.end_layer
self.pp_group = get_parallel().pp_group
if self.pp_group.is_last_rank:
self.lm_head = ParallelLMHead(