[Fix] Run KV canary hooks for context-parallel prefill (#40642)

This commit is contained in:
Cheng Wan
2026-09-21 16:46:22 -07:00
committed by GitHub
parent 00986c81be
commit 22587fb15c
2 changed files with 55 additions and 0 deletions
+8
View File
@@ -20,6 +20,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import (
get_disagg,
get_parallel,
get_spec,
)
@@ -123,6 +124,13 @@ def _patch_model_forward(*, model_runner: ModelRunner, manager: CanaryManager) -
return output
wrap_method(model_runner.model, "forward", wrapper=_with_canary_bracketing)
if get_parallel().enable_prefill_cp:
# CP prefill calls the transformer body directly, bypassing the outer
# model.forward. Decode still enters through the outer model; the shared
# bracket scope prevents the body from running a second pair of hooks.
wrap_method(
model_runner.model.model, "forward", wrapper=_with_canary_bracketing
)
def _extract_forward_batch(args, kwargs) -> Optional[ForwardBatch]: