G9 P0: loads 排队预估字段 + /query_cache_hit 命中查询端点

A. 负载预估数据源(决策②总输入长度口径):
- LoadSnapshot 新增 queued_total_input_tokens / drain_rate_tokens_per_s
- load_inquirer 复用 pending_token_queues 求和;scheduler 60s 滑动窗
  记录每请求最终 prefill chunk 完成事件(batch_result_processor 挂钩
  set_prefill_finished_time 处,恰好每请求一次)
- /v1/loads 与 PUB socket 同源自动暴露,gateway LoadMonitor 零改动消费

B. /query_cache_hit(决策④放 D):
- io_struct 新增 QueryCacheHitReqInput/Output IPC 类型
- scheduler.handle_query_cache_hit:tree_cache.match_prefix(L1/L2)
  + query_storage_hit_length(共享 L3,逻辑镜像 decode_hicache_mixin)
- tokenizer_control_mixin:fan-out 全部 DP rank 取最优命中
- serving_chat.query_cache_hit:复用 _convert_to_internal_request +
  _tokenize_one_request 保证 chat 模板/分词与真实路径一致;含图请求
  v1 拒绝(multimodal 400);http_server 注册 /query_cache_hit
This commit is contained in:
2026-10-09 18:48:27 +08:00
parent 7d67d57804
commit 4eabf5e228
8 changed files with 240 additions and 1 deletions
@@ -1776,6 +1776,18 @@ async def openai_v1_chat_completions(
)
@app.post("/query_cache_hit", dependencies=[Depends(validate_json_request)])
async def query_cache_hit(request: ChatCompletionRequest, raw_request: Request):
"""G9: probe prefix-cache hit (L1/L2 + L3 storage) for a chat request.
Deployed on decode engines; the PD router calls it before dispatch to
decide conditional bypass. Accepts a normal chat body or `input_ids`.
"""
return await raw_request.app.state.openai_serving_chat.query_cache_hit(
request, raw_request
)
@app.post(
"/v1/embeddings",
response_class=ORJSONResponse,
@@ -1140,6 +1140,59 @@ class OpenAIServingChat(OpenAIServingBase):
return "text", processed_messages.prompt_ids
return "input_ids", processed_messages.prompt_ids
async def query_cache_hit(
self, request: ChatCompletionRequest, raw_request: Request = None
) -> ORJSONResponse:
"""G9 probe: report how much of this chat request's prompt is already
cached (per-rank-best L1/L2 radix + shared L3 storage) without running
any prefill. The router uses it for conditional PD-bypass decisions.
Multimodal requests are rejected (v1): their sentinel-token expansion
costs ~10-100ms per image for only ~2% of traffic; the router skips
them for bypass anyway.
"""
try:
internal_req, _ = self._convert_to_internal_request(request, raw_request)
except Exception as e:
return ORJSONResponse(
status_code=400,
content={"error": f"failed to convert chat request: {e}"},
)
if internal_req.contains_mm_input():
return ORJSONResponse(
status_code=400,
content={"error": "multimodal requests are not supported by /query_cache_hit"},
)
try:
tokenized = await self.tokenizer_manager._tokenize_one_request(internal_req)
from sglang.srt.managers.io_struct import QueryCacheHitReqInput
result = await self.tokenizer_manager.query_cache_hit(
QueryCacheHitReqInput(
input_ids=list(tokenized.input_ids),
extra_key=getattr(internal_req, "extra_key", None),
cache_salt=getattr(internal_req, "cache_salt", None),
)
)
except Exception as e:
return ORJSONResponse(
status_code=500, content={"error": f"cache-hit query failed: {e}"}
)
if result.error is not None:
return ORJSONResponse(status_code=500, content={"error": result.error})
hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens
return ORJSONResponse(
content={
"total_tokens": result.total_tokens,
"l1_hit_tokens": result.l1_hit_tokens,
"l2_hit_tokens": result.l2_hit_tokens,
"l3_hit_tokens": result.l3_hit_tokens,
"hit_tokens": hit,
"new_tokens": max(0, result.total_tokens - hit),
"dp_rank": result.dp_rank,
}
)
def _convert_to_internal_request(
self,
request: ChatCompletionRequest,
+21
View File
@@ -2141,6 +2141,27 @@ class SetInternalStateReqOutput(BaseReq, kw_only=True):
updated: bool
class QueryCacheHitReqInput(BaseReq, kw_only=True):
"""G9: read-only probe of prefix-cache hit for a token sequence.
Sent to every DP rank of a (decode) engine; each rank answers with its own
L1/L2 radix match plus the shared L3 storage hit continuing from it.
"""
input_ids: List[int]
extra_key: Optional[str] = None
cache_salt: Optional[str] = None
class QueryCacheHitReqOutput(BaseReq, kw_only=True):
total_tokens: int = 0
l1_hit_tokens: int = 0
l2_hit_tokens: int = 0
l3_hit_tokens: int = 0
dp_rank: int = 0
error: Optional[str] = None
class ProfileReqType(Enum):
START_PROFILE = 1
STOP_PROFILE = 2
@@ -197,6 +197,12 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
num_running_reqs: int = 0
num_waiting_reqs: int = 0
num_waiting_uncached_tokens: int = 0
# G9 queue-time estimation (总输入长度口径, self-consistent numerator/denominator):
# sum of total input lengths (req.seqlen) over all pending queues, and the
# self-learned drain rate in the same unit — completed prefills' total
# input tokens / wall second over a sliding window.
queued_total_input_tokens: int = 0
drain_rate_tokens_per_s: float = 0.0
num_used_tokens: int = 0
num_total_tokens: int = 0
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it
+104
View File
@@ -161,6 +161,8 @@ from sglang.srt.managers.io_struct import (
PauseGenerationReqInput,
PdRoleSwitchReqInput,
ProfileReq,
QueryCacheHitReqInput,
QueryCacheHitReqOutput,
ReleaseMemoryOccupationReqInput,
RemoveExternalCorpusReqInput,
RemoveExternalCorpusReqOutput,
@@ -1806,6 +1808,7 @@ class Scheduler(
(ShutdownReq, self.handle_shutdown),
(GetInternalStateReq, self.get_internal_state),
(SetInternalStateReq, self.set_internal_state),
(QueryCacheHitReqInput, self.handle_query_cache_hit),
(RpcReqInput, self.handle_rpc_request),
(ExpertDistributionReq, self.expert_distribution_handle),
(LoadLoRAAdapterReqInput, self.load_lora_adapter),
@@ -2431,6 +2434,11 @@ class Scheduler(
self._prev_step: Optional[Tuple[int, float, bool]] = None
self._prev_prefill_end_ts: Optional[float] = None
self._sched_idled = False
# G9: sliding-window drain events (monotonic_ts, total_input_tokens) for
# the queue-time estimator. Recorded once per request when its final
# prefill chunk completes (batch_result_processor), in the same
# total-input-length unit as LoadSnapshot.queued_total_input_tokens.
self.prefill_drain_window: deque = deque()
self.load_inquirer = SchedulerLoadInquirer(
disaggregation_mode=self.disaggregation_mode,
server_args=self.server_args,
@@ -2469,8 +2477,35 @@ class Scheduler(
),
get_total_prefill_busy_us=lambda: self.total_prefill_busy_us,
get_decode_moment_totals=lambda: self.decode_moment_totals,
get_prefill_drain_rate=self.get_prefill_drain_rate,
)
PREFILL_DRAIN_WINDOW_S = 60.0
def record_prefill_drain(self, total_input_tokens: int) -> None:
"""G9: one drain event per request whose final prefill chunk completed.
Tokens are the request's total input length (seqlen), matching the
queued_total_input_tokens numerator so the router's
wait = queued_tokens / drain_rate is unit-consistent (决策②).
"""
now = time.monotonic()
self.prefill_drain_window.append((now, total_input_tokens))
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
self.prefill_drain_window.popleft()
def get_prefill_drain_rate(self) -> float:
"""Completed prefill input tokens per wall second over the window."""
now = time.monotonic()
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
self.prefill_drain_window.popleft()
if not self.prefill_drain_window:
return 0.0
span = max(now - self.prefill_drain_window[0][0], 1.0)
return sum(tokens for _, tokens in self.prefill_drain_window) / span
def init_output_streamer(self) -> None:
self.output_streamer = self.get_output_streamer_class()(
send_to_detokenizer=self.ipc_channels.send_to_detokenizer,
@@ -2521,6 +2556,7 @@ class Scheduler(
output_streamer=self.output_streamer,
beam_coordinator=self.beam_coordinator,
abort_request=self.abort_request,
record_prefill_drain=self.record_prefill_drain,
)
def init_req_max_new_tokens(self, req):
@@ -5215,6 +5251,74 @@ class Scheduler(
success = False
return success
def handle_query_cache_hit(self, recv_req: QueryCacheHitReqInput):
"""G9: read-only prefix-hit probe (L1/L2 radix + L3 storage).
The router calls this (via /query_cache_hit on the decode engine) to
decide conditional PD-bypass. Runs inside the scheduler event loop so
radix/storage state is consistent, and reuses the exact admission-time
match + L3 query logic (mirrors DecodeHiCachePreallocMixin
._build_decode_prefix_match) without creating a Req or mutating
request state. Node access metadata (LRU) may be touched; a hit found
here can still be evicted before the real request arrives (inherent
TOCTOU, acceptable for an estimate).
"""
from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams
from sglang.srt.mem_cache.radix_cache import RadixKey
total = len(recv_req.input_ids)
try:
match_result = self.tree_cache.match_prefix(
MatchPrefixParams(
key=RadixKey(
token_ids=recv_req.input_ids,
extra_key=recv_req.extra_key,
cache_salt=recv_req.cache_salt,
)
)
)
l1_hit = len(match_result.device_indices)
l2_hit = getattr(match_result, "host_hit_length", 0) or 0
l3_hit = 0
last_host_node = getattr(match_result, "last_host_node", None)
if (
getattr(self, "enable_decode_hicache", False)
and last_host_node is not None
and hasattr(self.tree_cache, "query_storage_hit_length")
and (
self.tree_cache.is_backuped(last_host_node)
or self.tree_cache.is_root(last_host_node)
)
):
matched_len = l1_hit + l2_hit
suffix_tokens = recv_req.input_ids[matched_len:]
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
prefix_keys = (
self.tree_cache.get_prefix_hash_values(last_host_node)
if getattr(
self.tree_cache, "hicache_storage_pass_prefix_keys", False
)
else None
)
l3_hit = self.tree_cache.query_storage_hit_length(
last_host_node,
suffix_tokens,
last_hash,
prefix_keys,
extra_key=recv_req.extra_key,
cache_salt=recv_req.cache_salt,
)
return QueryCacheHitReqOutput(
total_tokens=total,
l1_hit_tokens=l1_hit,
l2_hit_tokens=l2_hit,
l3_hit_tokens=l3_hit,
dp_rank=int(get_parallel().dp_rank or 0),
)
except Exception as e:
logger.warning(f"handle_query_cache_hit failed: {e}")
return QueryCacheHitReqOutput(total_tokens=total, error=str(e))
def get_internal_state(self, recv_req: GetInternalStateReq):
# Resolved config (pristine server_args + post-publish overrides) so a
# readback reflects values changed via /set_internal_state, not startup.
@@ -112,6 +112,8 @@ class SchedulerBatchResultProcessor:
output_streamer: SchedulerOutputStreamer
beam_coordinator: BeamCoordinator
abort_request: Callable
# G9: scheduler-side drain-event recorder for the queue-time estimator.
record_prefill_drain: Callable
def process_batch_result_prebuilt(self, batch: ScheduleBatch):
assert self.disaggregation_mode == DisaggregationMode.DECODE
@@ -343,6 +345,9 @@ class SchedulerBatchResultProcessor:
if req.inflight_middle_chunks <= 0:
req.time_stats.set_prefill_finished_time()
# G9: exactly once per request (final chunk only) — feed the
# queue-time estimator with the drained total input length.
self.record_prefill_drain(req.seqlen)
if sampling_mask_finish_reason is not None:
req.to_finish = sampling_mask_finish_reason
@@ -54,6 +54,7 @@ class SchedulerLoadInquirer:
get_total_prefill_uncached_tokens: Callable
get_total_prefill_busy_us: Callable
get_decode_moment_totals: Callable
get_prefill_drain_rate: Callable
def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int:
"""Get the total number of tokens pending prefill.
@@ -128,9 +129,10 @@ class SchedulerLoadInquirer:
num_used_tokens, kv_token_usage = (
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
)
num_total_tokens = num_used_tokens + sum(
queued_total_input_tokens = sum(
req.seqlen for queue in pending_token_queues for req in queue
)
num_total_tokens = num_used_tokens + queued_total_input_tokens
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
memory = None
@@ -220,6 +222,8 @@ class SchedulerLoadInquirer:
num_running_reqs=num_running_reqs,
num_waiting_reqs=num_waiting_reqs,
num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(),
queued_total_input_tokens=queued_total_input_tokens,
drain_rate_tokens_per_s=round(self.get_prefill_drain_rate(), 2),
num_used_tokens=num_used_tokens,
num_total_tokens=num_total_tokens,
num_active_tokens=num_active_tokens,
@@ -54,6 +54,8 @@ from sglang.srt.managers.io_struct import (
ProfileReq,
ProfileReqOutput,
ProfileReqType,
QueryCacheHitReqInput,
QueryCacheHitReqOutput,
ReleaseMemoryOccupationReqInput,
ReleaseMemoryOccupationReqOutput,
RemoveExternalCorpusReqInput,
@@ -129,6 +131,7 @@ _COMMUNICATOR_SPECS = [
("detach_hicache_storage", DetachHiCacheStorageReqOutput),
("profile", ProfileReqOutput),
("get_internal_state", GetInternalStateReqOutput),
("query_cache_hit", QueryCacheHitReqOutput),
("set_internal_state", SetInternalStateReqOutput),
("expert_distribution", ExpertDistributionReqOutput),
("update_lora_adapter", LoRAUpdateOutput),
@@ -905,6 +908,37 @@ class TokenizerControlMixin:
self.auto_create_handle_loop()
return await self.dumper_control_communicator(obj)
async def query_cache_hit(
self: TokenizerManager, obj: QueryCacheHitReqInput
) -> QueryCacheHitReqOutput:
"""G9: fan the probe out to all DP ranks and keep the best hit.
The decode node's DP controller routes new requests to any rank, so
the bypass decision uses the best per-rank L1/L2 (L3 is shared); the
real request still goes through normal dispatch afterwards.
"""
self.auto_create_handle_loop()
results: List[QueryCacheHitReqOutput] = await self.query_cache_hit_communicator(
obj
)
ok = [r for r in results if r.error is None]
if not ok:
msg = (
"; ".join(r.error or "empty response" for r in results)
or "no scheduler responded"
)
return QueryCacheHitReqOutput(total_tokens=len(obj.input_ids), error=msg)
best = max(
ok, key=lambda r: r.l1_hit_tokens + r.l2_hit_tokens + r.l3_hit_tokens
)
return QueryCacheHitReqOutput(
total_tokens=best.total_tokens,
l1_hit_tokens=best.l1_hit_tokens,
l2_hit_tokens=best.l2_hit_tokens,
l3_hit_tokens=best.l3_hit_tokens,
dp_rank=best.dp_rank,
)
async def get_loads(
self: TokenizerManager,
include: Optional[List[str]] = None,