From 4eabf5e2281cc78f13822852e5e0da331d9e8cd9 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Fri, 9 Oct 2026 18:48:27 +0800 Subject: [PATCH] =?UTF-8?q?G9=20P0:=20loads=20=E6=8E=92=E9=98=9F=E9=A2=84?= =?UTF-8?q?=E4=BC=B0=E5=AD=97=E6=AE=B5=20+=20/query=5Fcache=5Fhit=20?= =?UTF-8?q?=E5=91=BD=E4=B8=AD=E6=9F=A5=E8=AF=A2=E7=AB=AF=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A. 负载预估数据源(决策②总输入长度口径): - LoadSnapshot 新增 queued_total_input_tokens / drain_rate_tokens_per_s - load_inquirer 复用 pending_token_queues 求和;scheduler 60s 滑动窗 记录每请求最终 prefill chunk 完成事件(batch_result_processor 挂钩 set_prefill_finished_time 处,恰好每请求一次) - /v1/loads 与 PUB socket 同源自动暴露,gateway LoadMonitor 零改动消费 B. /query_cache_hit(决策④放 D): - io_struct 新增 QueryCacheHitReqInput/Output IPC 类型 - scheduler.handle_query_cache_hit:tree_cache.match_prefix(L1/L2) + query_storage_hit_length(共享 L3,逻辑镜像 decode_hicache_mixin) - tokenizer_control_mixin:fan-out 全部 DP rank 取最优命中 - serving_chat.query_cache_hit:复用 _convert_to_internal_request + _tokenize_one_request 保证 chat 模板/分词与真实路径一致;含图请求 v1 拒绝(multimodal 400);http_server 注册 /query_cache_hit --- python/sglang/srt/entrypoints/http_server.py | 12 ++ .../srt/entrypoints/openai/serving_chat.py | 53 +++++++++ python/sglang/srt/managers/io_struct.py | 21 ++++ python/sglang/srt/managers/load_snapshot.py | 6 + python/sglang/srt/managers/scheduler.py | 104 ++++++++++++++++++ .../batch_result_processor.py | 5 + .../scheduler_components/load_inquirer.py | 6 +- .../srt/managers/tokenizer_control_mixin.py | 34 ++++++ 8 files changed, 240 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index b60f82398b..9519b9ac2b 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -1776,6 +1776,18 @@ async def openai_v1_chat_completions( ) +@app.post("/query_cache_hit", dependencies=[Depends(validate_json_request)]) +async def query_cache_hit(request: ChatCompletionRequest, raw_request: Request): + """G9: probe prefix-cache hit (L1/L2 + L3 storage) for a chat request. + + Deployed on decode engines; the PD router calls it before dispatch to + decide conditional bypass. Accepts a normal chat body or `input_ids`. + """ + return await raw_request.app.state.openai_serving_chat.query_cache_hit( + request, raw_request + ) + + @app.post( "/v1/embeddings", response_class=ORJSONResponse, diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index e8b2ed61d7..3470751e03 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -1140,6 +1140,59 @@ class OpenAIServingChat(OpenAIServingBase): return "text", processed_messages.prompt_ids return "input_ids", processed_messages.prompt_ids + async def query_cache_hit( + self, request: ChatCompletionRequest, raw_request: Request = None + ) -> ORJSONResponse: + """G9 probe: report how much of this chat request's prompt is already + cached (per-rank-best L1/L2 radix + shared L3 storage) without running + any prefill. The router uses it for conditional PD-bypass decisions. + + Multimodal requests are rejected (v1): their sentinel-token expansion + costs ~10-100ms per image for only ~2% of traffic; the router skips + them for bypass anyway. + """ + try: + internal_req, _ = self._convert_to_internal_request(request, raw_request) + except Exception as e: + return ORJSONResponse( + status_code=400, + content={"error": f"failed to convert chat request: {e}"}, + ) + if internal_req.contains_mm_input(): + return ORJSONResponse( + status_code=400, + content={"error": "multimodal requests are not supported by /query_cache_hit"}, + ) + try: + tokenized = await self.tokenizer_manager._tokenize_one_request(internal_req) + from sglang.srt.managers.io_struct import QueryCacheHitReqInput + + result = await self.tokenizer_manager.query_cache_hit( + QueryCacheHitReqInput( + input_ids=list(tokenized.input_ids), + extra_key=getattr(internal_req, "extra_key", None), + cache_salt=getattr(internal_req, "cache_salt", None), + ) + ) + except Exception as e: + return ORJSONResponse( + status_code=500, content={"error": f"cache-hit query failed: {e}"} + ) + if result.error is not None: + return ORJSONResponse(status_code=500, content={"error": result.error}) + hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens + return ORJSONResponse( + content={ + "total_tokens": result.total_tokens, + "l1_hit_tokens": result.l1_hit_tokens, + "l2_hit_tokens": result.l2_hit_tokens, + "l3_hit_tokens": result.l3_hit_tokens, + "hit_tokens": hit, + "new_tokens": max(0, result.total_tokens - hit), + "dp_rank": result.dp_rank, + } + ) + def _convert_to_internal_request( self, request: ChatCompletionRequest, diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 83b2dcd302..29a7a6f035 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -2141,6 +2141,27 @@ class SetInternalStateReqOutput(BaseReq, kw_only=True): updated: bool +class QueryCacheHitReqInput(BaseReq, kw_only=True): + """G9: read-only probe of prefix-cache hit for a token sequence. + + Sent to every DP rank of a (decode) engine; each rank answers with its own + L1/L2 radix match plus the shared L3 storage hit continuing from it. + """ + + input_ids: List[int] + extra_key: Optional[str] = None + cache_salt: Optional[str] = None + + +class QueryCacheHitReqOutput(BaseReq, kw_only=True): + total_tokens: int = 0 + l1_hit_tokens: int = 0 + l2_hit_tokens: int = 0 + l3_hit_tokens: int = 0 + dp_rank: int = 0 + error: Optional[str] = None + + class ProfileReqType(Enum): START_PROFILE = 1 STOP_PROFILE = 2 diff --git a/python/sglang/srt/managers/load_snapshot.py b/python/sglang/srt/managers/load_snapshot.py index 968f438f8d..b343197f6a 100644 --- a/python/sglang/srt/managers/load_snapshot.py +++ b/python/sglang/srt/managers/load_snapshot.py @@ -197,6 +197,12 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True): num_running_reqs: int = 0 num_waiting_reqs: int = 0 num_waiting_uncached_tokens: int = 0 + # G9 queue-time estimation (总输入长度口径, self-consistent numerator/denominator): + # sum of total input lengths (req.seqlen) over all pending queues, and the + # self-learned drain rate in the same unit — completed prefills' total + # input tokens / wall second over a sliding window. + queued_total_input_tokens: int = 0 + drain_rate_tokens_per_s: float = 0.0 num_used_tokens: int = 0 num_total_tokens: int = 0 # num_total_tokens minus tokens still awaiting a KV transfer (equal to it diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 47e0e160d6..00663bdc33 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -161,6 +161,8 @@ from sglang.srt.managers.io_struct import ( PauseGenerationReqInput, PdRoleSwitchReqInput, ProfileReq, + QueryCacheHitReqInput, + QueryCacheHitReqOutput, ReleaseMemoryOccupationReqInput, RemoveExternalCorpusReqInput, RemoveExternalCorpusReqOutput, @@ -1806,6 +1808,7 @@ class Scheduler( (ShutdownReq, self.handle_shutdown), (GetInternalStateReq, self.get_internal_state), (SetInternalStateReq, self.set_internal_state), + (QueryCacheHitReqInput, self.handle_query_cache_hit), (RpcReqInput, self.handle_rpc_request), (ExpertDistributionReq, self.expert_distribution_handle), (LoadLoRAAdapterReqInput, self.load_lora_adapter), @@ -2431,6 +2434,11 @@ class Scheduler( self._prev_step: Optional[Tuple[int, float, bool]] = None self._prev_prefill_end_ts: Optional[float] = None self._sched_idled = False + # G9: sliding-window drain events (monotonic_ts, total_input_tokens) for + # the queue-time estimator. Recorded once per request when its final + # prefill chunk completes (batch_result_processor), in the same + # total-input-length unit as LoadSnapshot.queued_total_input_tokens. + self.prefill_drain_window: deque = deque() self.load_inquirer = SchedulerLoadInquirer( disaggregation_mode=self.disaggregation_mode, server_args=self.server_args, @@ -2469,8 +2477,35 @@ class Scheduler( ), get_total_prefill_busy_us=lambda: self.total_prefill_busy_us, get_decode_moment_totals=lambda: self.decode_moment_totals, + get_prefill_drain_rate=self.get_prefill_drain_rate, ) + PREFILL_DRAIN_WINDOW_S = 60.0 + + def record_prefill_drain(self, total_input_tokens: int) -> None: + """G9: one drain event per request whose final prefill chunk completed. + + Tokens are the request's total input length (seqlen), matching the + queued_total_input_tokens numerator so the router's + wait = queued_tokens / drain_rate is unit-consistent (决策②). + """ + now = time.monotonic() + self.prefill_drain_window.append((now, total_input_tokens)) + cutoff = now - self.PREFILL_DRAIN_WINDOW_S + while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff: + self.prefill_drain_window.popleft() + + def get_prefill_drain_rate(self) -> float: + """Completed prefill input tokens per wall second over the window.""" + now = time.monotonic() + cutoff = now - self.PREFILL_DRAIN_WINDOW_S + while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff: + self.prefill_drain_window.popleft() + if not self.prefill_drain_window: + return 0.0 + span = max(now - self.prefill_drain_window[0][0], 1.0) + return sum(tokens for _, tokens in self.prefill_drain_window) / span + def init_output_streamer(self) -> None: self.output_streamer = self.get_output_streamer_class()( send_to_detokenizer=self.ipc_channels.send_to_detokenizer, @@ -2521,6 +2556,7 @@ class Scheduler( output_streamer=self.output_streamer, beam_coordinator=self.beam_coordinator, abort_request=self.abort_request, + record_prefill_drain=self.record_prefill_drain, ) def init_req_max_new_tokens(self, req): @@ -5215,6 +5251,74 @@ class Scheduler( success = False return success + def handle_query_cache_hit(self, recv_req: QueryCacheHitReqInput): + """G9: read-only prefix-hit probe (L1/L2 radix + L3 storage). + + The router calls this (via /query_cache_hit on the decode engine) to + decide conditional PD-bypass. Runs inside the scheduler event loop so + radix/storage state is consistent, and reuses the exact admission-time + match + L3 query logic (mirrors DecodeHiCachePreallocMixin + ._build_decode_prefix_match) without creating a Req or mutating + request state. Node access metadata (LRU) may be touched; a hit found + here can still be evicted before the real request arrives (inherent + TOCTOU, acceptable for an estimate). + """ + from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams + from sglang.srt.mem_cache.radix_cache import RadixKey + + total = len(recv_req.input_ids) + try: + match_result = self.tree_cache.match_prefix( + MatchPrefixParams( + key=RadixKey( + token_ids=recv_req.input_ids, + extra_key=recv_req.extra_key, + cache_salt=recv_req.cache_salt, + ) + ) + ) + l1_hit = len(match_result.device_indices) + l2_hit = getattr(match_result, "host_hit_length", 0) or 0 + l3_hit = 0 + last_host_node = getattr(match_result, "last_host_node", None) + if ( + getattr(self, "enable_decode_hicache", False) + and last_host_node is not None + and hasattr(self.tree_cache, "query_storage_hit_length") + and ( + self.tree_cache.is_backuped(last_host_node) + or self.tree_cache.is_root(last_host_node) + ) + ): + matched_len = l1_hit + l2_hit + suffix_tokens = recv_req.input_ids[matched_len:] + last_hash = self.tree_cache.get_last_hash_value(last_host_node) + prefix_keys = ( + self.tree_cache.get_prefix_hash_values(last_host_node) + if getattr( + self.tree_cache, "hicache_storage_pass_prefix_keys", False + ) + else None + ) + l3_hit = self.tree_cache.query_storage_hit_length( + last_host_node, + suffix_tokens, + last_hash, + prefix_keys, + extra_key=recv_req.extra_key, + cache_salt=recv_req.cache_salt, + ) + return QueryCacheHitReqOutput( + total_tokens=total, + l1_hit_tokens=l1_hit, + l2_hit_tokens=l2_hit, + l3_hit_tokens=l3_hit, + dp_rank=int(get_parallel().dp_rank or 0), + ) + except Exception as e: + logger.warning(f"handle_query_cache_hit failed: {e}") + return QueryCacheHitReqOutput(total_tokens=total, error=str(e)) + def get_internal_state(self, recv_req: GetInternalStateReq): # Resolved config (pristine server_args + post-publish overrides) so a # readback reflects values changed via /set_internal_state, not startup. diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index 7ce4572f59..903de34e79 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -112,6 +112,8 @@ class SchedulerBatchResultProcessor: output_streamer: SchedulerOutputStreamer beam_coordinator: BeamCoordinator abort_request: Callable + # G9: scheduler-side drain-event recorder for the queue-time estimator. + record_prefill_drain: Callable def process_batch_result_prebuilt(self, batch: ScheduleBatch): assert self.disaggregation_mode == DisaggregationMode.DECODE @@ -343,6 +345,9 @@ class SchedulerBatchResultProcessor: if req.inflight_middle_chunks <= 0: req.time_stats.set_prefill_finished_time() + # G9: exactly once per request (final chunk only) — feed the + # queue-time estimator with the drained total input length. + self.record_prefill_drain(req.seqlen) if sampling_mask_finish_reason is not None: req.to_finish = sampling_mask_finish_reason diff --git a/python/sglang/srt/managers/scheduler_components/load_inquirer.py b/python/sglang/srt/managers/scheduler_components/load_inquirer.py index 4764a76873..5cfbd97eea 100644 --- a/python/sglang/srt/managers/scheduler_components/load_inquirer.py +++ b/python/sglang/srt/managers/scheduler_components/load_inquirer.py @@ -54,6 +54,7 @@ class SchedulerLoadInquirer: get_total_prefill_uncached_tokens: Callable get_total_prefill_busy_us: Callable get_decode_moment_totals: Callable + get_prefill_drain_rate: Callable def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int: """Get the total number of tokens pending prefill. @@ -128,9 +129,10 @@ class SchedulerLoadInquirer: num_used_tokens, kv_token_usage = ( self.pool_stats_observer.get_pool_stats().get_kv_token_stats() ) - num_total_tokens = num_used_tokens + sum( + queued_total_input_tokens = sum( req.seqlen for queue in pending_token_queues for req in queue ) + num_total_tokens = num_used_tokens + queued_total_input_tokens num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens) memory = None @@ -220,6 +222,8 @@ class SchedulerLoadInquirer: num_running_reqs=num_running_reqs, num_waiting_reqs=num_waiting_reqs, num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(), + queued_total_input_tokens=queued_total_input_tokens, + drain_rate_tokens_per_s=round(self.get_prefill_drain_rate(), 2), num_used_tokens=num_used_tokens, num_total_tokens=num_total_tokens, num_active_tokens=num_active_tokens, diff --git a/python/sglang/srt/managers/tokenizer_control_mixin.py b/python/sglang/srt/managers/tokenizer_control_mixin.py index 0170ac34ea..2e5bc638ab 100644 --- a/python/sglang/srt/managers/tokenizer_control_mixin.py +++ b/python/sglang/srt/managers/tokenizer_control_mixin.py @@ -54,6 +54,8 @@ from sglang.srt.managers.io_struct import ( ProfileReq, ProfileReqOutput, ProfileReqType, + QueryCacheHitReqInput, + QueryCacheHitReqOutput, ReleaseMemoryOccupationReqInput, ReleaseMemoryOccupationReqOutput, RemoveExternalCorpusReqInput, @@ -129,6 +131,7 @@ _COMMUNICATOR_SPECS = [ ("detach_hicache_storage", DetachHiCacheStorageReqOutput), ("profile", ProfileReqOutput), ("get_internal_state", GetInternalStateReqOutput), + ("query_cache_hit", QueryCacheHitReqOutput), ("set_internal_state", SetInternalStateReqOutput), ("expert_distribution", ExpertDistributionReqOutput), ("update_lora_adapter", LoRAUpdateOutput), @@ -905,6 +908,37 @@ class TokenizerControlMixin: self.auto_create_handle_loop() return await self.dumper_control_communicator(obj) + async def query_cache_hit( + self: TokenizerManager, obj: QueryCacheHitReqInput + ) -> QueryCacheHitReqOutput: + """G9: fan the probe out to all DP ranks and keep the best hit. + + The decode node's DP controller routes new requests to any rank, so + the bypass decision uses the best per-rank L1/L2 (L3 is shared); the + real request still goes through normal dispatch afterwards. + """ + self.auto_create_handle_loop() + results: List[QueryCacheHitReqOutput] = await self.query_cache_hit_communicator( + obj + ) + ok = [r for r in results if r.error is None] + if not ok: + msg = ( + "; ".join(r.error or "empty response" for r in results) + or "no scheduler responded" + ) + return QueryCacheHitReqOutput(total_tokens=len(obj.input_ids), error=msg) + best = max( + ok, key=lambda r: r.l1_hit_tokens + r.l2_hit_tokens + r.l3_hit_tokens + ) + return QueryCacheHitReqOutput( + total_tokens=best.total_tokens, + l1_hit_tokens=best.l1_hit_tokens, + l2_hit_tokens=best.l2_hit_tokens, + l3_hit_tokens=best.l3_hit_tokens, + dp_rank=best.dp_rank, + ) + async def get_loads( self: TokenizerManager, include: Optional[List[str]] = None,