[DCP] Use logical token capacity for PD admission and load reporting (#39731)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Khoa Pham <khoa.pham@radixark.ai>
This commit is contained in:
James Liu
2026-09-25 17:39:47 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Khoa Pham
parent 1f6ce4b068
commit 50e50e13ab
6 changed files with 174 additions and 14 deletions
+1 -1
View File
@@ -189,7 +189,7 @@ class PrefillBootstrapQueue:
self.scheduler = scheduler
self.scheduler_stage_metrics = scheduler_stage_metrics
self.max_total_num_tokens = (
self.scheduler.tp_worker.model_runner.effective_max_total_num_tokens
self.scheduler.tp_worker.model_runner.effective_logical_max_total_num_tokens
)
self.transfer_backend = transfer_backend
if envs.SGLANG_DISAGG_STAGING_BUFFER.get():
+4 -4
View File
@@ -1251,7 +1251,8 @@ class Scheduler(
# TODO: max_running_requests_under_SLO has no setter — dead chain.
max_running_requests_under_SLO=None,
page_size=self.page_size,
num_pages=self.max_total_num_tokens // self.page_size,
num_pages=self.max_total_num_tokens
// self.token_to_kv_pool_allocator.page_size,
context_len=self.model_config.context_len,
startup_available_gpu_memory_gb=self.startup_available_gpu_memory_gb,
)
@@ -2346,8 +2347,7 @@ class Scheduler(
enable_hisparse=self.enable_hisparse,
full_tokens_per_layer=self.full_tokens_per_layer,
swa_tokens_per_layer=self.swa_tokens_per_layer,
max_total_num_tokens=self.max_total_num_tokens
* get_parallel().attn_dcp_size,
max_total_num_tokens=self.max_total_num_tokens,
get_last_batch=lambda: self.last_batch,
get_running_batch=lambda: self.running_batch,
)
@@ -2530,7 +2530,7 @@ class Scheduler(
max_new_tokens = self.token_to_kv_pool_allocator.max_new_tokens_for_memory(
input_len,
max_new_tokens,
token_capacity=self.max_total_num_tokens * get_parallel().attn_dcp_size,
token_capacity=self.max_total_num_tokens,
sliding_window_size=self.sliding_window_size,
chunk_size=self.chunked_prefill_size,
)
+3 -9
View File
@@ -409,9 +409,7 @@ class TpModelWorker(BaseTpWorker):
assert self.model_runner.max_running_requests > 0, "max_running_request is zero"
max_req_len = min(
self.model_config.context_len - 1,
self.model_runner.effective_max_total_num_tokens
* get_parallel().attn_dcp_size
- 1,
self.model_runner.effective_logical_max_total_num_tokens - 1,
)
assert max_req_len > 0, "Memory pool size is too small"
@@ -539,17 +537,13 @@ class TpModelWorker(BaseTpWorker):
def get_worker_info(self):
max_req_len = min(
self.model_config.context_len - 1,
self.model_runner.effective_max_total_num_tokens
* get_parallel().attn_dcp_size
- 1,
self.model_runner.effective_logical_max_total_num_tokens - 1,
)
max_req_input_len = max_req_len - 5
if self.dllm_algorithm is not None:
max_req_input_len -= self.dllm_algorithm.block_size
return (
self.model_runner.req_to_token_pool.schedulable_token_capacity(
self.model_runner.max_total_num_tokens
),
self.model_runner.logical_max_total_num_tokens,
get_schedule().max_prefill_tokens,
self.model_runner.max_running_requests,
get_schedule().max_queued_requests,
@@ -316,6 +316,17 @@ class KVCacheConfigurator:
)
return full_capacity or swa_capacity
def logical_token_capacity(self, *, max_total_num_tokens: int) -> int:
"""Request tokens a pool of `max_total_num_tokens` per-rank rows holds."""
# SWA allocators never widen under DCP.
if self.is_hybrid_swa:
return max_total_num_tokens
# Target rows widen into attn_dcp_size ids; draft sizes already carry
# loc_space_scale.
return (
max_total_num_tokens * get_parallel().attn_dcp_size // self.loc_space_scale
)
def _build_fp4_quant_method(self, *, num_layers: int):
if not is_float4_e2m1fn_x2(self.kv_cache_dtype):
return None
@@ -1391,6 +1391,22 @@ class ModelRunner:
"""Unload a lora adapter that was previously loaded during initialization or dynamic loading."""
return self.lora_manager.unload_lora_adapter(lora_ref)
@property
def logical_max_total_num_tokens(self):
"""Request-token capacity in logical tokens, not per-rank DCP rows."""
return self.req_to_token_pool.schedulable_token_capacity(
self.kv_cache_configurator.logical_token_capacity(
max_total_num_tokens=self.max_total_num_tokens
)
)
@property
def effective_logical_max_total_num_tokens(self):
"""Logical request limit, preserving hybrid SWA's separate pool bounds."""
if self.is_hybrid_swa:
return self.effective_max_total_num_tokens
return self.logical_max_total_num_tokens
@property
def effective_max_total_num_tokens(self):
"""Return the max token pool size considering hybrid swa settings."""