[DCP] Use logical token capacity for PD admission and load reporting (#39731)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Khoa Pham <khoa.pham@radixark.ai>
This commit is contained in:
co-authored by
Claude Opus 5.5
Khoa Pham
parent
1f6ce4b068
commit
50e50e13ab
@@ -189,7 +189,7 @@ class PrefillBootstrapQueue:
|
||||
self.scheduler = scheduler
|
||||
self.scheduler_stage_metrics = scheduler_stage_metrics
|
||||
self.max_total_num_tokens = (
|
||||
self.scheduler.tp_worker.model_runner.effective_max_total_num_tokens
|
||||
self.scheduler.tp_worker.model_runner.effective_logical_max_total_num_tokens
|
||||
)
|
||||
self.transfer_backend = transfer_backend
|
||||
if envs.SGLANG_DISAGG_STAGING_BUFFER.get():
|
||||
|
||||
@@ -1251,7 +1251,8 @@ class Scheduler(
|
||||
# TODO: max_running_requests_under_SLO has no setter — dead chain.
|
||||
max_running_requests_under_SLO=None,
|
||||
page_size=self.page_size,
|
||||
num_pages=self.max_total_num_tokens // self.page_size,
|
||||
num_pages=self.max_total_num_tokens
|
||||
// self.token_to_kv_pool_allocator.page_size,
|
||||
context_len=self.model_config.context_len,
|
||||
startup_available_gpu_memory_gb=self.startup_available_gpu_memory_gb,
|
||||
)
|
||||
@@ -2346,8 +2347,7 @@ class Scheduler(
|
||||
enable_hisparse=self.enable_hisparse,
|
||||
full_tokens_per_layer=self.full_tokens_per_layer,
|
||||
swa_tokens_per_layer=self.swa_tokens_per_layer,
|
||||
max_total_num_tokens=self.max_total_num_tokens
|
||||
* get_parallel().attn_dcp_size,
|
||||
max_total_num_tokens=self.max_total_num_tokens,
|
||||
get_last_batch=lambda: self.last_batch,
|
||||
get_running_batch=lambda: self.running_batch,
|
||||
)
|
||||
@@ -2530,7 +2530,7 @@ class Scheduler(
|
||||
max_new_tokens = self.token_to_kv_pool_allocator.max_new_tokens_for_memory(
|
||||
input_len,
|
||||
max_new_tokens,
|
||||
token_capacity=self.max_total_num_tokens * get_parallel().attn_dcp_size,
|
||||
token_capacity=self.max_total_num_tokens,
|
||||
sliding_window_size=self.sliding_window_size,
|
||||
chunk_size=self.chunked_prefill_size,
|
||||
)
|
||||
|
||||
@@ -409,9 +409,7 @@ class TpModelWorker(BaseTpWorker):
|
||||
assert self.model_runner.max_running_requests > 0, "max_running_request is zero"
|
||||
max_req_len = min(
|
||||
self.model_config.context_len - 1,
|
||||
self.model_runner.effective_max_total_num_tokens
|
||||
* get_parallel().attn_dcp_size
|
||||
- 1,
|
||||
self.model_runner.effective_logical_max_total_num_tokens - 1,
|
||||
)
|
||||
assert max_req_len > 0, "Memory pool size is too small"
|
||||
|
||||
@@ -539,17 +537,13 @@ class TpModelWorker(BaseTpWorker):
|
||||
def get_worker_info(self):
|
||||
max_req_len = min(
|
||||
self.model_config.context_len - 1,
|
||||
self.model_runner.effective_max_total_num_tokens
|
||||
* get_parallel().attn_dcp_size
|
||||
- 1,
|
||||
self.model_runner.effective_logical_max_total_num_tokens - 1,
|
||||
)
|
||||
max_req_input_len = max_req_len - 5
|
||||
if self.dllm_algorithm is not None:
|
||||
max_req_input_len -= self.dllm_algorithm.block_size
|
||||
return (
|
||||
self.model_runner.req_to_token_pool.schedulable_token_capacity(
|
||||
self.model_runner.max_total_num_tokens
|
||||
),
|
||||
self.model_runner.logical_max_total_num_tokens,
|
||||
get_schedule().max_prefill_tokens,
|
||||
self.model_runner.max_running_requests,
|
||||
get_schedule().max_queued_requests,
|
||||
|
||||
@@ -316,6 +316,17 @@ class KVCacheConfigurator:
|
||||
)
|
||||
return full_capacity or swa_capacity
|
||||
|
||||
def logical_token_capacity(self, *, max_total_num_tokens: int) -> int:
|
||||
"""Request tokens a pool of `max_total_num_tokens` per-rank rows holds."""
|
||||
# SWA allocators never widen under DCP.
|
||||
if self.is_hybrid_swa:
|
||||
return max_total_num_tokens
|
||||
# Target rows widen into attn_dcp_size ids; draft sizes already carry
|
||||
# loc_space_scale.
|
||||
return (
|
||||
max_total_num_tokens * get_parallel().attn_dcp_size // self.loc_space_scale
|
||||
)
|
||||
|
||||
def _build_fp4_quant_method(self, *, num_layers: int):
|
||||
if not is_float4_e2m1fn_x2(self.kv_cache_dtype):
|
||||
return None
|
||||
|
||||
@@ -1391,6 +1391,22 @@ class ModelRunner:
|
||||
"""Unload a lora adapter that was previously loaded during initialization or dynamic loading."""
|
||||
return self.lora_manager.unload_lora_adapter(lora_ref)
|
||||
|
||||
@property
|
||||
def logical_max_total_num_tokens(self):
|
||||
"""Request-token capacity in logical tokens, not per-rank DCP rows."""
|
||||
return self.req_to_token_pool.schedulable_token_capacity(
|
||||
self.kv_cache_configurator.logical_token_capacity(
|
||||
max_total_num_tokens=self.max_total_num_tokens
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def effective_logical_max_total_num_tokens(self):
|
||||
"""Logical request limit, preserving hybrid SWA's separate pool bounds."""
|
||||
if self.is_hybrid_swa:
|
||||
return self.effective_max_total_num_tokens
|
||||
return self.logical_max_total_num_tokens
|
||||
|
||||
@property
|
||||
def effective_max_total_num_tokens(self):
|
||||
"""Return the max token pool size considering hybrid swa settings."""
|
||||
|
||||
Reference in New Issue
Block a user