[Feature] Add MiniCPM-SALA support (#30360)
Co-authored-by: Alex Nails <alex.nails@radixark.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Alex Nails
Claude Opus 5
parent
d251fa2453
commit
092d85eb87
@@ -1298,9 +1298,12 @@ class ModelRunner:
|
||||
def effective_max_total_num_tokens(self):
|
||||
"""Return the max token pool size considering hybrid swa settings."""
|
||||
if self.is_hybrid_swa:
|
||||
return self.full_max_total_num_tokens or self.swa_max_total_num_tokens
|
||||
capacity = self.full_max_total_num_tokens or self.swa_max_total_num_tokens
|
||||
else:
|
||||
return self.max_total_num_tokens
|
||||
capacity = self.max_total_num_tokens
|
||||
if (req_to_token_pool := getattr(self, "req_to_token_pool", None)) is not None:
|
||||
return req_to_token_pool.schedulable_token_capacity(capacity)
|
||||
return capacity
|
||||
|
||||
@property
|
||||
def max_token_pool_size(self):
|
||||
|
||||
@@ -96,6 +96,7 @@ def compute_post_capture_kv_resize(
|
||||
)
|
||||
pool.finalize_backing(config)
|
||||
model_runner.token_to_kv_pool_allocator.resize(config)
|
||||
model_runner.req_to_token_pool.reset_aux_cache_allocator()
|
||||
|
||||
capped_max_running_requests = None
|
||||
if model_runner.max_running_requests is not None:
|
||||
|
||||
Reference in New Issue
Block a user