[Core] Keep request priority within the signed 64-bit range in radix caches (#43920)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: qandrew <qandrew@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-13 11:33:26 -07:00
committed by GitHub
co-authored by Hanming Lu qandrew
parent 7f9f742535
commit 6ade23696d
6 changed files with 145 additions and 5 deletions
@@ -3902,13 +3902,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
return span_attrs
def _set_default_priority(self, obj: Union[GenerateReqInput, EmbeddingReqInput]):
"""Set the default priority value."""
"""Set the default priority and validate the signed 64-bit range."""
if (
self.enable_priority_scheduling
and obj.priority is None
and self.default_priority_value is not None
):
obj.priority = self.default_priority_value
if obj.priority is not None and not -(1 << 63) <= obj.priority < (1 << 63):
raise ValueError(
"priority must be a signed 64-bit integer "
"between -9223372036854775808 and 9223372036854775807"
)
class ServerStatus(Enum):
+1 -1
View File
@@ -497,7 +497,7 @@ class RadixCache(BasePrefixCache):
key=prompt_key,
value=values[: len(prompt_key)],
inserted_len=len(prompt_key),
priority=priority + 1,
priority=min(priority + 1, torch.iinfo(torch.int64).max),
)
)
@@ -1170,7 +1170,9 @@ class UnifiedRadixCache(BasePrefixCache):
value=values[: len(prompt_key)],
prev_prefix_len=len(prompt_key),
inserted_len=len(prompt_key),
priority=insert_params.priority + 1,
priority=min(
insert_params.priority + 1, torch.iinfo(torch.int64).max
),
)
)