[Core] Keep request priority within the signed 64-bit range in radix caches (#43920)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: qandrew <qandrew@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
qandrew
parent
7f9f742535
commit
6ade23696d
@@ -3902,13 +3902,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
return span_attrs
|
||||
|
||||
def _set_default_priority(self, obj: Union[GenerateReqInput, EmbeddingReqInput]):
|
||||
"""Set the default priority value."""
|
||||
"""Set the default priority and validate the signed 64-bit range."""
|
||||
if (
|
||||
self.enable_priority_scheduling
|
||||
and obj.priority is None
|
||||
and self.default_priority_value is not None
|
||||
):
|
||||
obj.priority = self.default_priority_value
|
||||
if obj.priority is not None and not -(1 << 63) <= obj.priority < (1 << 63):
|
||||
raise ValueError(
|
||||
"priority must be a signed 64-bit integer "
|
||||
"between -9223372036854775808 and 9223372036854775807"
|
||||
)
|
||||
|
||||
|
||||
class ServerStatus(Enum):
|
||||
|
||||
@@ -497,7 +497,7 @@ class RadixCache(BasePrefixCache):
|
||||
key=prompt_key,
|
||||
value=values[: len(prompt_key)],
|
||||
inserted_len=len(prompt_key),
|
||||
priority=priority + 1,
|
||||
priority=min(priority + 1, torch.iinfo(torch.int64).max),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -1170,7 +1170,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
value=values[: len(prompt_key)],
|
||||
prev_prefix_len=len(prompt_key),
|
||||
inserted_len=len(prompt_key),
|
||||
priority=insert_params.priority + 1,
|
||||
priority=min(
|
||||
insert_params.priority + 1, torch.iinfo(torch.int64).max
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user