Revert "[Feat] Add received_time in serving_base" (#14743)
This commit is contained in:
@@ -88,9 +88,6 @@ class OpenAIServingBase(ABC):
|
|||||||
"""Handle the specific request type with common pattern
|
"""Handle the specific request type with common pattern
|
||||||
If you want to override this method, you should be careful to record the validation time.
|
If you want to override this method, you should be careful to record the validation time.
|
||||||
"""
|
"""
|
||||||
received_time = time.time()
|
|
||||||
received_time_perf = time.perf_counter()
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Validate request
|
# Validate request
|
||||||
validation_start = time.perf_counter()
|
validation_start = time.perf_counter()
|
||||||
@@ -106,12 +103,6 @@ class OpenAIServingBase(ABC):
|
|||||||
if hasattr(adapted_request, "validation_time"):
|
if hasattr(adapted_request, "validation_time"):
|
||||||
adapted_request.validation_time = validation_time
|
adapted_request.validation_time = validation_time
|
||||||
|
|
||||||
if hasattr(adapted_request, "received_time"):
|
|
||||||
adapted_request.received_time = received_time
|
|
||||||
|
|
||||||
if hasattr(adapted_request, "received_time_perf"):
|
|
||||||
adapted_request.received_time_perf = received_time_perf
|
|
||||||
|
|
||||||
# Note(Xinyuan): raw_request below is only used for detecting the connection of the client
|
# Note(Xinyuan): raw_request below is only used for detecting the connection of the client
|
||||||
if hasattr(request, "stream") and request.stream:
|
if hasattr(request, "stream") and request.stream:
|
||||||
return await self._handle_streaming_request(
|
return await self._handle_streaming_request(
|
||||||
|
|||||||
@@ -204,12 +204,6 @@ class GenerateReqInput(BaseReq):
|
|||||||
# Validation step duration
|
# Validation step duration
|
||||||
validation_time: Optional[float] = None
|
validation_time: Optional[float] = None
|
||||||
|
|
||||||
# For metrics
|
|
||||||
received_time: Optional[float] = None
|
|
||||||
|
|
||||||
# Perf_counter equivalents for accurate time calculations
|
|
||||||
received_time_perf: Optional[float] = None
|
|
||||||
|
|
||||||
# For data parallel rank routing
|
# For data parallel rank routing
|
||||||
data_parallel_rank: Optional[int] = None
|
data_parallel_rank: Optional[int] = None
|
||||||
|
|
||||||
|
|||||||
@@ -431,7 +431,7 @@ class TokenizerManager(TokenizerCommunicatorMixin):
|
|||||||
obj: Union[GenerateReqInput, EmbeddingReqInput],
|
obj: Union[GenerateReqInput, EmbeddingReqInput],
|
||||||
request: Optional[fastapi.Request] = None,
|
request: Optional[fastapi.Request] = None,
|
||||||
):
|
):
|
||||||
created_time = obj.received_time if obj.received_time else time.time()
|
created_time = time.time()
|
||||||
self.auto_create_handle_loop()
|
self.auto_create_handle_loop()
|
||||||
obj.normalize_batch_and_arguments()
|
obj.normalize_batch_and_arguments()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user