[Fix] Make tokenizer request-state initialization atomic (#41911)
Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
This commit is contained in:
@@ -3721,15 +3721,22 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
for i in range(len(obj.rid))
|
||||
]
|
||||
|
||||
for rid, sub_obj, bootstrap_room in items:
|
||||
if rid in self.rid_to_state:
|
||||
rids = set()
|
||||
for rid, _, _ in items:
|
||||
if rid in self.rid_to_state or rid in rids:
|
||||
raise ValueError(f"Duplicate request ID detected: {rid}")
|
||||
rids.add(rid)
|
||||
|
||||
new_states = {}
|
||||
for rid, sub_obj, bootstrap_room in items:
|
||||
time_stats = APIServerReqTimeStats(disagg_mode=self.disaggregation_mode)
|
||||
state = ReqState([], False, asyncio.Event(), sub_obj, time_stats)
|
||||
self.rid_to_state[rid] = state
|
||||
if self.enable_trace:
|
||||
time_stats.init_trace_ctx(rid, bootstrap_room, external_trace_header)
|
||||
time_stats.set_created_time(created_time)
|
||||
new_states[rid] = state
|
||||
|
||||
self.rid_to_state.update(new_states)
|
||||
|
||||
def _release_req_states_on_failure(self, request_states: Dict[str, ReqState]):
|
||||
"""Release rid_to_state entries created for a failed handler.
|
||||
|
||||
Reference in New Issue
Block a user