[Fix] Make tokenizer request-state initialization atomic (#41911)

Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
This commit is contained in:
C
2026-10-12 03:10:57 -07:00
committed by GitHub
co-authored by Cheng Wan
parent 180933518e
commit 28621739fc
2 changed files with 169 additions and 3 deletions
@@ -3721,15 +3721,22 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
for i in range(len(obj.rid))
]
for rid, sub_obj, bootstrap_room in items:
if rid in self.rid_to_state:
rids = set()
for rid, _, _ in items:
if rid in self.rid_to_state or rid in rids:
raise ValueError(f"Duplicate request ID detected: {rid}")
rids.add(rid)
new_states = {}
for rid, sub_obj, bootstrap_room in items:
time_stats = APIServerReqTimeStats(disagg_mode=self.disaggregation_mode)
state = ReqState([], False, asyncio.Event(), sub_obj, time_stats)
self.rid_to_state[rid] = state
if self.enable_trace:
time_stats.init_trace_ctx(rid, bootstrap_room, external_trace_header)
time_stats.set_created_time(created_time)
new_states[rid] = state
self.rid_to_state.update(new_states)
def _release_req_states_on_failure(self, request_states: Dict[str, ReqState]):
"""Release rid_to_state entries created for a failed handler.