[Bug] Fix missing TTFT histogram for single-batch requests (#20122)
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
7662b8b919
commit
7105bf3782
@@ -134,6 +134,7 @@ class ReqState:
|
|||||||
obj: Union[GenerateReqInput, EmbeddingReqInput]
|
obj: Union[GenerateReqInput, EmbeddingReqInput]
|
||||||
time_stats: APIServerReqTimeStats
|
time_stats: APIServerReqTimeStats
|
||||||
last_completion_tokens: int = 1
|
last_completion_tokens: int = 1
|
||||||
|
ttft_observed: bool = False
|
||||||
|
|
||||||
# For streaming output
|
# For streaming output
|
||||||
last_output_offset: int = 0
|
last_output_offset: int = 0
|
||||||
@@ -1611,12 +1612,13 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
|
|||||||
}
|
}
|
||||||
|
|
||||||
state.finished = recv_obj.finished_reasons[i] is not None
|
state.finished = recv_obj.finished_reasons[i] is not None
|
||||||
if state.finished:
|
|
||||||
# Ensure first_token_time is set before computing decode_throughput.
|
# Set first_token_time on the first output batch.
|
||||||
# Without this, requests that finish on their first output batch
|
# This is the single write point for first_token_time.
|
||||||
# would have first_token_time=0.0, producing bogus throughput.
|
|
||||||
if state.time_stats.first_token_time == 0.0:
|
if state.time_stats.first_token_time == 0.0:
|
||||||
state.time_stats.set_first_token_time()
|
state.time_stats.set_first_token_time()
|
||||||
|
|
||||||
|
if state.finished:
|
||||||
state.time_stats.trace_ctx.trace_set_root_attrs(
|
state.time_stats.trace_ctx.trace_set_root_attrs(
|
||||||
self.convert_to_span_attrs(state, recv_obj, i)
|
self.convert_to_span_attrs(state, recv_obj, i)
|
||||||
)
|
)
|
||||||
@@ -1927,10 +1929,10 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
|
|||||||
if priority is not None:
|
if priority is not None:
|
||||||
labels["priority"] = str(priority)
|
labels["priority"] = str(priority)
|
||||||
if (
|
if (
|
||||||
state.time_stats.first_token_time == 0.0
|
not state.ttft_observed
|
||||||
and self.disaggregation_mode != DisaggregationMode.PREFILL
|
and self.disaggregation_mode != DisaggregationMode.PREFILL
|
||||||
):
|
):
|
||||||
state.time_stats.set_first_token_time()
|
state.ttft_observed = True
|
||||||
state.last_completion_tokens = completion_tokens
|
state.last_completion_tokens = completion_tokens
|
||||||
self.metrics_collector.observe_time_to_first_token(
|
self.metrics_collector.observe_time_to_first_token(
|
||||||
labels, state.time_stats.get_first_token_latency()
|
labels, state.time_stats.get_first_token_latency()
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ from sglang.test.test_utils import (
|
|||||||
register_cuda_ci(
|
register_cuda_ci(
|
||||||
est_time=60,
|
est_time=60,
|
||||||
suite="stage-b-test-small-1-gpu",
|
suite="stage-b-test-small-1-gpu",
|
||||||
disabled="Flaky: no TTFT histogram samples. See https://github.com/sgl-project/sglang/actions/runs/22787468608/job/66118130037",
|
|
||||||
)
|
)
|
||||||
register_amd_ci(est_time=60, suite="stage-b-test-small-1-gpu-amd")
|
register_amd_ci(est_time=60, suite="stage-b-test-small-1-gpu-amd")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user