fix(server): clamp piecewise_cuda_graph_max_tokens to context_length (#22516)

Co-authored-by: XueLiang Yang <102161631+xueliangyang-oeuler@users.noreply.github.com>
This commit is contained in:
Jan Bernlöhr
2026-06-08 19:43:54 -07:00
committed by GitHub
co-authored by XueLiang Yang
parent 95090b837e
commit 15c801f726
+8
View File
@@ -1576,6 +1576,14 @@ class ServerArgs:
self.piecewise_cuda_graph_max_tokens, 4096
)
# Clamp to context_length if explicitly set — prevents PCG warmup
# from compiling graphs with more tokens than the model buffers
# can hold, which causes illegal memory access (#21112)
if self.context_length is not None:
self.piecewise_cuda_graph_max_tokens = min(
self.piecewise_cuda_graph_max_tokens, self.context_length
)
if self.piecewise_cuda_graph_tokens is None:
self.piecewise_cuda_graph_tokens = (
self._generate_piecewise_cuda_graph_tokens()