fix(server): clamp piecewise_cuda_graph_max_tokens to context_length (#22516)
Co-authored-by: XueLiang Yang <102161631+xueliangyang-oeuler@users.noreply.github.com>
This commit is contained in:
co-authored by
XueLiang Yang
parent
95090b837e
commit
15c801f726
@@ -1576,6 +1576,14 @@ class ServerArgs:
|
||||
self.piecewise_cuda_graph_max_tokens, 4096
|
||||
)
|
||||
|
||||
# Clamp to context_length if explicitly set — prevents PCG warmup
|
||||
# from compiling graphs with more tokens than the model buffers
|
||||
# can hold, which causes illegal memory access (#21112)
|
||||
if self.context_length is not None:
|
||||
self.piecewise_cuda_graph_max_tokens = min(
|
||||
self.piecewise_cuda_graph_max_tokens, self.context_length
|
||||
)
|
||||
|
||||
if self.piecewise_cuda_graph_tokens is None:
|
||||
self.piecewise_cuda_graph_tokens = (
|
||||
self._generate_piecewise_cuda_graph_tokens()
|
||||
|
||||
Reference in New Issue
Block a user