fix(server): clamp piecewise_cuda_graph_max_tokens to context_length (#22516)
Co-authored-by: XueLiang Yang <102161631+xueliangyang-oeuler@users.noreply.github.com>
This commit is contained in:
co-authored by
XueLiang Yang
parent
95090b837e
commit
15c801f726
@@ -1576,6 +1576,14 @@ class ServerArgs:
|
|||||||
self.piecewise_cuda_graph_max_tokens, 4096
|
self.piecewise_cuda_graph_max_tokens, 4096
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Clamp to context_length if explicitly set — prevents PCG warmup
|
||||||
|
# from compiling graphs with more tokens than the model buffers
|
||||||
|
# can hold, which causes illegal memory access (#21112)
|
||||||
|
if self.context_length is not None:
|
||||||
|
self.piecewise_cuda_graph_max_tokens = min(
|
||||||
|
self.piecewise_cuda_graph_max_tokens, self.context_length
|
||||||
|
)
|
||||||
|
|
||||||
if self.piecewise_cuda_graph_tokens is None:
|
if self.piecewise_cuda_graph_tokens is None:
|
||||||
self.piecewise_cuda_graph_tokens = (
|
self.piecewise_cuda_graph_tokens = (
|
||||||
self._generate_piecewise_cuda_graph_tokens()
|
self._generate_piecewise_cuda_graph_tokens()
|
||||||
|
|||||||
Reference in New Issue
Block a user