minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[Bug fix] [PP] fix wrong dtype for quantified model (#12247)

Browse Source 
Signed-off-by: Xuchun Shang <xuchun.shang@gmail.com>
This commit is contained in:
Xuchun Shang
2025-10-28 01:27:24 -07:00
committed by GitHub
parent ea96106000
commit a1f2dc90e4
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_executor/cuda_graph_runner.py
+2 -2
View File
@@ -323,11 +323,11 @@ class CudaGraphRunner:
self.pp_proxy_tensors = { self.pp_proxy_tensors = {
"hidden_states": torch.zeros( "hidden_states": torch.zeros(
(self.max_bs, self.model_runner.model_config.hidden_size), (self.max_bs, self.model_runner.model_config.hidden_size),
dtype=torch.bfloat16, dtype=self.model_runner.model_config.dtype,
), ),
"residual": torch.zeros( "residual": torch.zeros(
(self.max_bs, self.model_runner.model_config.hidden_size), (self.max_bs, self.model_runner.model_config.hidden_size),
dtype=torch.bfloat16, dtype=self.model_runner.model_config.dtype,
), ),
} }