dsv4.1: remaining model and runtime integration (#38798)
Co-authored-by: BBuf <1182563586@qq.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Xiaoyu Zhang <xiaoyu.zhang@radixark.ai> Co-authored-by: Yuwei An <ayw.sirius19@gmail.com> Co-authored-by: Khoa Pham <khoa.pham@radixark.ai> Co-authored-by: Yuhao Yang <47235274+yhyang201@users.noreply.github.com> Co-authored-by: Zhichen Zeng <zczeng@uw.edu> Co-authored-by: Ke Bao <ispobaoke@gmail.com>
This commit is contained in:
co-authored by
BBuf
Claude Opus 5
Xiaoyu Zhang
Yuwei An
Khoa Pham
Yuhao Yang
Zhichen Zeng
Ke Bao
parent
1b200ffaaa
commit
a6cf05817f
@@ -2436,6 +2436,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`None`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Type: int</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--cuda-graph-max-seq-len-prefill`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Longest sequence a prefill CUDA graph replay admits; longer batches run eager prefill. Folds into `cuda_graph_config[prefill].max_seq_len`.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`None`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Type: int</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--cuda-graph-bs-decode`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Explicit list of batch sizes to capture for the decode CUDA graph.</td>
|
||||
|
||||
@@ -369,7 +369,7 @@ export const config = {
|
||||
"--mem-fraction-static 0.8",
|
||||
"--speculative-algorithm DSPARK",
|
||||
"--speculative-dspark-block-size 5",
|
||||
"--cuda-graph-max-bs 64",
|
||||
"--cuda-graph-max-bs-decode 64",
|
||||
"--cuda-graph-backend-prefill breakable",
|
||||
"--cuda-graph-max-bs-prefill 4096",
|
||||
"--reasoning-parser auto",
|
||||
|
||||
Reference in New Issue
Block a user