Remove deprecated Mamba flags from doc, wrong FP8 GEMM docstrings and change Nemotron image to 0.5.15 (#31094)
Co-authored-by: Brayden Zhong <brayden@radixark.ai>
This commit is contained in:
co-authored by
Brayden Zhong
parent
c00206c68c
commit
7fc3fb9657
@@ -1779,7 +1779,7 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>A2, A3</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--mamba-scheduler-strategy`</td>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--mamba-radix-cache-strategy`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`auto`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`auto`,<br/>`no_buffer`,<br/>`extra_buffer`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>A2, A3</td>
|
||||
|
||||
@@ -320,7 +320,7 @@ python3 -m sglang.launch_server \
|
||||
--max-prefill-tokens 131072 \
|
||||
--max-mamba-cache-size 320 \
|
||||
--prefill-max-requests 10 \
|
||||
--mamba-scheduler-strategy extra_buffer \
|
||||
--mamba-radix-cache-strategy extra_buffer \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 64 \
|
||||
--mem-fraction-static 0.6 \
|
||||
@@ -1110,7 +1110,7 @@ python3 -m sglang.launch_server \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 65536 \
|
||||
--max-mamba-cache-size 640 \
|
||||
--mamba-scheduler-strategy extra_buffer \
|
||||
--mamba-radix-cache-strategy extra_buffer \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 128 \
|
||||
--mem-fraction-static 0.6 \
|
||||
|
||||
@@ -284,7 +284,7 @@ python3 -m sglang.launch_server \
|
||||
--device npu \
|
||||
--chunked-prefill-size 32768 \
|
||||
--max-prefill-tokens 32768 \
|
||||
--mamba-scheduler-strategy extra_buffer \
|
||||
--mamba-radix-cache-strategy extra_buffer \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 20 \
|
||||
--max-mamba-cache-size 160 \
|
||||
|
||||
@@ -389,7 +389,7 @@ python3 -m sglang.launch_server \
|
||||
--max-prefill-tokens 65536 \
|
||||
--trust-remote-code \
|
||||
--enable-prefill-delayer \
|
||||
--mamba-scheduler-strategy extra_buffer \
|
||||
--mamba-radix-cache-strategy extra_buffer \
|
||||
--max-running-requests 103 \
|
||||
--max-mamba-cache-size 85 \
|
||||
--mem-fraction-static 0.85 \
|
||||
@@ -780,7 +780,7 @@ python3 -m sglang.launch_server \
|
||||
--max-total-tokens 470784 \
|
||||
--max-prefill-tokens 65536 \
|
||||
--trust-remote-code \
|
||||
--mamba-scheduler-strategy extra_buffer \
|
||||
--mamba-radix-cache-strategy extra_buffer \
|
||||
--max-running-requests 40 \
|
||||
--max-mamba-cache-size 200 \
|
||||
--mem-fraction-static 0.9 \
|
||||
|
||||
Reference in New Issue
Block a user