Update GLM-5 H200 FP8 (#25814)
Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com>
This commit is contained in:
@@ -109,6 +109,7 @@ sglang serve \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--enable-flashinfer-allreduce-fusion \
|
||||
--mem-fraction-static 0.85 \
|
||||
--host 0.0.0.0 \
|
||||
--port 30000
|
||||
|
||||
@@ -206,6 +206,11 @@ export const GLM5Deployment = () => {
|
||||
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
||||
}
|
||||
|
||||
// H200 FP8: flashinfer allreduce fusion.
|
||||
if (hardware === 'h200' && effectiveQuant === 'fp8') {
|
||||
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
||||
}
|
||||
|
||||
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||
return cmd;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user