Update GLM-5 H200 FP8 (#25814)
Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com>
This commit is contained in:
@@ -109,6 +109,7 @@ sglang serve \
|
|||||||
--speculative-num-steps 3 \
|
--speculative-num-steps 3 \
|
||||||
--speculative-eagle-topk 1 \
|
--speculative-eagle-topk 1 \
|
||||||
--speculative-num-draft-tokens 4 \
|
--speculative-num-draft-tokens 4 \
|
||||||
|
--enable-flashinfer-allreduce-fusion \
|
||||||
--mem-fraction-static 0.85 \
|
--mem-fraction-static 0.85 \
|
||||||
--host 0.0.0.0 \
|
--host 0.0.0.0 \
|
||||||
--port 30000
|
--port 30000
|
||||||
|
|||||||
@@ -206,6 +206,11 @@ export const GLM5Deployment = () => {
|
|||||||
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// H200 FP8: flashinfer allreduce fusion.
|
||||||
|
if (hardware === 'h200' && effectiveQuant === 'fp8') {
|
||||||
|
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
||||||
|
}
|
||||||
|
|
||||||
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||||
return cmd;
|
return cmd;
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user