docs(cookbook): Qwen3.5 FP8 on B200/B300 — trtllm-gen MoE + symm mem (#38374)
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
co-authored by
Po-Han Huang
parent
792543f98c
commit
f4bbf12423
@@ -410,8 +410,9 @@ export const Qwen35Deployment = () => {
|
||||
}
|
||||
}
|
||||
|
||||
// Enable NCCL symmetric memory for H100 FP8 deployments.
|
||||
if (hardware === 'h100' && quantization === 'fp8' && hwConfig.tp > 1) {
|
||||
// Enable NCCL symmetric memory for H100 and Blackwell FP8 deployments.
|
||||
const symmMemFp8Hw = ['h100', 'b200', 'b300'];
|
||||
if (symmMemFp8Hw.includes(hardware) && quantization === 'fp8' && hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --enable-symm-mem`;
|
||||
}
|
||||
|
||||
@@ -455,6 +456,16 @@ export const Qwen35Deployment = () => {
|
||||
cmd += ` \\\n --attention-backend flashinfer`;
|
||||
}
|
||||
|
||||
// Enable FlashInfer GDN (linear attention) prefill for Blackwell FP8 deployments.
|
||||
if ((hardware === 'b200' || hardware === 'b300') && quantization === 'fp8') {
|
||||
cmd += ` \\\n --linear-attn-prefill-backend flashinfer`;
|
||||
}
|
||||
|
||||
// Enable FlashInfer trtllm MoE for FP8 Blackwell deployments for MoE models.
|
||||
if ((hardware === 'b200' || hardware === 'b300') && quantization === 'fp8' && MOE_MODELS.has(model)) {
|
||||
cmd += ` \\\n --moe-runner-backend flashinfer_trtllm`;
|
||||
}
|
||||
|
||||
// Append AMD GPU-specific backend configurations.
|
||||
// All AMD MI GPUs use the AITER unified-attention backend (pair with
|
||||
// SGLANG_USE_AITER=1 and SGLANG_USE_AITER_UNIFIED_ATTN=1; see cookbook prose),
|
||||
|
||||
Reference in New Issue
Block a user