[Docs] Update Ascend NPU GGUF quantization documentation (#23845)

2026-04-27 17:30:24 +03:00
parent f2b84b90ac
commit 47b8eadbc4
3 changed files with 133 additions and 81 deletions
@@ -23,66 +23,66 @@ SGLang support **mix-bits** quantization (independently defines and loads each l
    <tr>
      <td>W4A4 dynamic</td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
    </tr>
    <tr>
      <td>W8A8 static</td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
    </tr>
    <tr>
      <td>W8A8 dynamic</td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
    </tr>
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/20922">MXFP8</a></td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: red;"&gt;x&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: red;"&gt;x&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: blue;"&gt;WIP&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: blue;"&gt;WIP&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'red'}}>x</strong></td>
+      <td><strong style={{color: 'red'}}>x</strong></td>
+      <td><strong style={{color: 'blue'}}>WIP</strong></td>
+      <td><strong style={{color: 'blue'}}>WIP</strong></td>
    </tr>
    <tr>
      <td>W4A4 dynamic</td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: red;"&gt;x&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
+      <td><strong style={{color: 'red'}}>x</strong></td>
    </tr>
    <tr>
      <td>W4A8 dynamic</td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: red;"&gt;x&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
+      <td><strong style={{color: 'red'}}>x</strong></td>
    </tr>
    <tr>
      <td>W8A8 dynamic</td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: red;"&gt;x&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
+      <td><strong style={{color: 'red'}}>x</strong></td>
    </tr>
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/20922">MXFP8</a></td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: red;"&gt;x&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: red;"&gt;x&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: blue;"&gt;WIP&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: red;"&gt;x&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'red'}}>x</strong></td>
+      <td><strong style={{color: 'red'}}>x</strong></td>
+      <td><strong style={{color: 'blue'}}>WIP</strong></td>
+      <td><strong style={{color: 'red'}}>x</strong></td>
    </tr>
  </tbody>
 </table>
@@ -102,23 +102,23 @@ SGLang support **mix-bits** quantization (independently defines and loads each l
    <tr>
      <td>W4A16</td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td>W8A16</td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td>W4A16</td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
  </tbody>
 </table>
@@ -138,30 +138,30 @@ GPTQ on Ascend support
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/15203">W4A16</a></td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/15203">W8A16</a></td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/16364">W4A16 MOE</a></td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/16364">W8A16 MOE</a></td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
  </tbody>
 </table>
@@ -181,30 +181,30 @@ GPTQ on Ascend support
    <tr>
      <td>W4A16</td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td>W8A16</td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td>W4A16</td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td>W8A16</td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
  </tbody>
 </table>
@@ -224,34 +224,86 @@ Compressed-tensors (LLM Compressor) on Ascend support:
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/14504">W8A8 dynamic</a></td>
      <td>Linear</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/14736">W4A8 dynamic with/without activation clip</a></td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/12759">W4A16 MOE</a></td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
    <tr>
      <td><a href="https://github.com/sgl-project/sglang/pull/14504">W8A8 dynamic</a></td>
      <td>MoE</td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: green;"&gt;√&lt;/span&gt;</strong></td>
-      <td><strong>&lt;span style="color: yellow;"&gt;TBD&lt;/span&gt;</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
    </tr>
  </tbody>
 </table>

 [GGUF on Ascend support](https://github.com/sgl-project/sglang/pull/17883)
+<table>
+  <thead>
+    <tr>
+      <th>Quantization type</th>
+      <th>Layer type</th>
+      <th>A2 Supported</th>
+      <th>A3 Supported</th>
+      <th>A5 Supported</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>All GGUF types (standard, K-quant)</td>
+      <td>Linear</td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
+    </tr>
+    <tr>
+      <td>All GGUF types (standard, K-quant)</td>
+      <td>MoE</td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'green'}}>√</strong></td>
+      <td><strong style={{color: 'orange'}}>TBD</strong></td>
+    </tr>
+  </tbody>
+</table>

-in progress
+**Usage Examples:**
+
+- Dense model (e.g. Qwen3-14B-Q4_K_M.gguf):
+
+```bash Command
+python3 -m sglang.launch_server \
+    --model-path Qwen3-14B-Q4_K_M.gguf \
+    --device npu --attention-backend ascend \
+    --host 0.0.0.0 --port 30000 \
+    --mem-fraction-static 0.7 --tp-size 2
+```
+
+- MoE model (e.g. Qwen3-30B-A3B-Q4_K_M.gguf):
+
+```bash Command
+python3 -m sglang.launch_server \
+    --model-path Qwen3-30B-A3B-Q4_K_M.gguf \
+    --device npu --attention-backend ascend \
+    --host 0.0.0.0 --port 30000 \
+    --mem-fraction-static 0.8 --tp-size 2
+```
+
+> **Implementation Notes:**
+> - GGUF weights are pre-dequantized to FP16/BF16 during model loading on CPU, then transferred to NPU for inference. This trades higher memory usage for faster runtime performance (no per-forward-pass dequantization overhead).
+> - MoE layers use `npu_grouped_matmul` and `npu_moe_init_routing` / `npu_moe_finalize_routing` for high-performance expert computation.
+> - TP (tensor parallelism) sharding is supported for both dense and MoE GGUF models.
@@ -61,7 +61,7 @@ click [Server Arguments](../../advanced_features/server_arguments).
    <tr>
      <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--load-format`</td>
      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`auto`</td>
-      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>auto</code>, <code>safetensors</code></td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>auto</code>, <code>safetensors</code>, <code>gguf</code></td>
      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>A2, A3</td>
    </tr>
    <tr>