[Cookbook][CPU]Update CPU model support info in Cookbook (#30520)
Co-authored-by: zijiexia <37504505+zijiexia@users.noreply.github.com>
This commit is contained in:
@@ -49,7 +49,7 @@ This section provides deployment configurations optimized for different hardware
|
|||||||
- **Single GPU Deployment:** DeepSeek-OCR-2 (~3B parameters) fits on a single modern GPU — no tensor parallelism required for low-latency serving.
|
- **Single GPU Deployment:** DeepSeek-OCR-2 (~3B parameters) fits on a single modern GPU — no tensor parallelism required for low-latency serving.
|
||||||
- **High Throughput:** For high-throughput scenarios, use data parallelism with the SGLang Model Gateway. See [DP, DPA and SGLang DP Router](../../../docs/advanced_features/sgl_model_gateway).
|
- **High Throughput:** For high-throughput scenarios, use data parallelism with the SGLang Model Gateway. See [DP, DPA and SGLang DP Router](../../../docs/advanced_features/sgl_model_gateway).
|
||||||
- **NCCL timeout:** If model loading is slow, increase `--dist-timeout 3600`.
|
- **NCCL timeout:** If model loading is slow, increase `--dist-timeout 3600`.
|
||||||
- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for NUMA binding settings.
|
- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -99,7 +99,7 @@ python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-R1 --tp 8 --tru
|
|||||||
|
|
||||||
**NCCL timeout:** If model loading is slow, increase: `--dist-timeout 3600`.
|
**NCCL timeout:** If model loading is slow, increase: `--dist-timeout 3600`.
|
||||||
|
|
||||||
For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
**Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -79,7 +79,7 @@ DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGL
|
|||||||
|
|
||||||
**NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`.
|
**NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`.
|
||||||
|
|
||||||
For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
**Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ DeepGEMM is enabled by default on Hopper/Blackwell and can be disabled with `SGL
|
|||||||
|
|
||||||
**NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`.
|
**NCCL timeout:** If model loading is slow and you hit an NCCL timeout, increase it: `--dist-timeout 3600`.
|
||||||
|
|
||||||
For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
**Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ SGLang offers multiple installation methods. You can choose the most suitable in
|
|||||||
|
|
||||||
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
||||||
|
|
||||||
|
For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation).
|
||||||
|
|
||||||
## 3. Model Deployment
|
## 3. Model Deployment
|
||||||
|
|
||||||
This section provides deployment configurations optimized for different hardware platforms and use cases.
|
This section provides deployment configurations optimized for different hardware platforms and use cases.
|
||||||
@@ -55,6 +57,10 @@ import { Llama31Deployment } from "/src/snippets/autoregressive/llama31-deployme
|
|||||||
- 70B/8B: Use AMD's optimized `amd/Llama-3.1-{size}-Instruct-FP8-KV`
|
- 70B/8B: Use AMD's optimized `amd/Llama-3.1-{size}-Instruct-FP8-KV`
|
||||||
- **Tool Calling**: Enable with `--tool-call-parser llama3` for Instruct models
|
- **Tool Calling**: Enable with `--tool-call-parser llama3` for Instruct models
|
||||||
|
|
||||||
|
**Xeon CPU Deployment:**
|
||||||
|
|
||||||
|
- Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
### 4.1 Basic Usage
|
### 4.1 Basic Usage
|
||||||
|
|||||||
@@ -24,9 +24,11 @@ For more details, please refer to the [official Llama models repository](https:/
|
|||||||
|
|
||||||
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
||||||
|
|
||||||
|
For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation).
|
||||||
|
|
||||||
## 3. Model Deployment
|
## 3. Model Deployment
|
||||||
|
|
||||||
This section provides deployment configurations optimized for AMD GPUs (MI300X, MI325X, MI355X).
|
This section provides deployment configurations optimized for AMD GPUs (MI300X, MI325X, MI355X) and Intel Xeon CPUs.
|
||||||
|
|
||||||
### 3.1 Interactive Configuration
|
### 3.1 Interactive Configuration
|
||||||
|
|
||||||
@@ -45,6 +47,10 @@ import { Llama33Deployment } from "/src/snippets/autoregressive/llama33-70b-depl
|
|||||||
- **Tool Calling**: Enable with `--tool-call-parser llama3` for function calling support
|
- **Tool Calling**: Enable with `--tool-call-parser llama3` for function calling support
|
||||||
- **Higher Throughput**: Optional TP=2 or TP=4 can be used for increased throughput
|
- **Higher Throughput**: Optional TP=2 or TP=4 can be used for increased throughput
|
||||||
|
|
||||||
|
**Xeon CPU Deployment:**
|
||||||
|
|
||||||
|
Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
### 4.1 Basic Usage
|
### 4.1 Basic Usage
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ SGLang offers multiple installation methods. You can choose the most suitable in
|
|||||||
|
|
||||||
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
||||||
|
|
||||||
|
For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation).
|
||||||
|
|
||||||
## 3. Model Deployment
|
## 3. Model Deployment
|
||||||
|
|
||||||
This section provides a progressive guide from quick deployment to performance optimization, suitable for users at different levels.
|
This section provides a progressive guide from quick deployment to performance optimization, suitable for users at different levels.
|
||||||
@@ -54,6 +56,7 @@ This section provides a progressive guide from quick deployment to performance o
|
|||||||
- **Multi-Modal:** Add `--enable-multimodal` to enable image input support.
|
- **Multi-Modal:** Add `--enable-multimodal` to enable image input support.
|
||||||
- **Hybrid KV Cache:** Set `--swa-full-tokens-ratio` to control the ratio of SWA (local attention) KV tokens to full-attention KV tokens (default: 0.8, range: 0–1).
|
- **Hybrid KV Cache:** Set `--swa-full-tokens-ratio` to control the ratio of SWA (local attention) KV tokens to full-attention KV tokens (default: 0.8, range: 0–1).
|
||||||
- **EAGLE Speculative Decoding:** Supported for Llama 4 Scout and Maverick via EAGLE3. Enable with the interactive command generator above.
|
- **EAGLE Speculative Decoding:** Supported for Llama 4 Scout and Maverick via EAGLE3. Enable with the interactive command generator above.
|
||||||
|
- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ SGLang offers multiple installation methods. You can choose the most suitable in
|
|||||||
|
|
||||||
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
||||||
|
|
||||||
|
For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation).
|
||||||
|
|
||||||
## 3.Model Deployment
|
## 3.Model Deployment
|
||||||
|
|
||||||
This section provides deployment configurations optimized for different hardware platforms and use cases.
|
This section provides deployment configurations optimized for different hardware platforms and use cases.
|
||||||
@@ -47,6 +49,7 @@ import { GPTOSSDeployment } from "/src/snippets/autoregressive/gpt-oss-deploymen
|
|||||||
- **MCP tool servers:** For production, point SGLang at external MCP SSE servers with `--tool-server ip-1:port-1,ip-2:port-2`.
|
- **MCP tool servers:** For production, point SGLang at external MCP SSE servers with `--tool-server ip-1:port-1,ip-2:port-2`.
|
||||||
- **Responses API:** GPT-OSS supports OpenAI's Responses API (`client.responses.create`) in addition to the standard Chat Completions API (see section 4.2.4).
|
- **Responses API:** GPT-OSS supports OpenAI's Responses API (`client.responses.create`) in addition to the standard Chat Completions API (see section 4.2.4).
|
||||||
- **Use Python 3.12** when running the demo Python tool.
|
- **Use Python 3.12** when running the demo Python tool.
|
||||||
|
- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4.Model Invocation
|
## 4.Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -29,13 +29,15 @@ SGLang offers multiple installation methods. You can choose the most suitable in
|
|||||||
|
|
||||||
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
Please refer to the [official SGLang installation guide](../../../docs/get-started/install) for installation instructions.
|
||||||
|
|
||||||
|
For SGLang CPU installation, please refer to the [CPU version installation guide](../../../docs/hardware-platforms/cpu_server#installation).
|
||||||
|
|
||||||
## 3. Model Deployment
|
## 3. Model Deployment
|
||||||
|
|
||||||
This section provides deployment configurations optimized for AMD MI300X, MI325X and MI355X hardware platforms and different use cases.
|
This section provides deployment configurations optimized for AMD MI300X, MI325X and MI355X as well as Intel Xeon CPU hardware platforms and different use cases.
|
||||||
|
|
||||||
### 3.1 Basic Configuration
|
### 3.1 Basic Configuration
|
||||||
|
|
||||||
The Qwen2.5-VL series offers models in various sizes. The following configurations have been verified on AMD MI300X, MI325X and MI355X GPUs.
|
The Qwen2.5-VL series offers models in various sizes. The following configurations have been verified on AMD MI300X, MI325X and MI355X GPUs as well as Intel Xeon CPUs.
|
||||||
|
|
||||||
**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model size.
|
**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model size.
|
||||||
|
|
||||||
@@ -45,6 +47,7 @@ The Qwen2.5-VL series offers models in various sizes. The following configuratio
|
|||||||
|
|
||||||
* **Memory Management**: For the 72B model on MI300X/MI325X/MI355X, we have verified successful deployment with `--context-length 128000`. Smaller context lengths can be used to reduce memory usage if needed.
|
* **Memory Management**: For the 72B model on MI300X/MI325X/MI355X, we have verified successful deployment with `--context-length 128000`. Smaller context lengths can be used to reduce memory usage if needed.
|
||||||
* **Multi-GPU Deployment**: Use Tensor Parallelism (`--tp`) to scale across multiple GPUs. For example, use `--tp 8` for the 72B model and `--tp 2` for the 32B model on MI300X/MI325X/MI355X.
|
* **Multi-GPU Deployment**: Use Tensor Parallelism (`--tp`) to scale across multiple GPUs. For example, use `--tp 8` for the 72B model and `--tp 2` for the 32B model on MI300X/MI325X/MI355X.
|
||||||
|
* **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ This section provides a progressive guide from quick deployment to performance o
|
|||||||
- **Mamba Radix Cache**: Qwen3-Coder-Next's hybrid Gated Delta Networks architecture supports two mamba scheduling strategies via `--mamba-scheduler-strategy`:
|
- **Mamba Radix Cache**: Qwen3-Coder-Next's hybrid Gated Delta Networks architecture supports two mamba scheduling strategies via `--mamba-scheduler-strategy`:
|
||||||
- **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage.
|
- **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage.
|
||||||
- **V2 (`extra_buffer`)**: Enables overlap scheduling and branching point caching with `--mamba-scheduler-strategy extra_buffer --page-size 64`. Requires FLA kernel backend. Trades higher mamba state memory for better throughput. Strictly superior in non-KV-cache-bound scenarios; in KV-cache-bound cases, weigh the overlap scheduling benefit against reduced max concurrency. `--page-size` must satisfy `FLA_CHUNK_SIZE % page_size == 0` or `page_size % FLA_CHUNK_SIZE == 0` (`FLA_CHUNK_SIZE` is currently 64).
|
- **V2 (`extra_buffer`)**: Enables overlap scheduling and branching point caching with `--mamba-scheduler-strategy extra_buffer --page-size 64`. Requires FLA kernel backend. Trades higher mamba state memory for better throughput. Strictly superior in non-KV-cache-bound scenarios; in KV-cache-bound cases, weigh the overlap scheduling benefit against reduced max concurrency. `--page-size` must satisfy `FLA_CHUNK_SIZE % page_size == 0` or `page_size % FLA_CHUNK_SIZE == 0` (`FLA_CHUNK_SIZE` is currently 64).
|
||||||
- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
- **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -53,12 +53,12 @@ This section provides deployment configurations verified on AMD MI300X, MI325X,
|
|||||||
* **NVFP4 Quantization**: Requires `--quantization modelopt_fp4` and uses a different model path (`nvidia/Qwen3-Coder-...`).
|
* **NVFP4 Quantization**: Requires `--quantization modelopt_fp4` and uses a different model path (`nvidia/Qwen3-Coder-...`).
|
||||||
* **DP Attention**: NVFP4 configuration supports `--enable-dp-attention` for improved throughput.
|
* **DP Attention**: NVFP4 configuration supports `--enable-dp-attention` for improved throughput.
|
||||||
|
|
||||||
|
**Intel Xeon CPU:**
|
||||||
|
* Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
**General:**
|
**General:**
|
||||||
* **Tool Use**: To enable tool calling capabilities, add `--tool-call-parser qwen3_coder` to the launch command.
|
* **Tool Use**: To enable tool calling capabilities, add `--tool-call-parser qwen3_coder` to the launch command.
|
||||||
|
|
||||||
**Intel Xeon CPU:**
|
|
||||||
* For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
### 4.1 Basic Usage
|
### 4.1 Basic Usage
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ The Qwen3-Next series comes in only one size but offers different thinking modes
|
|||||||
- **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage.
|
- **V1 (`no_buffer`)**: Default. No overlap scheduler, lower memory usage.
|
||||||
- **V2 (`extra_buffer`)**: Enables overlap scheduling and branching point caching with `--mamba-scheduler-strategy extra_buffer --page-size 64`. Requires FLA kernel backend. Trades higher mamba state memory for better throughput. Strictly superior in non-KV-cache-bound scenarios; in KV-cache-bound cases, weigh the overlap scheduling benefit against reduced max concurrency. `--page-size` must satisfy `FLA_CHUNK_SIZE % page_size == 0` or `page_size % FLA_CHUNK_SIZE == 0` (`FLA_CHUNK_SIZE` is currently 64).
|
- **V2 (`extra_buffer`)**: Enables overlap scheduling and branching point caching with `--mamba-scheduler-strategy extra_buffer --page-size 64`. Requires FLA kernel backend. Trades higher mamba state memory for better throughput. Strictly superior in non-KV-cache-bound scenarios; in KV-cache-bound cases, weigh the overlap scheduling benefit against reduced max concurrency. `--page-size` must satisfy `FLA_CHUNK_SIZE % page_size == 0` or `page_size % FLA_CHUNK_SIZE == 0` (`FLA_CHUNK_SIZE` is currently 64).
|
||||||
|
|
||||||
- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
- **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ python -m sglang.launch_server \
|
|||||||
--enable-metrics
|
--enable-metrics
|
||||||
```
|
```
|
||||||
|
|
||||||
* For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
* **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -228,7 +228,7 @@ This section provides deployment configurations optimized for different hardware
|
|||||||
**FP8 KV Cache**: `--kv-cache-dtype fp8_e4m3` quantizes the KV cache to FP8 at runtime. Since these FP8 model checkpoints do not include pre-calibrated KV cache scaling factors, SGLang defaults to a scale of 1.0, which may cause noticeable accuracy degradation on reasoning-heavy tasks. It is not included in the generated commands above; add it manually only if memory constraints require the trade-off.
|
**FP8 KV Cache**: `--kv-cache-dtype fp8_e4m3` quantizes the KV cache to FP8 at runtime. Since these FP8 model checkpoints do not include pre-calibrated KV cache scaling factors, SGLang defaults to a scale of 1.0, which may cause noticeable accuracy degradation on reasoning-heavy tasks. It is not included in the generated commands above; add it manually only if memory constraints require the trade-off.
|
||||||
</Warning>
|
</Warning>
|
||||||
|
|
||||||
- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
- **Xeon CPU service configuration**: Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -151,7 +151,7 @@ All Qwen3.6 variants (MoE 35B-A3B and Dense 27B) fit on a single supported GPU a
|
|||||||
</table>
|
</table>
|
||||||
|
|
||||||
|
|
||||||
- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -44,16 +44,16 @@ import { Qwen3Deployment } from "/src/snippets/autoregressive/qwen3-deployment.j
|
|||||||
|
|
||||||
### 3.2 Configuration Tips
|
### 3.2 Configuration Tips
|
||||||
|
|
||||||
- **Memory Management** : Set lower `--context-length` to conserve memory. A value of `128000` is sufficient for most scenarios, down from the default 262K.
|
- **Memory Management:** Set lower `--context-length` to conserve memory. A value of `128000` is sufficient for most scenarios, down from the default 262K.
|
||||||
- **Expert Parallelism** : SGLang supports Expert Parallelism (EP) via `--ep`, allowing experts in MoE models to be deployed on separate GPUs for better throughput. One thing to note is that, for quantized models, you need to set `--ep` to a value that satisfies the requirement: `(moe_intermediate_size / moe_tp_size) % weight_block_size_n == 0, where moe_tp_size is equal to tp_size divided by ep_size.` Note that EP may perform worse in low concurrency scenarios due to additional communication overhead. Check out [Expert Parallelism Deployment](../../../docs/advanced_features/expert_parallelism) for more details.
|
- **Expert Parallelism:** SGLang supports Expert Parallelism (EP) via `--ep`, allowing experts in MoE models to be deployed on separate GPUs for better throughput. One thing to note is that, for quantized models, you need to set `--ep` to a value that satisfies the requirement: `(moe_intermediate_size / moe_tp_size) % weight_block_size_n == 0, where moe_tp_size is equal to tp_size divided by ep_size.` Note that EP may perform worse in low concurrency scenarios due to additional communication overhead. Check out [Expert Parallelism Deployment](../../../docs/advanced_features/expert_parallelism) for more details.
|
||||||
- **Kernel Tuning** : For MoE Triton kernel tuning on your specific hardware, refer to [fused_moe_triton](https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton).
|
- **Kernel Tuning:** For MoE Triton kernel tuning on your specific hardware, refer to [fused_moe_triton](https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton).
|
||||||
- **Speculative Decoding**: Using Speculative Decoding for latency-sensitive scenarios.
|
- **Speculative Decoding:** Using Speculative Decoding for latency-sensitive scenarios.
|
||||||
- `--speculative-algorithm EAGLE3`: Speculative decoding algorithm
|
- `--speculative-algorithm EAGLE3`: Speculative decoding algorithm
|
||||||
- `--speculative-num-steps 3`: Number of speculative verification rounds
|
- `--speculative-num-steps 3`: Number of speculative verification rounds
|
||||||
- `--speculative-eagle-topk 1`: Top-k sampling for draft tokens
|
- `--speculative-eagle-topk 1`: Top-k sampling for draft tokens
|
||||||
- `--speculative-num-draft-tokens 4`: Number of draft tokens per step
|
- `--speculative-num-draft-tokens 4`: Number of draft tokens per step
|
||||||
- `--speculative-draft-model-path`: The path of the draft model weights. This can be a local folder or a Hugging Face repo ID such as [`lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan`](https://huggingface.co/lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan).
|
- `--speculative-draft-model-path`: The path of the draft model weights. This can be a local folder or a Hugging Face repo ID such as [`lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan`](https://huggingface.co/lmsys/SGLang-EAGLE3-Qwen3-235B-A22B-Instruct-2507-SpecForge-Meituan).
|
||||||
- For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
- **Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -168,7 +168,7 @@ sglang serve \
|
|||||||
|
|
||||||
Toggle the "Speculative Decoding (MTP)" option in the generator above to add these flags automatically. Tune `num-steps` / `num-draft-tokens` based on acceptance rate in your workload.
|
Toggle the "Speculative Decoding (MTP)" option in the generator above to add these flags automatically. Tune `num-steps` / `num-draft-tokens` based on acceptance rate in your workload.
|
||||||
|
|
||||||
For configuring CPU service, please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
**Xeon CPU service configuration:** Please refer to the `Notes` part in the serving engine launching section in [the SGLang CPU server document](../../../docs/hardware-platforms/cpu_server#launch-of-the-serving-engine) to better understand how to configure the arguments, especially for TP (tensor parallel) and NUMA binding settings.
|
||||||
|
|
||||||
## 4. Model Invocation
|
## 4. Model Invocation
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,8 @@ export const GPTOSSDeployment = () => {
|
|||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||||
|
{ id: 'xeon', label: 'XEON', default: false }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
modelsize: {
|
modelsize: {
|
||||||
@@ -56,6 +57,18 @@ export const GPTOSSDeployment = () => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const getDisplayOptions = (values) => ({
|
||||||
|
...options,
|
||||||
|
quantization: options.quantization,
|
||||||
|
speculative: {
|
||||||
|
...options.speculative,
|
||||||
|
items: options.speculative.items.map(item => ({
|
||||||
|
...item,
|
||||||
|
disabled: values.hardware === 'xeon' && item.id === 'enabled'
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Initialize state
|
// Initialize state
|
||||||
const getInitialState = () => {
|
const getInitialState = () => {
|
||||||
const initialState = {};
|
const initialState = {};
|
||||||
@@ -89,7 +102,13 @@ export const GPTOSSDeployment = () => {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleRadioChange = (optionName, value) => {
|
const handleRadioChange = (optionName, value) => {
|
||||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
setValues(prev => {
|
||||||
|
const next = { ...prev, [optionName]: value };
|
||||||
|
if (optionName === 'hardware' && value === 'xeon') {
|
||||||
|
next.speculative = 'disabled';
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
// Generate command
|
// Generate command
|
||||||
@@ -106,7 +125,8 @@ export const GPTOSSDeployment = () => {
|
|||||||
b300: { tp: 8 },
|
b300: { tp: 8 },
|
||||||
mi300x: { tp: 8 },
|
mi300x: { tp: 8 },
|
||||||
mi325x: { tp: 8 },
|
mi325x: { tp: 8 },
|
||||||
mi355x: { tp: 8 }
|
mi355x: { tp: 8 },
|
||||||
|
xeon: { tp: 3 }
|
||||||
},
|
},
|
||||||
'20b': {
|
'20b': {
|
||||||
baseName: '20b',
|
baseName: '20b',
|
||||||
@@ -116,7 +136,8 @@ export const GPTOSSDeployment = () => {
|
|||||||
b300: { tp: 1 },
|
b300: { tp: 1 },
|
||||||
mi300x: { tp: 1 },
|
mi300x: { tp: 1 },
|
||||||
mi325x: { tp: 1 },
|
mi325x: { tp: 1 },
|
||||||
mi355x: { tp: 1 }
|
mi355x: { tp: 1 },
|
||||||
|
xeon: { tp: 3 }
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -146,6 +167,7 @@ export const GPTOSSDeployment = () => {
|
|||||||
return '# MI300X/MI325X GPUs with MXFP4 quantization: Work In Progress';
|
return '# MI300X/MI325X GPUs with MXFP4 quantization: Work In Progress';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// AMD MI30x requires SGLANG_USE_AITER=0 due to YaRN RoPE precision issues
|
// AMD MI30x requires SGLANG_USE_AITER=0 due to YaRN RoPE precision issues
|
||||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||||
cmd += 'SGLANG_USE_AITER=0 ';
|
cmd += 'SGLANG_USE_AITER=0 ';
|
||||||
@@ -159,6 +181,12 @@ export const GPTOSSDeployment = () => {
|
|||||||
|
|
||||||
cmd += ` --model ${modelName}`;
|
cmd += ` --model ${modelName}`;
|
||||||
|
|
||||||
|
if (hardware === 'xeon') {
|
||||||
|
cmd += ` \\
|
||||||
|
--device cpu \\
|
||||||
|
--disable-overlap-schedule`;
|
||||||
|
}
|
||||||
|
|
||||||
if (hwConfig.tp > 1) {
|
if (hwConfig.tp > 1) {
|
||||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||||
}
|
}
|
||||||
@@ -206,7 +234,7 @@ export const GPTOSSDeployment = () => {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div style={containerStyle} className="not-prose">
|
<div style={containerStyle} className="not-prose">
|
||||||
{Object.entries(options).map(([key, option]) => (
|
{Object.entries(getDisplayOptions(values)).map(([key, option]) => (
|
||||||
<div key={key} style={cardStyle}>
|
<div key={key} style={cardStyle}>
|
||||||
<div style={titleStyle}>{option.title}</div>
|
<div style={titleStyle}>{option.title}</div>
|
||||||
<div style={itemsStyle}>
|
<div style={itemsStyle}>
|
||||||
@@ -225,9 +253,10 @@ export const GPTOSSDeployment = () => {
|
|||||||
) : (
|
) : (
|
||||||
option.items.map(item => {
|
option.items.map(item => {
|
||||||
const isChecked = values[option.name] === item.id;
|
const isChecked = values[option.name] === item.id;
|
||||||
|
const isDisabled = Boolean(item.disabled);
|
||||||
return (
|
return (
|
||||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
<input type="radio" name={option.name} value={item.id} checked={isChecked} disabled={isDisabled} onChange={() => !isDisabled && handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||||
{item.label}
|
{item.label}
|
||||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||||
</label>
|
</label>
|
||||||
|
|||||||
@@ -10,7 +10,8 @@ export const Llama31Deployment = () => {
|
|||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||||
|
{ id: 'xeon', label: 'XEON', default: false }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
modelsize: {
|
modelsize: {
|
||||||
@@ -57,6 +58,34 @@ export const Llama31Deployment = () => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const getDisplayOptions = (values) => {
|
||||||
|
const displayOptions = {
|
||||||
|
...options,
|
||||||
|
modelsize: {
|
||||||
|
...options.modelsize,
|
||||||
|
items: options.modelsize.items.map(item => ({
|
||||||
|
...item,
|
||||||
|
disabled: values.hardware === 'xeon' && item.id !== '8b'
|
||||||
|
}))
|
||||||
|
},
|
||||||
|
quantization: {
|
||||||
|
...options.quantization,
|
||||||
|
items: options.quantization.items.map(item => ({
|
||||||
|
...item,
|
||||||
|
disabled: values.hardware === 'xeon' && item.id === 'fp8'
|
||||||
|
}))
|
||||||
|
},
|
||||||
|
optimization: {
|
||||||
|
...options.optimization,
|
||||||
|
items: options.optimization.items.map(item => ({
|
||||||
|
...item,
|
||||||
|
disabled: values.hardware === 'xeon' && item.id !== 'basic'
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return displayOptions;
|
||||||
|
};
|
||||||
|
|
||||||
// Initialize state
|
// Initialize state
|
||||||
const getInitialState = () => {
|
const getInitialState = () => {
|
||||||
const initialState = {};
|
const initialState = {};
|
||||||
@@ -86,7 +115,15 @@ export const Llama31Deployment = () => {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleRadioChange = (optionName, value) => {
|
const handleRadioChange = (optionName, value) => {
|
||||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
setValues(prev => {
|
||||||
|
const next = { ...prev, [optionName]: value };
|
||||||
|
if (optionName === 'hardware' && value === 'xeon') {
|
||||||
|
next.modelsize = '8b';
|
||||||
|
next.quantization = 'bf16';
|
||||||
|
next.optimization = 'basic';
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
// Generate command
|
// Generate command
|
||||||
@@ -94,6 +131,8 @@ export const Llama31Deployment = () => {
|
|||||||
const { hardware, optimization, modelsize, category, toolcall, quantization } = values;
|
const { hardware, optimization, modelsize, category, toolcall, quantization } = values;
|
||||||
|
|
||||||
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
|
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
|
||||||
|
const isXeon = hardware === 'xeon';
|
||||||
|
const effectiveModelSize = isXeon ? '8b' : modelsize;
|
||||||
|
|
||||||
// Model size mapping
|
// Model size mapping
|
||||||
const sizeMap = {
|
const sizeMap = {
|
||||||
@@ -101,13 +140,13 @@ export const Llama31Deployment = () => {
|
|||||||
'70b': '70B',
|
'70b': '70B',
|
||||||
'405b': '405B'
|
'405b': '405B'
|
||||||
};
|
};
|
||||||
const sizeToken = sizeMap[modelsize] || '70B';
|
const sizeToken = sizeMap[effectiveModelSize] || '70B';
|
||||||
const categorySuffix = category === 'instruct' ? '-Instruct' : '';
|
const categorySuffix = category === 'instruct' ? '-Instruct' : '';
|
||||||
|
|
||||||
// Determine model path
|
// Determine model path
|
||||||
let modelPath;
|
let modelPath;
|
||||||
if (quantization === 'fp8' && category === 'instruct') {
|
if (quantization === 'fp8' && category === 'instruct' && !isXeon) {
|
||||||
if (modelsize === '405b') {
|
if (effectiveModelSize === '405b') {
|
||||||
// Meta official FP8 for 405B
|
// Meta official FP8 for 405B
|
||||||
modelPath = `meta-llama/Llama-3.1-${sizeToken}${categorySuffix}-FP8`;
|
modelPath = `meta-llama/Llama-3.1-${sizeToken}${categorySuffix}-FP8`;
|
||||||
} else if (isAMD) {
|
} else if (isAMD) {
|
||||||
@@ -142,13 +181,16 @@ export const Llama31Deployment = () => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
tpSize = quantization === 'fp8'
|
tpSize = quantization === 'fp8'
|
||||||
? amdTpConfig[hardware][modelsize].fp8
|
? amdTpConfig[hardware][effectiveModelSize].fp8
|
||||||
: amdTpConfig[hardware][modelsize].bf16;
|
: amdTpConfig[hardware][effectiveModelSize].bf16;
|
||||||
|
} else if (isXeon) {
|
||||||
|
// Intel Xeon CPU TP configuration
|
||||||
|
tpSize = 3;
|
||||||
} else {
|
} else {
|
||||||
// NVIDIA GPU TP configuration
|
// NVIDIA GPU TP configuration
|
||||||
if (modelsize === '405b') {
|
if (effectiveModelSize === '405b') {
|
||||||
tpSize = 8;
|
tpSize = 8;
|
||||||
} else if (modelsize === '70b' && (hardware === 'h100' || hardware === 'h200')) {
|
} else if (effectiveModelSize === '70b' && (hardware === 'h100' || hardware === 'h200')) {
|
||||||
tpSize = 2;
|
tpSize = 2;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -157,17 +199,22 @@ export const Llama31Deployment = () => {
|
|||||||
const args = [];
|
const args = [];
|
||||||
args.push(`--model-path ${modelPath}`);
|
args.push(`--model-path ${modelPath}`);
|
||||||
|
|
||||||
|
if (isXeon) {
|
||||||
|
args.push(`--device cpu`);
|
||||||
|
args.push(`--disable-overlap-schedule`);
|
||||||
|
}
|
||||||
|
|
||||||
if (tpSize) {
|
if (tpSize) {
|
||||||
args.push(`--tp ${tpSize}`);
|
args.push(`--tp ${tpSize}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add quantization flag only if not using FP8 variant model
|
// Add quantization flag only if not using FP8 variant model
|
||||||
if (quantization === 'fp8' && category !== 'instruct') {
|
if (quantization === 'fp8' && category !== 'instruct' && !isXeon) {
|
||||||
args.push(`--quantization fp8`);
|
args.push(`--quantization fp8`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// NVIDIA-specific optimizations
|
// NVIDIA-specific optimizations
|
||||||
if (!isAMD) {
|
if (!isAMD && !isXeon) {
|
||||||
if (optimization === 'throughput') {
|
if (optimization === 'throughput') {
|
||||||
args.push(`--enable-dp-attention`);
|
args.push(`--enable-dp-attention`);
|
||||||
args.push(`--mem-fraction-static 0.85`);
|
args.push(`--mem-fraction-static 0.85`);
|
||||||
@@ -176,7 +223,7 @@ export const Llama31Deployment = () => {
|
|||||||
args.push(`--speculative-num-steps 3`);
|
args.push(`--speculative-num-steps 3`);
|
||||||
args.push(`--speculative-eagle-topk 1`);
|
args.push(`--speculative-eagle-topk 1`);
|
||||||
args.push(`--speculative-num-draft-tokens 4`);
|
args.push(`--speculative-num-draft-tokens 4`);
|
||||||
if (modelsize === '8b' && category === 'instruct') {
|
if (effectiveModelSize === '8b' && category === 'instruct') {
|
||||||
args.push(`--speculative-draft-model-path yuhuili/EAGLE3-LLaMA3.1-Instruct-8B`);
|
args.push(`--speculative-draft-model-path yuhuili/EAGLE3-LLaMA3.1-Instruct-8B`);
|
||||||
} else {
|
} else {
|
||||||
args.push(`--speculative-draft-model-path \${EAGLE3_MODEL_PATH}`);
|
args.push(`--speculative-draft-model-path \${EAGLE3_MODEL_PATH}`);
|
||||||
@@ -212,7 +259,7 @@ export const Llama31Deployment = () => {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div style={containerStyle} className="not-prose">
|
<div style={containerStyle} className="not-prose">
|
||||||
{Object.entries(options).map(([key, option]) => (
|
{Object.entries(getDisplayOptions(values)).map(([key, option]) => (
|
||||||
<div key={key} style={cardStyle}>
|
<div key={key} style={cardStyle}>
|
||||||
<div style={titleStyle}>{option.title}</div>
|
<div style={titleStyle}>{option.title}</div>
|
||||||
<div style={itemsStyle}>
|
<div style={itemsStyle}>
|
||||||
@@ -231,9 +278,10 @@ export const Llama31Deployment = () => {
|
|||||||
) : (
|
) : (
|
||||||
option.items.map(item => {
|
option.items.map(item => {
|
||||||
const isChecked = values[option.name] === item.id;
|
const isChecked = values[option.name] === item.id;
|
||||||
|
const isDisabled = Boolean(item.disabled);
|
||||||
return (
|
return (
|
||||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
<input type="radio" name={option.name} value={item.id} checked={isChecked} disabled={isDisabled} onChange={() => !isDisabled && handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||||
{item.label}
|
{item.label}
|
||||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||||
</label>
|
</label>
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ export const Llama33Deployment = () => {
|
|||||||
items: [
|
items: [
|
||||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||||
|
{ id: 'xeon', label: 'XEON', default: false }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
quantization: {
|
quantization: {
|
||||||
@@ -28,6 +29,17 @@ export const Llama33Deployment = () => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const getDisplayOptions = (values) => ({
|
||||||
|
...options,
|
||||||
|
quantization: {
|
||||||
|
...options.quantization,
|
||||||
|
items: options.quantization.items.map(item => ({
|
||||||
|
...item,
|
||||||
|
disabled: values.hardware === 'xeon' && item.id === 'fp8'
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Initialize state
|
// Initialize state
|
||||||
const getInitialState = () => {
|
const getInitialState = () => {
|
||||||
const initialState = {};
|
const initialState = {};
|
||||||
@@ -57,7 +69,13 @@ export const Llama33Deployment = () => {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleRadioChange = (optionName, value) => {
|
const handleRadioChange = (optionName, value) => {
|
||||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
setValues(prev => {
|
||||||
|
const next = { ...prev, [optionName]: value };
|
||||||
|
if (optionName === 'hardware' && value === 'xeon') {
|
||||||
|
next.quantization = 'bf16';
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
// Generate command
|
// Generate command
|
||||||
@@ -65,14 +83,20 @@ export const Llama33Deployment = () => {
|
|||||||
const { hardware, quantization, toolcall } = values;
|
const { hardware, quantization, toolcall } = values;
|
||||||
|
|
||||||
// Select model based on quantization
|
// Select model based on quantization
|
||||||
const modelPath = quantization === 'fp8'
|
const modelPath = quantization === 'fp8' && hardware !== 'xeon'
|
||||||
? 'amd/Llama-3.3-70B-Instruct-FP8-KV'
|
? 'amd/Llama-3.3-70B-Instruct-FP8-KV'
|
||||||
: 'meta-llama/Llama-3.3-70B-Instruct';
|
: 'meta-llama/Llama-3.3-70B-Instruct';
|
||||||
|
|
||||||
// Build command
|
// Build command
|
||||||
let cmd = 'python -m sglang.launch_server \\\n';
|
let cmd = 'python -m sglang.launch_server \\\n';
|
||||||
cmd += ` --model-path ${modelPath} \\\n`;
|
cmd += ` --model-path ${modelPath} \\\n`;
|
||||||
|
if (hardware === 'xeon') {
|
||||||
|
cmd += ` --device cpu \\\n`;
|
||||||
|
cmd += ` --disable-overlap-schedule \\\n`;
|
||||||
|
cmd += ` --tp 6`;
|
||||||
|
} else {
|
||||||
cmd += ` --tp 1`;
|
cmd += ` --tp 1`;
|
||||||
|
}
|
||||||
|
|
||||||
// Add tool calling parser
|
// Add tool calling parser
|
||||||
if (toolcall === 'enabled') {
|
if (toolcall === 'enabled') {
|
||||||
@@ -98,7 +122,7 @@ export const Llama33Deployment = () => {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div style={containerStyle} className="not-prose">
|
<div style={containerStyle} className="not-prose">
|
||||||
{Object.entries(options).map(([key, option]) => (
|
{Object.entries(getDisplayOptions(values)).map(([key, option]) => (
|
||||||
<div key={key} style={cardStyle}>
|
<div key={key} style={cardStyle}>
|
||||||
<div style={titleStyle}>{option.title}</div>
|
<div style={titleStyle}>{option.title}</div>
|
||||||
<div style={itemsStyle}>
|
<div style={itemsStyle}>
|
||||||
@@ -117,9 +141,10 @@ export const Llama33Deployment = () => {
|
|||||||
) : (
|
) : (
|
||||||
option.items.map(item => {
|
option.items.map(item => {
|
||||||
const isChecked = values[option.name] === item.id;
|
const isChecked = values[option.name] === item.id;
|
||||||
|
const isDisabled = Boolean(item.disabled);
|
||||||
return (
|
return (
|
||||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
<input type="radio" name={option.name} value={item.id} checked={isChecked} disabled={isDisabled} onChange={() => !isDisabled && handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||||
{item.label}
|
{item.label}
|
||||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||||
</label>
|
</label>
|
||||||
|
|||||||
@@ -8,15 +8,16 @@ export const Llama4MaverickDeployment = () => {
|
|||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300x', default: true },
|
{ id: 'mi300x', label: 'MI300x', default: true },
|
||||||
{ id: 'mi325x', label: 'MI325x', default: false },
|
{ id: 'mi325x', label: 'MI325x', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355x', default: false }
|
{ id: 'mi355x', label: 'MI355x', default: false },
|
||||||
|
{ id: 'xeon', label: 'XEON', default: false }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
quantization: {
|
quantization: {
|
||||||
name: 'quantization',
|
name: 'quantization',
|
||||||
title: 'Quantization',
|
title: 'Quantization',
|
||||||
items: [
|
getDynamicItems: (values) => [
|
||||||
{ id: 'bf16', label: 'BF16', default: true },
|
{ id: 'bf16', label: 'BF16', default: true },
|
||||||
{ id: 'fp8', label: 'FP8', default: false }
|
{ id: 'fp8', label: 'FP8', default: false, disabled: values.hardware === 'xeon' }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
toolcall: {
|
toolcall: {
|
||||||
@@ -30,6 +31,7 @@ export const Llama4MaverickDeployment = () => {
|
|||||||
speculative: {
|
speculative: {
|
||||||
name: 'speculative',
|
name: 'speculative',
|
||||||
title: 'Speculative Decoding (EAGLE3)',
|
title: 'Speculative Decoding (EAGLE3)',
|
||||||
|
condition: (values) => values.hardware !== 'xeon',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'disabled', label: 'Disabled', default: true },
|
{ id: 'disabled', label: 'Disabled', default: true },
|
||||||
{ id: 'enabled', label: 'Enable EAGLE3', default: false }
|
{ id: 'enabled', label: 'Enable EAGLE3', default: false }
|
||||||
@@ -63,9 +65,11 @@ export const Llama4MaverickDeployment = () => {
|
|||||||
cmd += ` \\\n --tp 8`;
|
cmd += ` \\\n --tp 8`;
|
||||||
} else if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
} else if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||||
cmd += ` \\\n --tp 8`;
|
cmd += ` \\\n --tp 8`;
|
||||||
|
} else if (hardware === 'xeon') {
|
||||||
|
cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule \\\n --tp 6`;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (quantization === 'fp8') {
|
if (quantization === 'fp8' && hardware !== 'xeon') {
|
||||||
cmd += ` \\\n --quantization fp8`;
|
cmd += ` \\\n --quantization fp8`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,7 +77,7 @@ export const Llama4MaverickDeployment = () => {
|
|||||||
cmd += ` \\\n --tool-call-parser pythonic`;
|
cmd += ` \\\n --tool-call-parser pythonic`;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (speculative === 'enabled') {
|
if (speculative === 'enabled' && hardware !== 'xeon') {
|
||||||
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n`;
|
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n`;
|
||||||
cmd += ` --speculative-draft-model-path lmsys/sglang-EAGLE3-Llama-4-Maverick-17B-128E-Instruct-v1 \\\n`;
|
cmd += ` --speculative-draft-model-path lmsys/sglang-EAGLE3-Llama-4-Maverick-17B-128E-Instruct-v1 \\\n`;
|
||||||
cmd += ` --speculative-num-steps 3 \\\n`;
|
cmd += ` --speculative-num-steps 3 \\\n`;
|
||||||
@@ -151,7 +155,14 @@ export const Llama4MaverickDeployment = () => {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleRadioChange = (optionName, value) => {
|
const handleRadioChange = (optionName, value) => {
|
||||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
setValues((prev) => {
|
||||||
|
const next = { ...prev, [optionName]: value };
|
||||||
|
if (optionName === 'hardware' && value === 'xeon') {
|
||||||
|
next.quantization = 'bf16';
|
||||||
|
next.speculative = 'disabled';
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||||
|
|||||||
@@ -9,15 +9,16 @@ export const Llama4ScoutDeployment = () => {
|
|||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300x', default: false },
|
{ id: 'mi300x', label: 'MI300x', default: false },
|
||||||
{ id: 'mi325x', label: 'MI325x', default: false },
|
{ id: 'mi325x', label: 'MI325x', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355x', default: false }
|
{ id: 'mi355x', label: 'MI355x', default: false },
|
||||||
|
{ id: 'xeon', label: 'XEON', default: false }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
quantization: {
|
quantization: {
|
||||||
name: 'quantization',
|
name: 'quantization',
|
||||||
title: 'Quantization',
|
title: 'Quantization',
|
||||||
items: [
|
getDynamicItems: (values) => [
|
||||||
{ id: 'bf16', label: 'BF16', default: true },
|
{ id: 'bf16', label: 'BF16', default: true },
|
||||||
{ id: 'fp8', label: 'FP8', default: false }
|
{ id: 'fp8', label: 'FP8', default: false, disabled: values.hardware === 'xeon' }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
toolcall: {
|
toolcall: {
|
||||||
@@ -31,6 +32,7 @@ export const Llama4ScoutDeployment = () => {
|
|||||||
speculative: {
|
speculative: {
|
||||||
name: 'speculative',
|
name: 'speculative',
|
||||||
title: 'Speculative Decoding (EAGLE3)',
|
title: 'Speculative Decoding (EAGLE3)',
|
||||||
|
condition: (values) => values.hardware !== 'xeon',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'disabled', label: 'Disabled', default: true },
|
{ id: 'disabled', label: 'Disabled', default: true },
|
||||||
{ id: 'enabled', label: 'Enable EAGLE3', default: false }
|
{ id: 'enabled', label: 'Enable EAGLE3', default: false }
|
||||||
@@ -64,9 +66,11 @@ export const Llama4ScoutDeployment = () => {
|
|||||||
cmd += ` \\\n --tp 8`;
|
cmd += ` \\\n --tp 8`;
|
||||||
} else if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
} else if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||||
cmd += ` \\\n --tp 8`;
|
cmd += ` \\\n --tp 8`;
|
||||||
|
} else if (hardware === 'xeon') {
|
||||||
|
cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule \\\n --tp 6`;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (quantization === 'fp8') {
|
if (quantization === 'fp8' && hardware !== 'xeon') {
|
||||||
cmd += ` \\\n --quantization fp8`;
|
cmd += ` \\\n --quantization fp8`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,7 +78,7 @@ export const Llama4ScoutDeployment = () => {
|
|||||||
cmd += ` \\\n --tool-call-parser pythonic`;
|
cmd += ` \\\n --tool-call-parser pythonic`;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (speculative === 'enabled') {
|
if (speculative === 'enabled' && hardware !== 'xeon') {
|
||||||
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n`;
|
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n`;
|
||||||
cmd += ` --speculative-draft-model-path lmsys/sglang-EAGLE3-Llama-4-Scout-17B-16E-Instruct-v1 \\\n`;
|
cmd += ` --speculative-draft-model-path lmsys/sglang-EAGLE3-Llama-4-Scout-17B-16E-Instruct-v1 \\\n`;
|
||||||
cmd += ` --speculative-num-steps 3 \\\n`;
|
cmd += ` --speculative-num-steps 3 \\\n`;
|
||||||
@@ -152,7 +156,14 @@ export const Llama4ScoutDeployment = () => {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleRadioChange = (optionName, value) => {
|
const handleRadioChange = (optionName, value) => {
|
||||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
setValues((prev) => {
|
||||||
|
const next = { ...prev, [optionName]: value };
|
||||||
|
if (optionName === 'hardware' && value === 'xeon') {
|
||||||
|
next.quantization = 'bf16';
|
||||||
|
next.speculative = 'disabled';
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ export const Qwen25VLDeployment = () => {
|
|||||||
items: [
|
items: [
|
||||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||||
|
{ id: 'xeon', label: 'XEON', default: false }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
modelsize: {
|
modelsize: {
|
||||||
@@ -33,25 +34,29 @@ export const Qwen25VLDeployment = () => {
|
|||||||
baseName: '72B',
|
baseName: '72B',
|
||||||
mi300x: { tp: 8, ep: 0 },
|
mi300x: { tp: 8, ep: 0 },
|
||||||
mi325x: { tp: 8, ep: 0 },
|
mi325x: { tp: 8, ep: 0 },
|
||||||
mi355x: { tp: 8, ep: 0 }
|
mi355x: { tp: 8, ep: 0 },
|
||||||
|
xeon: { tp: 6, ep: 0 }
|
||||||
},
|
},
|
||||||
'32b': {
|
'32b': {
|
||||||
baseName: '32B',
|
baseName: '32B',
|
||||||
mi300x: { tp: 2, ep: 0 },
|
mi300x: { tp: 2, ep: 0 },
|
||||||
mi325x: { tp: 2, ep: 0 },
|
mi325x: { tp: 2, ep: 0 },
|
||||||
mi355x: { tp: 2, ep: 0 }
|
mi355x: { tp: 2, ep: 0 },
|
||||||
|
xeon: { tp: 6, ep: 0 }
|
||||||
},
|
},
|
||||||
'7b': {
|
'7b': {
|
||||||
baseName: '7B',
|
baseName: '7B',
|
||||||
mi300x: { tp: 1, ep: 0 },
|
mi300x: { tp: 1, ep: 0 },
|
||||||
mi325x: { tp: 1, ep: 0 },
|
mi325x: { tp: 1, ep: 0 },
|
||||||
mi355x: { tp: 1, ep: 0 }
|
mi355x: { tp: 1, ep: 0 },
|
||||||
|
xeon: { tp: 3, ep: 0 }
|
||||||
},
|
},
|
||||||
'3b': {
|
'3b': {
|
||||||
baseName: '3B',
|
baseName: '3B',
|
||||||
mi300x: { tp: 1, ep: 0 },
|
mi300x: { tp: 1, ep: 0 },
|
||||||
mi325x: { tp: 1, ep: 0 },
|
mi325x: { tp: 1, ep: 0 },
|
||||||
mi355x: { tp: 1, ep: 0 }
|
mi355x: { tp: 1, ep: 0 },
|
||||||
|
xeon: { tp: 3, ep: 0 }
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -73,6 +78,10 @@ export const Qwen25VLDeployment = () => {
|
|||||||
let cmd = 'python -m sglang.launch_server \\\n';
|
let cmd = 'python -m sglang.launch_server \\\n';
|
||||||
cmd += ` --model ${modelName}`;
|
cmd += ` --model ${modelName}`;
|
||||||
|
|
||||||
|
if (hardware === 'xeon') {
|
||||||
|
cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule`;
|
||||||
|
}
|
||||||
|
|
||||||
if (hwConfig.tp > 1) {
|
if (hwConfig.tp > 1) {
|
||||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user