diff --git a/docs/cookbook/autoregressive/IFM/K2-Horizon.mdx b/docs/cookbook/autoregressive/IFM/K2-Horizon.mdx index 5086df954..9d0b5de04 100644 --- a/docs/cookbook/autoregressive/IFM/K2-Horizon.mdx +++ b/docs/cookbook/autoregressive/IFM/K2-Horizon.mdx @@ -41,7 +41,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; ## 1. Model introduction -**K2 Horizon** is an IFM model family spanning six checkpoints from **0.9B** to **375B** parameters. The family includes dense models, Mixture-of-Experts models, and the 36B checkpoint's routed Mixture-of-Value-Attention path. This page provides native SGLang serving recipes for the family. +**K2 Horizon** is an IFM model family spanning six checkpoints from **0.9B** to **375B** parameters. The family includes dense models, Mixture-of-Experts models, and the MoVA-36B-A4B checkpoint's routed Mixture-of-Value-Attention path. This page provides native SGLang serving recipes for the family. @@ -84,13 +84,13 @@ import { Playground } from "/src/snippets/_playground.jsx"; - + - + @@ -102,21 +102,21 @@ import { Playground } from "/src/snippets/_playground.jsx"; All six checkpoints use the Apache License 2.0. -**Resources:** [K2 Horizon collection](https://huggingface.co/collections/IFM/k2-horizon) · [K2 Horizon 375B model card](https://huggingface.co/IFM/K2-Horizon-375B). +**Resources:** [K2 Horizon collection](https://huggingface.co/collections/IFM/k2-horizon) · [K2 Horizon 375B-A23B model card](https://huggingface.co/IFM/K2-Horizon-375B-A23B). ## 2. Configuration tips - **Hardware and precision:** The current matrix covers NVIDIA H200 GPUs and BF16 checkpoints only. -- **Parallelism:** Use TP1 for 0.9B, 3.7B, and 7B; TP2 for 32B and 36B; and TP8 for 375B. Each recipe stays within one node. +- **Parallelism:** Use TP1 for 0.9B, 3.7B, and 7B; TP2 for 32B and MoVA-36B-A4B; and TP8 for 375B-A23B. Each recipe stays within one node. - **Attention:** The verified base recipes select FlashAttention-3 with `--attention-backend fa3`. -- **Revisions:** The recipes pin revisions `9b9ec1f7e17f62ed218df542687a144116219d84` (0.9B), `c177771836a4c460743c00002c22483f6f18d1eb` (3.7B), `69ada542b68fe13d767479db2ab9421baff88681` (7B), `e1fd0277713e4eefcd3416348fd6fedacf7f2392` (32B), `16d20c739c687c08423422d1a2fbba6c529014cd` (36B), and `12812264242a14dce44aa7ae27f931ff4584bcbf` (375B). +- **Revisions:** The recipes pin revisions `9b9ec1f7e17f62ed218df542687a144116219d84` (0.9B), `c177771836a4c460743c00002c22483f6f18d1eb` (3.7B), `69ada542b68fe13d767479db2ab9421baff88681` (7B), `e1fd0277713e4eefcd3416348fd6fedacf7f2392` (32B), `16d20c739c687c08423422d1a2fbba6c529014cd` (MoVA-36B-A4B), and `12812264242a14dce44aa7ae27f931ff4584bcbf` (375B-A23B). - **Parsers:** The base recipes enable reasoning with `--reasoning-parser k2_horizon`. Enable `--tool-call-parser k2_horizon` from the **Parsers** card in the [Playground](#playground) when you need tool calling. -- **36B router provenance:** The 36B checkpoint's source xLLM router used two GEMM partitions. `--json-model-override-args '{"xllm_source_router_gemm_partitions":2}'` preserves its BF16 partial-GEMM rounding and FP32 reduction order; SGLang does not infer this source topology from runtime TP. -- **375B loading:** The 375B recipe disables multithreaded weight loading with `--model-loader-extra-config '{"enable_multithread_load":false}'`. +- **MoVA-36B-A4B router provenance:** The MoVA-36B-A4B checkpoint's source xLLM router used two GEMM partitions. `--json-model-override-args '{"xllm_source_router_gemm_partitions":2}'` preserves its BF16 partial-GEMM rounding and FP32 reduction order; SGLang does not infer this source topology from runtime TP. +- **375B-A23B loading:** The 375B-A23B recipe disables multithreaded weight loading with `--model-loader-extra-config '{"enable_multithread_load":false}'`. - **Playground features:** TP, EP, NGRAM, PD disaggregation, and HiCache are optional, unverified command overrides. The tool-call parser was functionally exercised separately. - **Deliberate omissions:** Context parallelism is unsupported or unvalidated for xLLM. DP-attention and alternate MoE backend selectors are not exposed until they are validated. - **Validation:** All six recipes completed native speed and full GSM8K evaluation on H200 using the K2 Horizon runtime support in [PR #37654](https://github.com/sgl-project/sglang/pull/37654). -- **375B provenance:** The 375B benchmark launched a directly mounted checkpoint and passed revision `12812264242a14dce44aa7ae27f931ff4584bcbf` to the server. The other five launches used materialized Hugging Face snapshots at their pinned revisions. +- **375B-A23B provenance:** The 375B-A23B benchmark launched a directly mounted checkpoint and passed revision `12812264242a14dce44aa7ae27f931ff4584bcbf` to the server. The other five launches used materialized Hugging Face snapshots at their pinned revisions. ## 3. Benchmark results @@ -130,8 +130,8 @@ GSM8K used the full 1,319-example test split with `sgl-eval`, 32 client threads, | 3.7B | 92.12% | 91.89% | 92.00% | | 7B | 95.30% | 94.47% | 94.88% | | 32B | 95.75% | 96.21% | 95.98% | -| 36B | 95.45% | 94.84% | 95.15% | -| 375B | 95.53% | 95.60% | 95.56% | +| MoVA-36B-A4B | 95.45% | 94.84% | 95.15% | +| 375B-A23B | 95.53% | 95.60% | 95.56% | ## 4. Advanced usage diff --git a/docs/src/snippets/configs/IFM/k2-horizon.jsx b/docs/src/snippets/configs/IFM/k2-horizon.jsx index c2016a296..fd5ed38cb 100644 --- a/docs/src/snippets/configs/IFM/k2-horizon.jsx +++ b/docs/src/snippets/configs/IFM/k2-horizon.jsx @@ -18,8 +18,8 @@ export const config = { { id: "3.7b", label: "3.7B", subtitle: "Dense" }, { id: "7b", label: "7B", subtitle: "Dense" }, { id: "32b", label: "32B", subtitle: "Dense" }, - { id: "36b", label: "36B", subtitle: "MoE + MoVA" }, - { id: "375b", label: "375B", subtitle: "MoE" }, + { id: "36b", label: "MoVA-36B-A4B", subtitle: "MoE + MoVA" }, + { id: "375b", label: "375B-A23B", subtitle: "MoE" }, ], quantizations: [ { id: "bf16", label: "BF16" }, @@ -36,8 +36,8 @@ export const config = { "3.7b|bf16": "IFM/K2-Horizon-3.7B", "7b|bf16": "IFM/K2-Horizon-7B", "32b|bf16": "IFM/K2-Horizon-32B", - "36b|bf16": "IFM/K2-Horizon-36B", - "375b|bf16": "IFM/K2-Horizon-375B", + "36b|bf16": "IFM/K2-Horizon-MoVA-36B-A4B", + "375b|bf16": "IFM/K2-Horizon-375B-A23B", }, placeholders: { @@ -85,7 +85,7 @@ sgl-eval run gsm8k \\ // The page covers a family, so use the largest checkpoint as the canonical // issue-form model while each recipe still resolves its exact HF repository. github: { - cookbookModel: "IFM/K2-Horizon-375B", + cookbookModel: "IFM/K2-Horizon-375B-A23B", }, playgroundFeatures: { @@ -99,18 +99,18 @@ sgl-eval run gsm8k \\ values: [ null, { value: 1, disable: { variant: ["375b"] }, - disableReason: "375B BF16 does not fit on one H200 at TP=1." }, + disableReason: "375B-A23B BF16 does not fit on one H200 at TP=1." }, { value: 2, disable: { variant: ["375b"] }, - disableReason: "375B BF16 does not fit on two H200 GPUs." }, + disableReason: "375B-A23B BF16 does not fit on two H200 GPUs." }, { value: 4, disable: { variant: ["375b"] }, - disableReason: "375B BF16 requires TP=8 to fit on an eight-H200 node." }, + disableReason: "375B-A23B BF16 requires TP=8 to fit on an eight-H200 node." }, 8, ], }, ], }, - // K2 Horizon 36B and 375B contain sparse MoE feed-forward layers. Keep + // K2 Horizon MoVA-36B-A4B and 375B-A23B contain sparse MoE feed-forward layers. Keep // expert parallelism disabled on the dense variants. moe: { ep: { @@ -119,29 +119,29 @@ sgl-eval run gsm8k \\ null, { value: 1, disable: { variant: ["0.9b", "3.7b", "7b", "32b"] }, - disableReason: "Expert parallelism applies only to the sparse 36B and 375B variants." }, + disableReason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." }, { value: 2, disable: [ { when: { variant: ["0.9b", "3.7b", "7b", "32b"] }, - reason: "Expert parallelism applies only to the sparse 36B and 375B variants." }, + reason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." }, { when: { effTp: [1] }, reason: "EP=2 requires an effective TP degree of at least 2." }, ] }, { value: 4, disable: [ { when: { variant: ["0.9b", "3.7b", "7b", "32b"] }, - reason: "Expert parallelism applies only to the sparse 36B and 375B variants." }, + reason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." }, { when: { effTp: [1, 2] }, reason: "EP=4 requires an effective TP degree of at least 4." }, ] }, { value: 8, disable: [ { when: { variant: ["0.9b", "3.7b", "7b", "32b"] }, - reason: "Expert parallelism applies only to the sparse 36B and 375B variants." }, + reason: "Expert parallelism applies only to the sparse MoVA-36B-A4B and 375B-A23B variants." }, { when: { effTp: [1, 2, 4] }, reason: "EP=8 requires an effective TP degree of at least 8." }, { when: { variant: ["36b"] }, - reason: "36B has 100 routed experts, which is not divisible by EP=8." }, + reason: "MoVA-36B-A4B has 100 routed experts, which is not divisible by EP=8." }, ] }, ], },
BF16 · TP2
36BMoVA-36B-A4B MoE + MoVA 524,288 BF16 · TP2
375B375B-A23B MoE 524,288 BF16 · TP8