docs: add K2 Horizon cookbook recipes and H200 results (#37655)
This commit is contained in:
Yash Akhauri
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
@@ -0,0 +1,254 @@
|
||||
---
|
||||
title: K2 Horizon
|
||||
description: "Deploy the IFM K2 Horizon family with SGLang — BF16 recipes for six dense and Mixture-of-Experts checkpoints on NVIDIA H200 GPUs."
|
||||
tag: NEW
|
||||
---
|
||||
|
||||
## Deployment
|
||||
|
||||
<a id="install" />
|
||||
|
||||
<Accordion title="Install SGLang">
|
||||
|
||||
Install from a source checkout that includes the K2 Horizon runtime support in [PR #37654](https://github.com/sgl-project/sglang/pull/37654) before using the generated commands:
|
||||
|
||||
```bash Command
|
||||
pip install --upgrade pip
|
||||
pip install -e "python"
|
||||
```
|
||||
|
||||
For general source-install guidance, see [Install → Method 2: From source](/docs/get-started/install#method-2-from-source).
|
||||
|
||||
</Accordion>
|
||||
|
||||
Choose a model size to generate its validated single-node H200 launch command. The serving settings in all six base recipes completed two independent H200 benchmark launches at the pinned model revisions using the K2 Horizon runtime support in [PR #37654](https://github.com/sgl-project/sglang/pull/37654).
|
||||
|
||||
import { Deployment } from "/src/snippets/_deployment.jsx";
|
||||
import { config } from "/src/snippets/configs/IFM/k2-horizon.jsx";
|
||||
import { benchmarks } from "/src/snippets/configs/IFM/k2-horizon-benchmarks.jsx";
|
||||
|
||||
<Deployment config={config} benchmarks={benchmarks} />
|
||||
|
||||
Each speed and accuracy value in the benchmark cards is the arithmetic mean of two independent server launches.
|
||||
|
||||
## Playground
|
||||
|
||||
The base recipes enable the `k2_horizon` reasoning parser. Use the Playground to configure the `k2_horizon` tool-call parser, TP and MoE expert-parallel overrides, NGRAM speculative decoding, PD disaggregation, and HiCache. A separate validation pass functionally exercised the tool-call parser on all six checkpoints; every other Playground override remains unverified.
|
||||
|
||||
import { Playground } from "/src/snippets/_playground.jsx";
|
||||
|
||||
<Playground config={config} />
|
||||
|
||||
## 1. Model introduction
|
||||
|
||||
**K2 Horizon** is an IFM model family spanning six checkpoints from **0.9B** to **375B** parameters. The family includes dense models, Mixture-of-Experts models, and the 36B checkpoint's routed Mixture-of-Value-Attention path. This page provides native SGLang serving recipes for the family.
|
||||
|
||||
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
|
||||
<colgroup>
|
||||
<col style={{width: "22%"}} />
|
||||
<col style={{width: "25%"}} />
|
||||
<col style={{width: "23%"}} />
|
||||
<col style={{width: "30%"}} />
|
||||
</colgroup>
|
||||
<thead>
|
||||
<tr style={{borderBottom: "2px solid #d55816"}}>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Checkpoint</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Architecture</th>
|
||||
<th style={{textAlign: "right", padding: "10px 12px", fontWeight: 700}}>Context length</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>H200 recipe</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/K2-Horizon-0.9B">0.9B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>Dense</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>131,072</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP1</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/K2-Horizon-3.7B">3.7B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>Dense</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP1</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/K2-Horizon-7B">7B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>Dense</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP1</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/K2-Horizon-32B">32B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>Dense</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP2</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/K2-Horizon-36B">36B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>MoE + MoVA</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP2</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/IFM/K2-Horizon-375B">375B</a></strong></td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>MoE</td>
|
||||
<td style={{padding: "9px 12px", textAlign: "right"}}>524,288</td>
|
||||
<td style={{padding: "9px 12px", background: "rgba(255,255,255,0.02)"}}>BF16 · TP8</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<Note>
|
||||
All six checkpoints use the Apache License 2.0.
|
||||
</Note>
|
||||
|
||||
**Resources:** [K2 Horizon collection](https://huggingface.co/collections/IFM/k2-horizon) · [K2 Horizon 375B model card](https://huggingface.co/IFM/K2-Horizon-375B).
|
||||
|
||||
## 2. Configuration tips
|
||||
|
||||
- **Hardware and precision:** The current matrix covers NVIDIA H200 GPUs and BF16 checkpoints only.
|
||||
- **Parallelism:** Use TP1 for 0.9B, 3.7B, and 7B; TP2 for 32B and 36B; and TP8 for 375B. Each recipe stays within one node.
|
||||
- **Attention:** The verified base recipes select FlashAttention-3 with `--attention-backend fa3`.
|
||||
- **Revisions:** The recipes pin revisions `9b9ec1f7e17f62ed218df542687a144116219d84` (0.9B), `c177771836a4c460743c00002c22483f6f18d1eb` (3.7B), `69ada542b68fe13d767479db2ab9421baff88681` (7B), `e1fd0277713e4eefcd3416348fd6fedacf7f2392` (32B), `16d20c739c687c08423422d1a2fbba6c529014cd` (36B), and `12812264242a14dce44aa7ae27f931ff4584bcbf` (375B).
|
||||
- **Parsers:** The base recipes enable reasoning with `--reasoning-parser k2_horizon`. Enable `--tool-call-parser k2_horizon` from the **Parsers** card in the [Playground](#playground) when you need tool calling.
|
||||
- **36B router provenance:** The 36B checkpoint's source xLLM router used two GEMM partitions. `--json-model-override-args '{"xllm_source_router_gemm_partitions":2}'` preserves its BF16 partial-GEMM rounding and FP32 reduction order; SGLang does not infer this source topology from runtime TP.
|
||||
- **375B loading:** The 375B recipe disables multithreaded weight loading with `--model-loader-extra-config '{"enable_multithread_load":false}'`.
|
||||
- **Playground features:** TP, EP, NGRAM, PD disaggregation, and HiCache are optional, unverified command overrides. The tool-call parser was functionally exercised separately.
|
||||
- **Deliberate omissions:** Context parallelism is unsupported or unvalidated for xLLM. DP-attention and alternate MoE backend selectors are not exposed until they are validated.
|
||||
- **Validation:** All six recipes completed native speed and full GSM8K evaluation on H200 using the K2 Horizon runtime support in [PR #37654](https://github.com/sgl-project/sglang/pull/37654).
|
||||
- **375B provenance:** The 375B benchmark launched a directly mounted checkpoint and passed revision `12812264242a14dce44aa7ae27f931ff4584bcbf` to the server. The other five launches used materialized Hugging Face snapshots at their pinned revisions.
|
||||
|
||||
## 3. Benchmark results
|
||||
|
||||
Speed was measured with native `sglang.benchmark.serving` using fixed random token IDs: 8,192 input tokens, 1,024 output tokens, 64 warmup requests, a cache flush, and seed `20260901`. The latency run used 32 prompts at concurrency 1. The throughput run used 256 prompts at concurrency 64 for TP1, 32 for TP2, and 8 for TP8. TTFT and TPOT are medians within each launch; the benchmark cards show the arithmetic mean across the two launches. Throughput per GPU is `(total input tokens + total output tokens) / duration / TP`.
|
||||
|
||||
GSM8K used the full 1,319-example test split with `sgl-eval`, 32 client threads, a 32,768-token output limit, temperature 0, top-p 0.95, seed 0, and high reasoning effort. The mean and both independent server-launch scores are shown below. All requests completed without errors. Responses that reached the output limit remain in the score: truncation was 6.90% in both 0.9B launches, 4.09% and 4.70% for 3.7B, and at most 1.14% for the other checkpoints.
|
||||
|
||||
| Checkpoint | Launch 1 | Launch 2 | Mean |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| 0.9B | 85.22% | 85.29% | 85.25% |
|
||||
| 3.7B | 92.12% | 91.89% | 92.00% |
|
||||
| 7B | 95.30% | 94.47% | 94.88% |
|
||||
| 32B | 95.75% | 96.21% | 95.98% |
|
||||
| 36B | 95.45% | 94.84% | 95.15% |
|
||||
| 375B | 95.53% | 95.60% | 95.56% |
|
||||
|
||||
## 4. Advanced usage
|
||||
|
||||
### 4.1 Reasoning
|
||||
|
||||
The base recipes include `--reasoning-parser k2_horizon`. The parser returns thinking in `reasoning_content` and the final answer in `content`.
|
||||
|
||||
The example output below was captured with the 3.7B checkpoint and both parser flags enabled.
|
||||
|
||||
<Accordion title="Reasoning example (Python)">
|
||||
|
||||
```python Example
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")
|
||||
response = client.chat.completions.create(
|
||||
model="IFM/K2-Horizon-3.7B",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
)
|
||||
|
||||
message = response.choices[0].message
|
||||
reasoning = getattr(message, "reasoning_content", None)
|
||||
if reasoning is None:
|
||||
reasoning = (message.model_extra or {}).get("reasoning_content")
|
||||
print("Reasoning:", reasoning)
|
||||
print("Answer:", message.content)
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Example Output">
|
||||
|
||||
```text Output
|
||||
Reasoning: We have a user who just said "Hello". We need to respond. As an AI, we should be friendly, perhaps ask how can we help. If says "Hello" alone, respond with greeting and offer assistance. Potentially ask what they need. There's no special instruction; just typical.
|
||||
|
||||
But maybe we must think about the context: The user gave just a greeting; we should respond with a greeting and ask how we can help. Could also ask if they want anything specific.
|
||||
|
||||
We should be concise but friendly: "Hello! How can I assist you today?" Provide some content.
|
||||
|
||||
But maybe the user expects something else. But in this scenario, we just reply with a greeting.
|
||||
|
||||
Thus answer: "Hello! What can I do for you?" Etc.
|
||||
|
||||
Check any style guidelines: We want to keep it short or perhaps ask follow-up. The user didn't ask any question, just saying hello. Could also respond: "Hello! ..."
|
||||
|
||||
Thus arguably the answer: "Hi there! How can I help you today?" Let's do that.
|
||||
|
||||
But also maybe we could ask if they need help with something specific. That's appropriate.
|
||||
|
||||
Thus the final answer: "Hello! How can I assist you today?" Possibly add friendly banter.
|
||||
|
||||
We'll respond.
|
||||
|
||||
Answer:
|
||||
Hello! 👋 How can I help you today?
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
### 4.2 Tool calling
|
||||
|
||||
Enable **Tool Call Parser** in the **Parsers** card of the [Playground](#playground) to add `--tool-call-parser k2_horizon`. The parser exposes K2 tool output as OpenAI-compatible `message.tool_calls`.
|
||||
|
||||
The example output below was captured with the 32B checkpoint and both parser flags enabled.
|
||||
|
||||
<Accordion title="Tool-calling example (Python)">
|
||||
|
||||
```python Example
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Return weather for a city.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="IFM/K2-Horizon-32B",
|
||||
messages=[{"role": "user", "content": "Call get_weather for Paris."}],
|
||||
tools=tools,
|
||||
tool_choice={"type": "function", "function": {"name": "get_weather"}},
|
||||
max_tokens=1024,
|
||||
)
|
||||
|
||||
message = response.choices[0].message
|
||||
reasoning = getattr(message, "reasoning_content", None)
|
||||
if reasoning is None:
|
||||
reasoning = (message.model_extra or {}).get("reasoning_content")
|
||||
print("Reasoning:", reasoning)
|
||||
print("Content:", message.content or "")
|
||||
for tool_call in message.tool_calls or []:
|
||||
print(f"Tool call: {tool_call.function.name}({tool_call.function.arguments})")
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Example Output">
|
||||
|
||||
```text Output
|
||||
Reasoning: Okay, the user wants me to call the get_weather function for Paris. Let me check the tools available. There's a function called get_weather that takes a city parameter. The required parameter is city, and it's a string. So I need to make sure to pass "Paris" as the city. I don't see any other parameters needed. The user didn't mention any other details, so I'll just use the city name. Let me structure the tool call correctly. The function name is get_weather, and the arguments should be a JSON object with "city": "Paris". I need to make sure the JSON is properly formatted. Alright, that should do it.
|
||||
|
||||
Content:
|
||||
Tool call: get_weather({"city": "Paris"})
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
### 4.3 HiCache
|
||||
|
||||
For large checkpoints or repeated long-context prefixes, enable **HiCache** in the [Playground](#playground), then choose a storage backend and write policy. Treat the generated override as unverified until it is exercised on your storage stack.
|
||||
@@ -7,6 +7,12 @@ metatags:
|
||||
---
|
||||
|
||||
<CardGroup cols={3}>
|
||||
<Card
|
||||
title="IFM"
|
||||
mode="card"
|
||||
href="/cookbook/autoregressive/IFM/K2-Horizon"
|
||||
img="/cards/logos/ifm.png"
|
||||
/>
|
||||
<Card
|
||||
title="Kimi (Moonshot AI)"
|
||||
mode="card"
|
||||
|
||||
@@ -1208,6 +1208,12 @@
|
||||
"group": "Autoregressive Models",
|
||||
"pages": [
|
||||
"cookbook/autoregressive/intro",
|
||||
{
|
||||
"group": "IFM",
|
||||
"pages": [
|
||||
"cookbook/autoregressive/IFM/K2-Horizon"
|
||||
]
|
||||
},
|
||||
{
|
||||
"group": "Kimi (Moonshot AI)",
|
||||
"pages": [
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
// Direct NVIDIA H200 measurements. Each displayed value is the mean of two
|
||||
// independent server launches; methodology and commands are in the cookbook.
|
||||
|
||||
export const benchmarks = [
|
||||
{
|
||||
match: { hw: "h200", variant: "0.9b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "PR #37654",
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 47.55,
|
||||
tpot_ms: 1.74,
|
||||
tokens_per_sec_per_gpu: 5025.24,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 1418.67,
|
||||
tpot_ms: 11.18,
|
||||
tokens_per_sec_per_gpu: 45773.21,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 85.25 },
|
||||
notes: "Speed and GSM8K are the mean of two independent server launches. GSM8K truncation was 6.90% in both launches.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "3.7b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "PR #37654",
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 158.71,
|
||||
tpot_ms: 5.10,
|
||||
tokens_per_sec_per_gpu: 1712.15,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 5174.71,
|
||||
tpot_ms: 28.83,
|
||||
tokens_per_sec_per_gpu: 16998.94,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 92.00 },
|
||||
notes: "Speed and GSM8K are the mean of two independent server launches. GSM8K truncation was 4.09% and 4.70%.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "7b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "PR #37654",
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 249.31,
|
||||
tpot_ms: 6.74,
|
||||
tokens_per_sec_per_gpu: 1289.51,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 8175.68,
|
||||
tpot_ms: 33.81,
|
||||
tokens_per_sec_per_gpu: 13784.03,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 94.88 },
|
||||
notes: "Speed and GSM8K are the mean of two independent server launches. GSM8K truncation was 0.91% and 1.14%.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "32b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "PR #37654",
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 590.81,
|
||||
tpot_ms: 13.18,
|
||||
tokens_per_sec_per_gpu: 327.51,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 32, num_prompts: 256 },
|
||||
ttft_ms: 10370.53,
|
||||
tpot_ms: 30.79,
|
||||
tokens_per_sec_per_gpu: 3503.00,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 95.98 },
|
||||
notes: "Speed and GSM8K are the mean of two independent server launches. GSM8K truncation was 0.45% and 0.53%.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "36b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "PR #37654",
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 218.54,
|
||||
tpot_ms: 10.77,
|
||||
tokens_per_sec_per_gpu: 410.17,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 32, num_prompts: 256 },
|
||||
ttft_ms: 3639.21,
|
||||
tpot_ms: 27.67,
|
||||
tokens_per_sec_per_gpu: 4613.54,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 95.15 },
|
||||
notes: "Speed and GSM8K are the mean of two independent server launches. GSM8K truncation was 0.83% and 0.99%.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "375b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "PR #37654",
|
||||
speed: [
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 32 },
|
||||
ttft_ms: 299.65,
|
||||
tpot_ms: 9.65,
|
||||
tokens_per_sec_per_gpu: 113.25,
|
||||
},
|
||||
{
|
||||
workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 8, num_prompts: 256 },
|
||||
ttft_ms: 1408.10,
|
||||
tpot_ms: 15.68,
|
||||
tokens_per_sec_per_gpu: 524.00,
|
||||
},
|
||||
],
|
||||
accuracy: { gsm8k_pct: 95.56 },
|
||||
notes: "Speed and GSM8K are the mean of two independent server launches. GSM8K truncation was 0.45% and 0.38%.",
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,303 @@
|
||||
// Single `export const config` literal — Mintlify re-evaluates this module at
|
||||
// hydration time, so keep the cookbook data self-contained.
|
||||
//
|
||||
// The serving settings in the six base recipes below were benchmarked with the
|
||||
// K2 Horizon runtime support in sgl-project/sglang#37654 and the pinned model
|
||||
// revisions. Playground overrides remain separate from the verified commands.
|
||||
|
||||
export const config = {
|
||||
modelName: "K2 Horizon",
|
||||
|
||||
latencyPercentile: "P50",
|
||||
|
||||
supportedHardware: ["h200"],
|
||||
runModes: ["python"],
|
||||
|
||||
variants: [
|
||||
{ id: "0.9b", label: "0.9B", subtitle: "Dense" },
|
||||
{ id: "3.7b", label: "3.7B", subtitle: "Dense" },
|
||||
{ id: "7b", label: "7B", subtitle: "Dense" },
|
||||
{ id: "32b", label: "32B", subtitle: "Dense" },
|
||||
{ id: "36b", label: "36B", subtitle: "MoE + MoVA" },
|
||||
{ id: "375b", label: "375B", subtitle: "MoE" },
|
||||
],
|
||||
quantizations: [
|
||||
{ id: "bf16", label: "BF16" },
|
||||
],
|
||||
strategies: [
|
||||
{ id: "balanced", label: "Balanced" },
|
||||
],
|
||||
nodesOptions: [
|
||||
{ id: "single", label: "Single Node" },
|
||||
],
|
||||
|
||||
modelNames: {
|
||||
"0.9b|bf16": "IFM/K2-Horizon-0.9B",
|
||||
"3.7b|bf16": "IFM/K2-Horizon-3.7B",
|
||||
"7b|bf16": "IFM/K2-Horizon-7B",
|
||||
"32b|bf16": "IFM/K2-Horizon-32B",
|
||||
"36b|bf16": "IFM/K2-Horizon-36B",
|
||||
"375b|bf16": "IFM/K2-Horizon-375B",
|
||||
},
|
||||
|
||||
placeholders: {
|
||||
HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" },
|
||||
PORT: { target: "command", label: "Bind port", default: "30000" },
|
||||
CURL_HOST: { target: "curl", label: "Server host", default: "localhost" },
|
||||
CURL_PORT: { target: "curl", label: "Server port", default: "30000" },
|
||||
},
|
||||
|
||||
curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\
|
||||
-H 'Content-Type: application/json' \\
|
||||
-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`,
|
||||
|
||||
benchmarkCommands: {
|
||||
speed:
|
||||
`python3 -m sglang.benchmark.serving \\
|
||||
--backend sglang \\
|
||||
--base-url http://{{CURL_HOST}}:{{CURL_PORT}} \\
|
||||
--model {{MODEL_NAME}} --served-model-name {{MODEL_NAME}} \\
|
||||
--tokenizer {{MODEL_NAME}} \\
|
||||
--dataset-name {{DATASET}} --tokenize-prompt \\
|
||||
--random-input-len {{ISL}} --random-output-len {{OSL}} \\
|
||||
--random-range-ratio 1.0 \\
|
||||
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
|
||||
--request-rate inf --seed 20260901 \\
|
||||
--temperature 0.0 --top-p 1.0 \\
|
||||
--warmup-requests 64 --flush-cache \\
|
||||
--output-file benchmark.raw.jsonl --output-details`,
|
||||
accuracy: {
|
||||
gsm8k_pct:
|
||||
`pip install sgl-eval
|
||||
sgl-eval run gsm8k \\
|
||||
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
|
||||
--model {{MODEL_NAME}} \\
|
||||
--num-examples 1319 --num-threads 32 --n-repeats 1 \\
|
||||
--max-tokens 32768 --temperature 0.0 --top-p 0.95 \\
|
||||
--seed 0 --reasoning-effort high --prompt math`,
|
||||
},
|
||||
},
|
||||
|
||||
accuracyLabels: [
|
||||
["gsm8k_pct", "GSM8K", "%"],
|
||||
],
|
||||
|
||||
// The page covers a family, so use the largest checkpoint as the canonical
|
||||
// issue-form model while each recipe still resolves its exact HF repository.
|
||||
github: {
|
||||
cookbookModel: "IFM/K2-Horizon-375B",
|
||||
},
|
||||
|
||||
playgroundFeatures: {
|
||||
// CP is unsupported/unvalidated for the xLLM path. DP-attention and
|
||||
// alternate MoE backends are also intentionally omitted until validated.
|
||||
attention: {
|
||||
knobs: [
|
||||
{
|
||||
id: "tp",
|
||||
label: "TP",
|
||||
values: [
|
||||
null,
|
||||
{ value: 1, disable: { variant: ["375b"] },
|
||||
disableReason: "375B BF16 does not fit on one H200 at TP=1." },
|
||||
{ value: 2, disable: { variant: ["375b"] },
|
||||
disableReason: "375B BF16 does not fit on two H200 GPUs." },
|
||||
{ value: 4, disable: { variant: ["375b"] },
|
||||
disableReason: "375B BF16 requires TP=8 to fit on an eight-H200 node." },
|
||||
8,
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
// K2 Horizon 36B and 375B contain sparse MoE feed-forward layers. Keep
|
||||
// expert parallelism disabled on the dense variants.
|
||||
moe: {
|
||||
ep: {
|
||||
label: "EP",
|
||||
values: [
|
||||
null,
|
||||
{ value: 1,
|
||||
disable: { variant: ["0.9b", "3.7b", "7b", "32b"] },
|
||||
disableReason: "Expert parallelism applies only to the sparse 36B and 375B variants." },
|
||||
{ value: 2,
|
||||
disable: [
|
||||
{ when: { variant: ["0.9b", "3.7b", "7b", "32b"] },
|
||||
reason: "Expert parallelism applies only to the sparse 36B and 375B variants." },
|
||||
{ when: { effTp: [1] },
|
||||
reason: "EP=2 requires an effective TP degree of at least 2." },
|
||||
] },
|
||||
{ value: 4,
|
||||
disable: [
|
||||
{ when: { variant: ["0.9b", "3.7b", "7b", "32b"] },
|
||||
reason: "Expert parallelism applies only to the sparse 36B and 375B variants." },
|
||||
{ when: { effTp: [1, 2] },
|
||||
reason: "EP=4 requires an effective TP degree of at least 4." },
|
||||
] },
|
||||
{ value: 8,
|
||||
disable: [
|
||||
{ when: { variant: ["0.9b", "3.7b", "7b", "32b"] },
|
||||
reason: "Expert parallelism applies only to the sparse 36B and 375B variants." },
|
||||
{ when: { effTp: [1, 2, 4] },
|
||||
reason: "EP=8 requires an effective TP degree of at least 8." },
|
||||
{ when: { variant: ["36b"] },
|
||||
reason: "36B has 100 routed experts, which is not divisible by EP=8." },
|
||||
] },
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
parsers: {
|
||||
items: [
|
||||
{ id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser k2_horizon" },
|
||||
{ id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser k2_horizon" },
|
||||
],
|
||||
},
|
||||
|
||||
speculative: {
|
||||
options: [
|
||||
{ id: "current", label: "Inherited from base" },
|
||||
{ id: "off", label: "Off" },
|
||||
{ id: "ngram", label: "NGRAM",
|
||||
flags: ["--speculative-algorithm NGRAM",
|
||||
"--speculative-num-draft-tokens 16",
|
||||
"--speculative-ngram-max-bfs-breadth 10"] },
|
||||
],
|
||||
},
|
||||
|
||||
pdDisagg: {
|
||||
modes: [
|
||||
{ id: "off", label: "Off" },
|
||||
{ id: "prefill", label: "Prefill role" },
|
||||
{ id: "decode", label: "Decode role" },
|
||||
],
|
||||
transferBackends: [
|
||||
{ id: "mooncake", label: "Mooncake" },
|
||||
{ id: "nixl", label: "NiXL" },
|
||||
],
|
||||
ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"],
|
||||
router: {
|
||||
port: 8000,
|
||||
command:
|
||||
`python3 -m sglang_router.launch_router \\
|
||||
--pd-disaggregation \\
|
||||
--prefill http://<prefill-host>:{{PREFILL_PORT}} \\
|
||||
--decode http://<decode-host>:{{DECODE_PORT}} \\
|
||||
--host 0.0.0.0 --port {{ROUTER_PORT}} \\
|
||||
--disable-circuit-breaker \\
|
||||
--health-check-interval-secs 999999`,
|
||||
},
|
||||
},
|
||||
|
||||
hicache: {
|
||||
backends: [
|
||||
{ id: null, label: "Auto" },
|
||||
{ id: "file", label: "File" },
|
||||
{ id: "mooncake", label: "Mooncake" },
|
||||
{ id: "hf3fs", label: "HF3FS" },
|
||||
{ id: "nixl", label: "NiXL" },
|
||||
],
|
||||
writePolicies: [
|
||||
{ id: "auto", label: "Auto" },
|
||||
{ id: "write_through", label: "Write-through" },
|
||||
{ id: "write_back", label: "Write-back" },
|
||||
{ id: "write_through_selective", label: "Write-through (selective)" },
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
cells: [
|
||||
{
|
||||
match: { hw: "h200", variant: "0.9b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--revision 9b9ec1f7e17f62ed218df542687a144116219d84",
|
||||
"--tp 1",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "3.7b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--revision c177771836a4c460743c00002c22483f6f18d1eb",
|
||||
"--tp 1",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "7b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--revision 69ada542b68fe13d767479db2ab9421baff88681",
|
||||
"--tp 1",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "32b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--revision e1fd0277713e4eefcd3416348fd6fedacf7f2392",
|
||||
"--tp 2",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "36b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--revision 16d20c739c687c08423422d1a2fbba6c529014cd",
|
||||
"--tp 2",
|
||||
"--dtype bfloat16",
|
||||
"--json-model-override-args '{\"xllm_source_router_gemm_partitions\":2}'",
|
||||
"--attention-backend fa3",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "375b", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--revision 12812264242a14dce44aa7ae27f931ff4584bcbf",
|
||||
"--tp 8",
|
||||
"--dtype bfloat16",
|
||||
"--attention-backend fa3",
|
||||
"--model-loader-extra-config '{\"enable_multithread_load\":false}'",
|
||||
"--reasoning-parser k2_horizon",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
Reference in New Issue
Block a user