[Model] Complete dots.note.omni support with native encoders, video preprocessing, and MTP decoding (#33829)
Co-authored-by: miraclezqc <dysania@pku.edu.cn>
This commit is contained in:
Jianfei Wang
co-authored by
miraclezqc
miraclezqc
parent
c35683fda0
commit
af39ad9349
@@ -57,6 +57,14 @@ import { config } from "/src/snippets/configs/rednote/dots3-note.jsx";
|
||||
|
||||
<Deployment config={config} />
|
||||
|
||||
## Playground
|
||||
|
||||
The Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations signed off on this page; the Playground lets you turn on additional knobs on top of whichever cell the Deploy panel is currently showing.
|
||||
|
||||
import { Playground } from "/src/snippets/_playground.jsx";
|
||||
|
||||
<Playground config={config} />
|
||||
|
||||
## 1. Model Introduction
|
||||
|
||||
dots3.note is RedNote's native multimodal omni model, built on the dots3 language model. It accepts text, image, audio, and native video input.
|
||||
@@ -66,7 +74,12 @@ dots3.note is RedNote's native multimodal omni model, built on the dots3 languag
|
||||
- **Hybrid attention** — dots3 combines MLA with full-attention and sliding-window layers of different geometry, attention gates, and optional DSA indexing on full-attention layers.
|
||||
- **MTP speculative decoding** — a full-sharing MTP/NextN architecture exposes one recursively shared, SWA-shaped MTP layer and shares the target LM head.
|
||||
|
||||
**Resources:** [Hugging Face](https://huggingface.co/dots-studio/dots3-note-prev) · [SGLang PR #33829](https://github.com/sgl-project/sglang/pull/33829)
|
||||
**Available checkpoints:**
|
||||
|
||||
- **BF16**: [dots-studio/dots3-note-prev](https://huggingface.co/dots-studio/dots3-note-prev)
|
||||
- **FP8**: [dots-studio/dots3-note-prev-fp8](https://huggingface.co/dots-studio/dots3-note-prev-fp8)
|
||||
|
||||
**Resources:** [Hugging Face (BF16)](https://huggingface.co/dots-studio/dots3-note-prev) · [Hugging Face (FP8)](https://huggingface.co/dots-studio/dots3-note-prev-fp8) · [SGLang PR #33829](https://github.com/sgl-project/sglang/pull/33829)
|
||||
|
||||
## 2. Configuration Tips
|
||||
|
||||
@@ -112,10 +125,12 @@ response = client.chat.completions.create(
|
||||
}
|
||||
],
|
||||
extra_body={
|
||||
"seq": 131072,
|
||||
"audio_cap": 0.5,
|
||||
"audio_sr": 16000,
|
||||
"k_mode": "eval_ek",
|
||||
"video_config": {
|
||||
"seq": 131072,
|
||||
"audio_cap": 0.5,
|
||||
"audio_sr": 16000,
|
||||
"k_mode": "eval_ek",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -132,28 +147,93 @@ Pending update...
|
||||
|
||||
</Accordion>
|
||||
|
||||
Per-request video preprocessing controls (all optional, passed via `extra_body`):
|
||||
Per-request video preprocessing controls are grouped under `video_config` in
|
||||
`extra_body`:
|
||||
|
||||
| Field | Default | Purpose |
|
||||
|-------|---------|---------|
|
||||
| `seq` | `131072` | Total sequence budget used by the video flattener. |
|
||||
| `audio_cap` | `1.0` | Maximum fraction of the input budget assigned to audio; `0` disables audio processing. |
|
||||
| `audio_sr` | `16000` | Audio sample rate. |
|
||||
| `k_mode` | `eval_ek` | Deterministic evaluation/sampling mode of the flattener. |
|
||||
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
|
||||
<colgroup>
|
||||
<col style={{width: "22%"}} />
|
||||
<col style={{width: "18%"}} />
|
||||
<col style={{width: "60%"}} />
|
||||
</colgroup>
|
||||
<thead>
|
||||
<tr style={{borderBottom: "2px solid #d55816"}}>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, backgroundColor: "rgba(255,255,255,0.02)"}}>Field</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, backgroundColor: "rgba(255,255,255,0.05)"}}>Default</th>
|
||||
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, backgroundColor: "rgba(255,255,255,0.02)"}}>Purpose</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>seq</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>131072</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Total sequence budget used by the video flattener.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>audio_cap</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>1.0</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Maximum fraction of the input budget assigned to audio; <code>0</code> disables audio processing.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>audio_sr</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>16000</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Audio sample rate.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>k_mode</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>eval_ek</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Deterministic evaluation/sampling mode of the flattener.</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
These controls are request-scoped so that evaluation jobs with different context budgets can share one server. The flattener reserves room for `max_new_tokens` inside the budget and falls back to visual-only processing if audio would exceed the configured token budget.
|
||||
These controls are request-scoped so that evaluation jobs with different context budgets can share one server. For example: `extra_body={"video_config": {"seq": 131072, "audio_cap": 0.5}}`. The flattener reserves room for `max_new_tokens` inside the budget and falls back to visual-only processing if audio would exceed the configured token budget.
|
||||
|
||||
<Warning>
|
||||
Native video currently supports one video per request, and a native video cannot be mixed with separate image or audio inputs in the same request.
|
||||
</Warning>
|
||||
A request may carry several videos, and videos can be mixed with image and audio parts. Each video is flattened independently under the same per-request budget, and the flattened frames and audio segments are spliced back at the position of their `video_url` part, so the modality ordering of the prompt is preserved.
|
||||
|
||||
### 3.2 Image and audio input
|
||||
|
||||
Outside the native-video path, images and audio clips use the standard OpenAI multimodal message format and SGLang's multimodal serving (`--enable-multimodal` is in every cell). The vision and audio towers run in-process, so no extra server is needed.
|
||||
Outside the native-video path, images and audio clips use the standard OpenAI multimodal message format. `--enable-multimodal` is in every cell; the vision and audio towers run in-process, so no extra server is needed.
|
||||
|
||||
<Accordion title="Image Example (Python)">
|
||||
|
||||
```python Example
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="dots3.note",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/sample.jpg"},
|
||||
},
|
||||
{"type": "text", "text": "Describe this image."},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Example Output">
|
||||
|
||||
```text Output
|
||||
Pending update...
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
### 3.3 Tool Calling
|
||||
|
||||
The cells launch with `--tool-call-parser dots`, so structured tool calls surface via `message.tool_calls` out of the box.
|
||||
Toggle **Tool Call Parser** (`--tool-call-parser dots`) and **Reasoning Parser** (`--reasoning-parser dots`) in the **Parsers** card of the [Playground above](#playground). Structured tool calls then surface via `message.tool_calls`.
|
||||
|
||||
<Accordion title="Tool Calling Example (Python)">
|
||||
|
||||
@@ -195,7 +275,7 @@ Pending update...
|
||||
|
||||
### 3.4 Encoder/LLM Disaggregation (EPD)
|
||||
|
||||
`Dot3NoteForCausalLM` supports both roles of an encoder/LLM-disaggregated deployment:
|
||||
`Dots3NoteForCausalLM` supports both roles of an encoder/LLM-disaggregated deployment:
|
||||
|
||||
- **Encoder role** — serve with `--encoder-only`; the instance runs only the vision and audio towers.
|
||||
- **Language role** — serve with `--language-only`; the instance skips tower construction, leaving the memory to the language model.
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
// Dots3-Note cookbook config. Consumed by _deployment.jsx + _playground.jsx.
|
||||
// Single `export const config` literal - no spreads/calls/IIFE (Mintlify re-evals at hydration).
|
||||
// Single `export const config` literal — no spreads/calls/IIFE (Mintlify re-evals at hydration).
|
||||
|
||||
export const config = {
|
||||
modelName: "Dots3-Note",
|
||||
|
||||
// No Playground on this page — the only extra knob (the dots tool-call parser)
|
||||
// is already baked into the cells.
|
||||
showPlaygroundLink: false,
|
||||
|
||||
// Hopper only for now — no Blackwell support.
|
||||
supportedHardware: ["h200", "h100"],
|
||||
|
||||
@@ -50,14 +46,78 @@ export const config = {
|
||||
{"type": "video_url", "video_url": {"url": "https://example.com/sample.mp4"}},
|
||||
{"type": "text", "text": "Summarize what happens in this video."}
|
||||
]
|
||||
}]
|
||||
}],
|
||||
"video_config": {
|
||||
"seq": 131072,
|
||||
"audio_cap": 0.5,
|
||||
"audio_sr": 16000,
|
||||
"k_mode": "eval_ek"
|
||||
}
|
||||
}'`,
|
||||
|
||||
dockerImages: {
|
||||
h200: "lmsysorg/sglang:dev",
|
||||
h100: "lmsysorg/sglang:dev",
|
||||
h200: "lmsysorg/sglang:dev-dots3-note",
|
||||
h100: "lmsysorg/sglang:dev-dots3-note",
|
||||
},
|
||||
|
||||
github: {
|
||||
cookbookModel: "dots-studio/dots3-note-prev",
|
||||
},
|
||||
|
||||
playgroundFeatures: {
|
||||
attention: {
|
||||
knobs: [
|
||||
{ id: "tp", label: "TP", values: [null, 4, 8] },
|
||||
{
|
||||
id: "dpAttn",
|
||||
label: "DP-Attention",
|
||||
values: [null, false, 4, 8],
|
||||
labels: { "auto": "Auto", "false": "Off" },
|
||||
},
|
||||
],
|
||||
},
|
||||
moe: {
|
||||
backend: {
|
||||
options: [
|
||||
{ id: null, label: "Inherited" },
|
||||
{ id: "deepep", label: "DeepEP", flags: ["--moe-a2a-backend deepep"] },
|
||||
],
|
||||
},
|
||||
ep: { label: "EP", values: [null, 4, 8] },
|
||||
},
|
||||
parsers: {
|
||||
items: [
|
||||
{
|
||||
id: "reasoning",
|
||||
label: "Reasoning Parser",
|
||||
flag: "--reasoning-parser dots",
|
||||
},
|
||||
{
|
||||
id: "toolCall",
|
||||
label: "Tool Call Parser",
|
||||
flag: "--tool-call-parser dots",
|
||||
},
|
||||
],
|
||||
},
|
||||
speculative: {
|
||||
options: [
|
||||
{ id: "current", label: "Inherited from base" },
|
||||
{ id: "off", label: "Off (greedy)" },
|
||||
{
|
||||
id: "nextn-314",
|
||||
label: "NEXTN / MTP 3-1-4",
|
||||
flags: [
|
||||
"--speculative-algorithm NEXTN",
|
||||
"--speculative-num-steps 3",
|
||||
"--speculative-eagle-topk 1",
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
cells: [
|
||||
{
|
||||
@@ -74,20 +134,25 @@ export const config = {
|
||||
],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--context-length 524288",
|
||||
"--trust-remote-code",
|
||||
"--enable-dp-attention",
|
||||
"--dp-size 8",
|
||||
"--tp-size 8",
|
||||
"--ep-size 8",
|
||||
"--tp 8",
|
||||
"--dp 8",
|
||||
"--ep 8",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend deep_gemm",
|
||||
"--deepep-dispatcher-output-dtype bf16",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--context-length 524288",
|
||||
"--mem-fraction-static 0.87",
|
||||
"--max-running-requests 256",
|
||||
"--chunked-prefill-size 16384",
|
||||
"--trust-remote-code",
|
||||
"--swa-full-tokens-ratio 0.03",
|
||||
"--prefill-attention-backend fa3",
|
||||
"--decode-attention-backend fa3",
|
||||
"--page-size 64",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--cuda-graph-backend-decode full",
|
||||
"--cuda-graph-backend-prefill disabled",
|
||||
"--cuda-graph-max-bs-decode 32",
|
||||
@@ -97,15 +162,8 @@ export const config = {
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend deep_gemm",
|
||||
"--deepep-dispatcher-output-dtype bf16",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--enable-multimodal",
|
||||
"--enable-metrics",
|
||||
"--tool-call-parser dots",
|
||||
"--reasoning-parser qwen3",
|
||||
"--watchdog-timeout 1800",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
@@ -125,20 +183,25 @@ export const config = {
|
||||
],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--context-length 524288",
|
||||
"--trust-remote-code",
|
||||
"--enable-dp-attention",
|
||||
"--dp-size 8",
|
||||
"--tp-size 8",
|
||||
"--ep-size 8",
|
||||
"--tp 8",
|
||||
"--dp 8",
|
||||
"--ep 8",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend auto",
|
||||
"--deepep-dispatcher-output-dtype auto",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--context-length 524288",
|
||||
"--mem-fraction-static 0.87",
|
||||
"--max-running-requests 256",
|
||||
"--chunked-prefill-size 16384",
|
||||
"--trust-remote-code",
|
||||
"--swa-full-tokens-ratio 0.03",
|
||||
"--prefill-attention-backend fa3",
|
||||
"--decode-attention-backend fa3",
|
||||
"--page-size 64",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--cuda-graph-backend-decode full",
|
||||
"--cuda-graph-backend-prefill disabled",
|
||||
"--cuda-graph-max-bs-decode 32",
|
||||
@@ -148,15 +211,8 @@ export const config = {
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend auto",
|
||||
"--deepep-dispatcher-output-dtype auto",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--enable-multimodal",
|
||||
"--enable-metrics",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser dots",
|
||||
"--watchdog-timeout 1800",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
@@ -176,20 +232,25 @@ export const config = {
|
||||
],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--context-length 524288",
|
||||
"--trust-remote-code",
|
||||
"--enable-dp-attention",
|
||||
"--dp-size 8",
|
||||
"--tp-size 8",
|
||||
"--ep-size 8",
|
||||
"--tp 8",
|
||||
"--dp 8",
|
||||
"--ep 8",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend deep_gemm",
|
||||
"--deepep-dispatcher-output-dtype bf16",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--context-length 524288",
|
||||
"--mem-fraction-static 0.87",
|
||||
"--max-running-requests 256",
|
||||
"--chunked-prefill-size 16384",
|
||||
"--trust-remote-code",
|
||||
"--swa-full-tokens-ratio 0.03",
|
||||
"--prefill-attention-backend fa3",
|
||||
"--decode-attention-backend fa3",
|
||||
"--page-size 64",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--cuda-graph-backend-decode full",
|
||||
"--cuda-graph-backend-prefill disabled",
|
||||
"--cuda-graph-max-bs-decode 32",
|
||||
@@ -199,15 +260,8 @@ export const config = {
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend deep_gemm",
|
||||
"--deepep-dispatcher-output-dtype bf16",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--enable-multimodal",
|
||||
"--enable-metrics",
|
||||
"--tool-call-parser dots",
|
||||
"--reasoning-parser qwen3",
|
||||
"--watchdog-timeout 1800",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
@@ -227,20 +281,25 @@ export const config = {
|
||||
],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--context-length 524288",
|
||||
"--trust-remote-code",
|
||||
"--enable-dp-attention",
|
||||
"--dp-size 8",
|
||||
"--tp-size 8",
|
||||
"--ep-size 8",
|
||||
"--tp 8",
|
||||
"--dp 8",
|
||||
"--ep 8",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend auto",
|
||||
"--deepep-dispatcher-output-dtype auto",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--context-length 524288",
|
||||
"--mem-fraction-static 0.87",
|
||||
"--max-running-requests 256",
|
||||
"--chunked-prefill-size 16384",
|
||||
"--trust-remote-code",
|
||||
"--swa-full-tokens-ratio 0.03",
|
||||
"--prefill-attention-backend fa3",
|
||||
"--decode-attention-backend fa3",
|
||||
"--page-size 64",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--cuda-graph-backend-decode full",
|
||||
"--cuda-graph-backend-prefill disabled",
|
||||
"--cuda-graph-max-bs-decode 32",
|
||||
@@ -250,15 +309,8 @@ export const config = {
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend auto",
|
||||
"--deepep-dispatcher-output-dtype auto",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--enable-multimodal",
|
||||
"--enable-metrics",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser dots",
|
||||
"--watchdog-timeout 1800",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
|
||||
Reference in New Issue
Block a user