[Model] Complete dots.note.omni support with native encoders, video preprocessing, and MTP decoding (#33829)
Co-authored-by: miraclezqc <dysania@pku.edu.cn>
This commit is contained in:
Jianfei Wang
co-authored by
miraclezqc
miraclezqc
parent
c35683fda0
commit
af39ad9349
@@ -1,13 +1,9 @@
|
||||
// Dots3-Note cookbook config. Consumed by _deployment.jsx + _playground.jsx.
|
||||
// Single `export const config` literal - no spreads/calls/IIFE (Mintlify re-evals at hydration).
|
||||
// Single `export const config` literal — no spreads/calls/IIFE (Mintlify re-evals at hydration).
|
||||
|
||||
export const config = {
|
||||
modelName: "Dots3-Note",
|
||||
|
||||
// No Playground on this page — the only extra knob (the dots tool-call parser)
|
||||
// is already baked into the cells.
|
||||
showPlaygroundLink: false,
|
||||
|
||||
// Hopper only for now — no Blackwell support.
|
||||
supportedHardware: ["h200", "h100"],
|
||||
|
||||
@@ -50,14 +46,78 @@ export const config = {
|
||||
{"type": "video_url", "video_url": {"url": "https://example.com/sample.mp4"}},
|
||||
{"type": "text", "text": "Summarize what happens in this video."}
|
||||
]
|
||||
}]
|
||||
}],
|
||||
"video_config": {
|
||||
"seq": 131072,
|
||||
"audio_cap": 0.5,
|
||||
"audio_sr": 16000,
|
||||
"k_mode": "eval_ek"
|
||||
}
|
||||
}'`,
|
||||
|
||||
dockerImages: {
|
||||
h200: "lmsysorg/sglang:dev",
|
||||
h100: "lmsysorg/sglang:dev",
|
||||
h200: "lmsysorg/sglang:dev-dots3-note",
|
||||
h100: "lmsysorg/sglang:dev-dots3-note",
|
||||
},
|
||||
|
||||
github: {
|
||||
cookbookModel: "dots-studio/dots3-note-prev",
|
||||
},
|
||||
|
||||
playgroundFeatures: {
|
||||
attention: {
|
||||
knobs: [
|
||||
{ id: "tp", label: "TP", values: [null, 4, 8] },
|
||||
{
|
||||
id: "dpAttn",
|
||||
label: "DP-Attention",
|
||||
values: [null, false, 4, 8],
|
||||
labels: { "auto": "Auto", "false": "Off" },
|
||||
},
|
||||
],
|
||||
},
|
||||
moe: {
|
||||
backend: {
|
||||
options: [
|
||||
{ id: null, label: "Inherited" },
|
||||
{ id: "deepep", label: "DeepEP", flags: ["--moe-a2a-backend deepep"] },
|
||||
],
|
||||
},
|
||||
ep: { label: "EP", values: [null, 4, 8] },
|
||||
},
|
||||
parsers: {
|
||||
items: [
|
||||
{
|
||||
id: "reasoning",
|
||||
label: "Reasoning Parser",
|
||||
flag: "--reasoning-parser dots",
|
||||
},
|
||||
{
|
||||
id: "toolCall",
|
||||
label: "Tool Call Parser",
|
||||
flag: "--tool-call-parser dots",
|
||||
},
|
||||
],
|
||||
},
|
||||
speculative: {
|
||||
options: [
|
||||
{ id: "current", label: "Inherited from base" },
|
||||
{ id: "off", label: "Off (greedy)" },
|
||||
{
|
||||
id: "nextn-314",
|
||||
label: "NEXTN / MTP 3-1-4",
|
||||
flags: [
|
||||
"--speculative-algorithm NEXTN",
|
||||
"--speculative-num-steps 3",
|
||||
"--speculative-eagle-topk 1",
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
cells: [
|
||||
{
|
||||
@@ -74,20 +134,25 @@ export const config = {
|
||||
],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--context-length 524288",
|
||||
"--trust-remote-code",
|
||||
"--enable-dp-attention",
|
||||
"--dp-size 8",
|
||||
"--tp-size 8",
|
||||
"--ep-size 8",
|
||||
"--tp 8",
|
||||
"--dp 8",
|
||||
"--ep 8",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend deep_gemm",
|
||||
"--deepep-dispatcher-output-dtype bf16",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--context-length 524288",
|
||||
"--mem-fraction-static 0.87",
|
||||
"--max-running-requests 256",
|
||||
"--chunked-prefill-size 16384",
|
||||
"--trust-remote-code",
|
||||
"--swa-full-tokens-ratio 0.03",
|
||||
"--prefill-attention-backend fa3",
|
||||
"--decode-attention-backend fa3",
|
||||
"--page-size 64",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--cuda-graph-backend-decode full",
|
||||
"--cuda-graph-backend-prefill disabled",
|
||||
"--cuda-graph-max-bs-decode 32",
|
||||
@@ -97,15 +162,8 @@ export const config = {
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend deep_gemm",
|
||||
"--deepep-dispatcher-output-dtype bf16",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--enable-multimodal",
|
||||
"--enable-metrics",
|
||||
"--tool-call-parser dots",
|
||||
"--reasoning-parser qwen3",
|
||||
"--watchdog-timeout 1800",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
@@ -125,20 +183,25 @@ export const config = {
|
||||
],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--context-length 524288",
|
||||
"--trust-remote-code",
|
||||
"--enable-dp-attention",
|
||||
"--dp-size 8",
|
||||
"--tp-size 8",
|
||||
"--ep-size 8",
|
||||
"--tp 8",
|
||||
"--dp 8",
|
||||
"--ep 8",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend auto",
|
||||
"--deepep-dispatcher-output-dtype auto",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--context-length 524288",
|
||||
"--mem-fraction-static 0.87",
|
||||
"--max-running-requests 256",
|
||||
"--chunked-prefill-size 16384",
|
||||
"--trust-remote-code",
|
||||
"--swa-full-tokens-ratio 0.03",
|
||||
"--prefill-attention-backend fa3",
|
||||
"--decode-attention-backend fa3",
|
||||
"--page-size 64",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--cuda-graph-backend-decode full",
|
||||
"--cuda-graph-backend-prefill disabled",
|
||||
"--cuda-graph-max-bs-decode 32",
|
||||
@@ -148,15 +211,8 @@ export const config = {
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend auto",
|
||||
"--deepep-dispatcher-output-dtype auto",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--enable-multimodal",
|
||||
"--enable-metrics",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser dots",
|
||||
"--watchdog-timeout 1800",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
@@ -176,20 +232,25 @@ export const config = {
|
||||
],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--context-length 524288",
|
||||
"--trust-remote-code",
|
||||
"--enable-dp-attention",
|
||||
"--dp-size 8",
|
||||
"--tp-size 8",
|
||||
"--ep-size 8",
|
||||
"--tp 8",
|
||||
"--dp 8",
|
||||
"--ep 8",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend deep_gemm",
|
||||
"--deepep-dispatcher-output-dtype bf16",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--context-length 524288",
|
||||
"--mem-fraction-static 0.87",
|
||||
"--max-running-requests 256",
|
||||
"--chunked-prefill-size 16384",
|
||||
"--trust-remote-code",
|
||||
"--swa-full-tokens-ratio 0.03",
|
||||
"--prefill-attention-backend fa3",
|
||||
"--decode-attention-backend fa3",
|
||||
"--page-size 64",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--cuda-graph-backend-decode full",
|
||||
"--cuda-graph-backend-prefill disabled",
|
||||
"--cuda-graph-max-bs-decode 32",
|
||||
@@ -199,15 +260,8 @@ export const config = {
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend deep_gemm",
|
||||
"--deepep-dispatcher-output-dtype bf16",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--enable-multimodal",
|
||||
"--enable-metrics",
|
||||
"--tool-call-parser dots",
|
||||
"--reasoning-parser qwen3",
|
||||
"--watchdog-timeout 1800",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
@@ -227,20 +281,25 @@ export const config = {
|
||||
],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--context-length 524288",
|
||||
"--trust-remote-code",
|
||||
"--enable-dp-attention",
|
||||
"--dp-size 8",
|
||||
"--tp-size 8",
|
||||
"--ep-size 8",
|
||||
"--tp 8",
|
||||
"--dp 8",
|
||||
"--ep 8",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend auto",
|
||||
"--deepep-dispatcher-output-dtype auto",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--context-length 524288",
|
||||
"--mem-fraction-static 0.87",
|
||||
"--max-running-requests 256",
|
||||
"--chunked-prefill-size 16384",
|
||||
"--trust-remote-code",
|
||||
"--swa-full-tokens-ratio 0.03",
|
||||
"--prefill-attention-backend fa3",
|
||||
"--decode-attention-backend fa3",
|
||||
"--page-size 64",
|
||||
"--moe-dense-tp-size 1",
|
||||
"--cuda-graph-backend-decode full",
|
||||
"--cuda-graph-backend-prefill disabled",
|
||||
"--cuda-graph-max-bs-decode 32",
|
||||
@@ -250,15 +309,8 @@ export const config = {
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--speculative-draft-model-path {{MODEL_NAME}}",
|
||||
"--speculative-draft-attention-backend fa3",
|
||||
"--moe-a2a-backend deepep",
|
||||
"--moe-runner-backend auto",
|
||||
"--deepep-dispatcher-output-dtype auto",
|
||||
"--deepep-mode auto",
|
||||
"--enable-nccl-nvls",
|
||||
"--enable-multimodal",
|
||||
"--enable-metrics",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser dots",
|
||||
"--watchdog-timeout 1800",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
|
||||
Reference in New Issue
Block a user