diff --git a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx index 65ba1fea6..91ca0b36f 100644 --- a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx @@ -42,7 +42,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; Ling-3.0-flash is a hybrid-attention Mixture-of-Experts (MoE) language model from the BailingMoeV3 family. It interleaves Kimi Delta Attention (KDA) linear-attention layers with gated Multi-head Latent Attention (MLA) full-attention layers, on top of a fine-grained MoE feed-forward network. This keeps per-token inference cost close to a small model — **124B total parameters with only 5.1B active** — while retaining large-model capacity. -It is a hybrid-reasoning model with thinking enabled by default, and it supports structured tool calling. Native context length is 128K, extendable to 256K with YaRN. +It is a hybrid-reasoning model with thinking enabled by default, and it supports structured tool calling. Native context length is 256K. **Available Models:** @@ -61,7 +61,7 @@ It is a hybrid-reasoning model with thinking enabled by default, and it supports - Both the chat template and the `ling3` reasoning parser default to thinking on. A single request can turn it off with `"chat_template_kwargs": {"enable_thinking": false}` (see §3.1). - `--mem-fraction-static 0.8` reserves headroom for CUDA graphs and concurrent decoding; with the default allocation the NEXTN recipes can OOM under concurrent requests (e.g. a 32-thread GSM8K run). - The checkpoint ships a built-in MTP layer (`num_nextn_predict_layers: 1`); enable it with `--speculative-algorithm NEXTN` — no separate draft model is needed. The Low-Latency recipes have it on; toggle it in the **Speculative Decoding** card of the [Playground](#playground). -- Native context is 128K. The recipes set `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1` to acknowledge the longer context explicitly, then use `--context-length 262144` and YaRN with factor 2.0 to extend it to 256K. +- Native context is 256K; SGLang reads it from the checkpoint's `max_position_embeddings`, so no `--context-length` flag is needed. - The **HiCache** card in the [Playground](#playground) exposes the validated Mooncake L3 path. It adds the hybrid-KDA scheduler and prefix-key settings together; see §3.3 for the required services. ## 3. Advanced Usage diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx index ff1392bd9..e330de05f 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx @@ -125,13 +125,10 @@ sgl-eval run gsm8k \\ { match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -140,13 +137,10 @@ sgl-eval run gsm8k \\ { match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -155,13 +149,10 @@ sgl-eval run gsm8k \\ { match: { hw: "h800", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -170,13 +161,10 @@ sgl-eval run gsm8k \\ { match: { hw: "h100", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -185,13 +173,10 @@ sgl-eval run gsm8k \\ { match: { hw: "b200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -200,13 +185,10 @@ sgl-eval run gsm8k \\ { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -215,14 +197,11 @@ sgl-eval run gsm8k \\ { match: { hw: "h20-3e", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -231,14 +210,11 @@ sgl-eval run gsm8k \\ { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -247,14 +223,11 @@ sgl-eval run gsm8k \\ { match: { hw: "h800", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", "--ep-size 8", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -263,14 +236,11 @@ sgl-eval run gsm8k \\ { match: { hw: "h100", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", "--ep-size 8", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -279,14 +249,11 @@ sgl-eval run gsm8k \\ { match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -295,14 +262,11 @@ sgl-eval run gsm8k \\ { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -311,12 +275,9 @@ sgl-eval run gsm8k \\ { match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -325,12 +286,9 @@ sgl-eval run gsm8k \\ { match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -339,12 +297,9 @@ sgl-eval run gsm8k \\ { match: { hw: "h800", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -353,12 +308,9 @@ sgl-eval run gsm8k \\ { match: { hw: "h100", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -367,12 +319,9 @@ sgl-eval run gsm8k \\ { match: { hw: "b200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -381,12 +330,9 @@ sgl-eval run gsm8k \\ { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -395,13 +341,10 @@ sgl-eval run gsm8k \\ { match: { hw: "h20-3e", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -410,13 +353,10 @@ sgl-eval run gsm8k \\ { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -425,13 +365,10 @@ sgl-eval run gsm8k \\ { match: { hw: "h800", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", "--ep-size 8", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -440,13 +377,10 @@ sgl-eval run gsm8k \\ { match: { hw: "h100", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: false, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 8", "--ep-size 8", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -455,13 +389,10 @@ sgl-eval run gsm8k \\ { match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -470,13 +401,10 @@ sgl-eval run gsm8k \\ { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, verified: true, - env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -489,7 +417,6 @@ sgl-eval run gsm8k \\ match: { hw: "h200", variant: "default", quant: "bf16", strategy: "hicache", nodes: "single" }, verified: true, env: [ - "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1", "MOONCAKE_MASTER={{MOONCAKE_MASTER}}", "MOONCAKE_PROTOCOL=tcp", "MC_MS_AUTO_DISC=0", @@ -500,9 +427,7 @@ sgl-eval run gsm8k \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--enable-hierarchical-cache", "--hicache-storage-backend mooncake", @@ -520,7 +445,6 @@ sgl-eval run gsm8k \\ match: { hw: "h200", variant: "default", quant: "fp8", strategy: "hicache", nodes: "single" }, verified: false, env: [ - "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1", "MOONCAKE_MASTER={{MOONCAKE_MASTER}}", "MOONCAKE_PROTOCOL=tcp", "MC_MS_AUTO_DISC=0", @@ -532,9 +456,7 @@ sgl-eval run gsm8k \\ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--enable-hierarchical-cache", "--hicache-storage-backend mooncake", @@ -552,7 +474,6 @@ sgl-eval run gsm8k \\ match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "hicache", nodes: "single" }, verified: false, env: [ - "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1", "MOONCAKE_MASTER={{MOONCAKE_MASTER}}", "MOONCAKE_PROTOCOL=tcp", "MC_MS_AUTO_DISC=0", @@ -563,9 +484,7 @@ sgl-eval run gsm8k \\ flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--enable-hierarchical-cache", "--hicache-storage-backend mooncake", @@ -583,7 +502,6 @@ sgl-eval run gsm8k \\ match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "hicache", nodes: "single" }, verified: false, env: [ - "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1", "MOONCAKE_MASTER={{MOONCAKE_MASTER}}", "MOONCAKE_PROTOCOL=tcp", "MC_MS_AUTO_DISC=0", @@ -595,9 +513,7 @@ sgl-eval run gsm8k \\ "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--context-length 262144", "--speculative-algorithm NEXTN", - "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'", "--mem-fraction-static 0.8", "--enable-hierarchical-cache", "--hicache-storage-backend mooncake",