minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[CI] Install sgl-eval from PyPI through the test extra (#37504)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-09-02 01:45:25 -07:00
committed by GitHub
parent fe3d4b9bbb
commit ebfd8c60e5
32 changed files with 52 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx
+1 -1
View File
@@ -849,7 +849,7 @@ Results on AIME 2025 (8×B200), evaluated with [sgl-eval](https://github.com/sgl
**Reproduction.** Install [sgl-eval](https://github.com/sgl-project/sgl-eval), launch the server with the tool-call and reasoning parsers, then run `sgl-eval run`. The AIME 2025 dataset ships with sgl-eval, and thinking is on by default for it:
```bash Command
pip install git+https://github.com/sgl-project/sgl-eval.git
pip install sgl-eval
sgl-eval run aime25 \
--base-url http://localhost:30000/v1 \
docs/cookbook/autoregressive/Google/Gemma4.mdx
+1 -1
View File
@@ -1455,7 +1455,7 @@ Median ITL (ms): 15.11
Reproduce against a running server (`--base-url` points at your endpoint):
```bash Command
pip install git+https://github.com/sgl-project/sgl-eval
pip install sgl-eval
# Sanity-check the endpoint
sgl-eval ping --base-url http://localhost:30000/v1
docs/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx
+1 -1
View File
@@ -497,7 +497,7 @@ This section uses **industry-standard configurations** for comparable benchmark
- Prompt: `eval/aai/mcq-4choices` (4-choice multiple choice, matching [Artificial Analysis methodology](https://artificialanalysis.ai/methodology/intelligence-benchmarking)) -- sgl-eval's default for this benchmark
- Evaluation command:
```bash Command
pip install git+https://github.com/sgl-project/sgl-eval.git
pip install sgl-eval
sgl-eval run gpqa \
--base-url http://localhost:30000/v1 \
docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx
+1 -1
View File
@@ -628,7 +628,7 @@ python tool_calls_eval.py tool-calls/samples.jsonl \
**Evaluation Command:**
```shell Command
pip install git+https://github.com/sgl-project/sgl-eval.git
pip install sgl-eval
# The dataset ships with sgl-eval; --n-repeats runs all 32 samples per problem.
sgl-eval run aime25 \
docs/src/snippets/configs/LiquidAI/lfm2.5.jsx
+4 -4
View File
@@ -71,26 +71,26 @@ export const config = {
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`,
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
--num-threads 128`,
gpqa_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
# GPQA's HF dataset (Idavidrein/gpqa) is gated — accept its terms with your HF account first.
sgl-eval run gpqa \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
--num-threads 128`,
mmlu_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run mmlu \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
--num-threads 128`,
aime25_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run aime25 \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
docs/src/snippets/configs/MiniMaxAI/minimax-m3.jsx
+3 -3
View File
@@ -61,21 +61,21 @@ export const config = {
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`,
accuracy: {
gsm8k_pct:
`pip install git+https://github.com/sgl-project/sgl-eval
`pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
--temperature 1.0 --top-p 0.95 \\
--thinking`,
gpqa_pct:
`pip install git+https://github.com/sgl-project/sgl-eval
`pip install sgl-eval
sgl-eval run gpqa \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
--temperature 1.0 --top-p 0.95 \\
--thinking --n-repeats 4 --max-tokens 40960`,
mmmu_pro_pct:
`pip install git+https://github.com/sgl-project/sgl-eval
`pip install sgl-eval
sgl-eval run mmmu_pro \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx
+8 -8
View File
@@ -98,13 +98,13 @@ export const config = {
--warmup-requests 64 --flush-cache`,
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
gpqa_pct: {
flash:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gpqa \\
--model {{MODEL_NAME}} --api-key <api-key> \\
--n-repeats 16 --max-tokens 200000 \\
@@ -112,7 +112,7 @@ sgl-eval run gpqa \\
--out-dir /sgl-workspace/logs \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`,
"flash-official":
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gpqa \\
--model {{MODEL_NAME}} --api-key <api-key> \\
--n-repeats 16 --max-tokens 200000 \\
@@ -120,7 +120,7 @@ sgl-eval run gpqa \\
--out-dir /sgl-workspace/logs \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`,
pro:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gpqa \\
--model {{MODEL_NAME}} --api-key <api-key> \\
--n-repeats 16 --max-tokens 400000 \\
@@ -130,7 +130,7 @@ sgl-eval run gpqa \\
},
aime25_pct: {
"flash-official":
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run aime25 \\
--model {{MODEL_NAME}} --api-key <api-key> \\
--n-repeats 16 --max-tokens 200000 \\
@@ -138,7 +138,7 @@ sgl-eval run aime25 \\
--out-dir /sgl-workspace/logs \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`,
flash:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run aime25 \\
--model {{MODEL_NAME}} --api-key <api-key> \\
--n-repeats 16 --max-tokens 200000 \\
@@ -146,7 +146,7 @@ sgl-eval run aime25 \\
--out-dir /sgl-workspace/logs \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`,
pro:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run aime25 \\
--model {{MODEL_NAME}} --api-key <api-key> \\
--n-repeats 16 --max-tokens 400000 \\
@@ -156,7 +156,7 @@ sgl-eval run aime25 \\
},
mmmu_pro_pct: {
"flash-vision":
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run mmmu_pro \\
--reasoning-effort max \\
--temperature 1.0 --top-p 0.95 \\
docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx
+1 -1
View File
@@ -63,7 +63,7 @@ export const config = {
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
--flush-cache`,
accuracy: {
gsm8k_pct: `# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
gsm8k_pct: `# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
docs/src/snippets/configs/inclusionAI/ling-3.0-tiny.jsx
+1 -1
View File
@@ -57,7 +57,7 @@ export const config = {
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
--flush-cache`,
accuracy: {
gsm8k_pct: `# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
gsm8k_pct: `# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32 \\
docs/src/snippets/configs/internlm/intern-s2-mobius.jsx
+2 -2
View File
@@ -51,12 +51,12 @@ export const config = {
--warmup-requests 8 --flush-cache`,
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
gpqa_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gpqa \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 16`,
docs/src/snippets/configs/meituan-longcat/longcat-2.0.jsx
+1 -1
View File
@@ -57,7 +57,7 @@ export const config = {
--warmup-requests 64 --flush-cache`,
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
docs/src/snippets/configs/nvidia/nemotron-3.5-lightning.jsx
+1 -1
View File
@@ -51,7 +51,7 @@ export const config = {
--flush-cache`,
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
docs/src/snippets/configs/poolside/laguna-m1.jsx
+1 -1
View File
@@ -82,7 +82,7 @@ export const config = {
// (AIME 25 to be added back once truncation-free numbers are measured.)
accuracy: {
gsm8k_pct:
`# pip install git+https://github.com/sgl-project/sgl-eval
`# pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 128`,
docs/src/snippets/configs/poolside/laguna-s21.jsx
+2 -2
View File
@@ -72,7 +72,7 @@ export const config = {
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`,
accuracy: {
gsm8k_pct:
`# pip install git+https://github.com/sgl-project/sgl-eval
`# pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 128`,
@@ -80,7 +80,7 @@ sgl-eval run gsm8k \\
// Serve with a copy of the model's chat template whose enable_thinking default
// is flipped to true. For BF16: use --max-tokens 131072 (see Configuration Tips).
aime25_pct:
`# pip install git+https://github.com/sgl-project/sgl-eval
`# pip install sgl-eval
# Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking).
# For BF16: use --max-tokens 131072 (see Configuration Tips: BF16 reasoning length).
sgl-eval run aime25 \\
docs/src/snippets/configs/poolside/laguna-xs21.jsx
+2 -2
View File
@@ -112,7 +112,7 @@ export const config = {
// GSM8K is the required accuracy sanity on every verified cell (cookbook_guide §3), via sgl-eval.
accuracy: {
gsm8k_pct:
`# pip install git+https://github.com/sgl-project/sgl-eval
`# pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 128`,
@@ -121,7 +121,7 @@ sgl-eval run gsm8k \\
// AIME25 with thinking, serve with a copy of the model's chat template whose
// enable_thinking default is flipped to true, passed via --chat-template.
aime25_pct:
`# pip install git+https://github.com/sgl-project/sgl-eval
`# pip install sgl-eval
# Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking).
sgl-eval run aime25 \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
docs/src/snippets/configs/tencent/hy3.jsx
+2 -2
View File
@@ -62,12 +62,12 @@ export const config = {
--warmup-requests 64`,
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
aime26_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run aime26 \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} --api-key <api-key> \\
docs/src/snippets/configs/tencent/hy4-preview.jsx
+1 -1
View File
@@ -72,7 +72,7 @@ export const config = {
// GSM8K harness — keep these exact settings for comparability across runs.
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
docs/src/snippets/configs/zai-org/glm-5.2.jsx
+2 -2
View File
@@ -63,12 +63,12 @@ export const config = {
--warmup-requests 64 --flush-cache`,
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
aime25_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run aime25 \\
--model {{MODEL_NAME}} --api-key <api-key> \\
--n-repeats 16 --max-tokens 64000 \\
docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx
+1 -1
View File
@@ -158,7 +158,7 @@ python3 -m sglang.bench_serving \\
numPromptsByConc: { 1: 16, 16: 80, 64: 320, 256: 1280, 1024: 5120 },
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
docs/src/snippets/configs/zai-org/glm-5.3.jsx
+3 -3
View File
@@ -61,12 +61,12 @@ export const config = {
--warmup-requests 64 --flush-cache`,
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
aime26_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run aime26 \\
--model {{MODEL_NAME}} --api-key <api-key> \\
--n-repeats 16 --max-tokens 64000 \\
@@ -74,7 +74,7 @@ sgl-eval run aime26 \\
--out-dir /sgl-workspace/logs \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`,
aime25_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
`# To install sgl-eval: pip install sgl-eval
sgl-eval run aime25 \\
--model {{MODEL_NAME}} --api-key <api-key> \\
--n-repeats 16 --max-tokens 64000 \\