diff --git a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl index e3a8d1634..dfb5e537a 100644 --- a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl +++ b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl @@ -104,7 +104,7 @@ export const config = { // in _deployment.jsx + the per-cell/defaultAccuracy keys. accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32`, diff --git a/.github/workflows/_pr-test-stage-cpu.yml b/.github/workflows/_pr-test-stage-cpu.yml index 10389ddac..1ebff6ea0 100644 --- a/.github/workflows/_pr-test-stage-cpu.yml +++ b/.github/workflows/_pr-test-stage-cpu.yml @@ -119,8 +119,6 @@ jobs: UV_SYSTEM_PYTHON: "1" run: | uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow - source scripts/ci/utils/sgl_eval_ref.sh - uv pip install "$SGL_EVAL_SPEC" --index-strategy unsafe-best-match # Hosted runners are ephemeral, so models are re-fetched every run and the # Hub occasionally returns 429s. Persist the HF cache in GitHub's cache diff --git a/docker/xeon.Dockerfile b/docker/xeon.Dockerfile index 2a45c17c5..08863e6d2 100644 --- a/docker/xeon.Dockerfile +++ b/docker/xeon.Dockerfile @@ -43,8 +43,7 @@ RUN source /opt/.venv/bin/activate && \ cd sglang/kernels/aot && \ cp pyproject_cpu.toml pyproject.toml && \ uv pip install . && \ - source /sgl-workspace/sglang/scripts/ci/utils/sgl_eval_ref.sh && \ - uv pip install "$SGL_EVAL_SPEC" + uv pip install "sgl-eval==0.1.0" ENV SGLANG_USE_CPU_ENGINE=1 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4:/usr/lib/x86_64-linux-gnu/libtbbmalloc.so:/opt/.venv/lib/libiomp5.so diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index ddd4baf70..9e85b4703 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -849,7 +849,7 @@ Results on AIME 2025 (8×B200), evaluated with [sgl-eval](https://github.com/sgl **Reproduction.** Install [sgl-eval](https://github.com/sgl-project/sgl-eval), launch the server with the tool-call and reasoning parsers, then run `sgl-eval run`. The AIME 2025 dataset ships with sgl-eval, and thinking is on by default for it: ```bash Command -pip install git+https://github.com/sgl-project/sgl-eval.git +pip install sgl-eval sgl-eval run aime25 \ --base-url http://localhost:30000/v1 \ diff --git a/docs/cookbook/autoregressive/Google/Gemma4.mdx b/docs/cookbook/autoregressive/Google/Gemma4.mdx index 946fd5dc0..a1e4ef921 100644 --- a/docs/cookbook/autoregressive/Google/Gemma4.mdx +++ b/docs/cookbook/autoregressive/Google/Gemma4.mdx @@ -1455,7 +1455,7 @@ Median ITL (ms): 15.11 Reproduce against a running server (`--base-url` points at your endpoint): ```bash Command -pip install git+https://github.com/sgl-project/sgl-eval +pip install sgl-eval # Sanity-check the endpoint sgl-eval ping --base-url http://localhost:30000/v1 diff --git a/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx b/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx index 7cbf6666f..97296c7cf 100644 --- a/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx +++ b/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.7.mdx @@ -497,7 +497,7 @@ This section uses **industry-standard configurations** for comparable benchmark - Prompt: `eval/aai/mcq-4choices` (4-choice multiple choice, matching [Artificial Analysis methodology](https://artificialanalysis.ai/methodology/intelligence-benchmarking)) -- sgl-eval's default for this benchmark - Evaluation command: ```bash Command -pip install git+https://github.com/sgl-project/sgl-eval.git +pip install sgl-eval sgl-eval run gpqa \ --base-url http://localhost:30000/v1 \ diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx index 368c014b8..696859a10 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx @@ -628,7 +628,7 @@ python tool_calls_eval.py tool-calls/samples.jsonl \ **Evaluation Command:** ```shell Command -pip install git+https://github.com/sgl-project/sgl-eval.git +pip install sgl-eval # The dataset ships with sgl-eval; --n-repeats runs all 32 samples per problem. sgl-eval run aime25 \ diff --git a/docs/src/snippets/configs/LiquidAI/lfm2.5.jsx b/docs/src/snippets/configs/LiquidAI/lfm2.5.jsx index 0c185e4da..367153e11 100644 --- a/docs/src/snippets/configs/LiquidAI/lfm2.5.jsx +++ b/docs/src/snippets/configs/LiquidAI/lfm2.5.jsx @@ -71,26 +71,26 @@ export const config = { --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`, accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} \\ --num-threads 128`, gpqa_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval # GPQA's HF dataset (Idavidrein/gpqa) is gated — accept its terms with your HF account first. sgl-eval run gpqa \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} \\ --num-threads 128`, mmlu_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run mmlu \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} \\ --num-threads 128`, aime25_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run aime25 \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} \\ diff --git a/docs/src/snippets/configs/MiniMaxAI/minimax-m3.jsx b/docs/src/snippets/configs/MiniMaxAI/minimax-m3.jsx index 397c18199..333c4ff1d 100644 --- a/docs/src/snippets/configs/MiniMaxAI/minimax-m3.jsx +++ b/docs/src/snippets/configs/MiniMaxAI/minimax-m3.jsx @@ -61,21 +61,21 @@ export const config = { --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`, accuracy: { gsm8k_pct: -`pip install git+https://github.com/sgl-project/sgl-eval +`pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} \\ --temperature 1.0 --top-p 0.95 \\ --thinking`, gpqa_pct: -`pip install git+https://github.com/sgl-project/sgl-eval +`pip install sgl-eval sgl-eval run gpqa \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} \\ --temperature 1.0 --top-p 0.95 \\ --thinking --n-repeats 4 --max-tokens 40960`, mmmu_pro_pct: -`pip install git+https://github.com/sgl-project/sgl-eval +`pip install sgl-eval sgl-eval run mmmu_pro \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} \\ diff --git a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index f5fb6a542..365b876b6 100644 --- a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -98,13 +98,13 @@ export const config = { --warmup-requests 64 --flush-cache`, accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32`, gpqa_pct: { flash: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gpqa \\ --model {{MODEL_NAME}} --api-key \\ --n-repeats 16 --max-tokens 200000 \\ @@ -112,7 +112,7 @@ sgl-eval run gpqa \\ --out-dir /sgl-workspace/logs \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`, "flash-official": -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gpqa \\ --model {{MODEL_NAME}} --api-key \\ --n-repeats 16 --max-tokens 200000 \\ @@ -120,7 +120,7 @@ sgl-eval run gpqa \\ --out-dir /sgl-workspace/logs \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`, pro: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gpqa \\ --model {{MODEL_NAME}} --api-key \\ --n-repeats 16 --max-tokens 400000 \\ @@ -130,7 +130,7 @@ sgl-eval run gpqa \\ }, aime25_pct: { "flash-official": -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run aime25 \\ --model {{MODEL_NAME}} --api-key \\ --n-repeats 16 --max-tokens 200000 \\ @@ -138,7 +138,7 @@ sgl-eval run aime25 \\ --out-dir /sgl-workspace/logs \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`, flash: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run aime25 \\ --model {{MODEL_NAME}} --api-key \\ --n-repeats 16 --max-tokens 200000 \\ @@ -146,7 +146,7 @@ sgl-eval run aime25 \\ --out-dir /sgl-workspace/logs \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`, pro: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run aime25 \\ --model {{MODEL_NAME}} --api-key \\ --n-repeats 16 --max-tokens 400000 \\ @@ -156,7 +156,7 @@ sgl-eval run aime25 \\ }, mmmu_pro_pct: { "flash-vision": -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run mmmu_pro \\ --reasoning-effort max \\ --temperature 1.0 --top-p 0.95 \\ diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx index 51aa8262d..abd21f443 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx @@ -63,7 +63,7 @@ export const config = { --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ --flush-cache`, accuracy: { - gsm8k_pct: `# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval + gsm8k_pct: `# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32`, diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-tiny.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-tiny.jsx index f2641791d..1e2aa415e 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-tiny.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-tiny.jsx @@ -57,7 +57,7 @@ export const config = { --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ --flush-cache`, accuracy: { - gsm8k_pct: `# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval + gsm8k_pct: `# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32 \\ diff --git a/docs/src/snippets/configs/internlm/intern-s2-mobius.jsx b/docs/src/snippets/configs/internlm/intern-s2-mobius.jsx index c053f6b74..215c6a1a4 100644 --- a/docs/src/snippets/configs/internlm/intern-s2-mobius.jsx +++ b/docs/src/snippets/configs/internlm/intern-s2-mobius.jsx @@ -51,12 +51,12 @@ export const config = { --warmup-requests 8 --flush-cache`, accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32`, gpqa_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gpqa \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 16`, diff --git a/docs/src/snippets/configs/meituan-longcat/longcat-2.0.jsx b/docs/src/snippets/configs/meituan-longcat/longcat-2.0.jsx index a641f5507..5122556b8 100644 --- a/docs/src/snippets/configs/meituan-longcat/longcat-2.0.jsx +++ b/docs/src/snippets/configs/meituan-longcat/longcat-2.0.jsx @@ -57,7 +57,7 @@ export const config = { --warmup-requests 64 --flush-cache`, accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32`, diff --git a/docs/src/snippets/configs/nvidia/nemotron-3.5-lightning.jsx b/docs/src/snippets/configs/nvidia/nemotron-3.5-lightning.jsx index d9a32b696..5c07b6587 100644 --- a/docs/src/snippets/configs/nvidia/nemotron-3.5-lightning.jsx +++ b/docs/src/snippets/configs/nvidia/nemotron-3.5-lightning.jsx @@ -51,7 +51,7 @@ export const config = { --flush-cache`, accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32`, diff --git a/docs/src/snippets/configs/poolside/laguna-m1.jsx b/docs/src/snippets/configs/poolside/laguna-m1.jsx index 99dc2b22e..8e2f3acb9 100644 --- a/docs/src/snippets/configs/poolside/laguna-m1.jsx +++ b/docs/src/snippets/configs/poolside/laguna-m1.jsx @@ -82,7 +82,7 @@ export const config = { // (AIME 25 to be added back once truncation-free numbers are measured.) accuracy: { gsm8k_pct: -`# pip install git+https://github.com/sgl-project/sgl-eval +`# pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 128`, diff --git a/docs/src/snippets/configs/poolside/laguna-s21.jsx b/docs/src/snippets/configs/poolside/laguna-s21.jsx index cdf9ac360..5adfb99c4 100644 --- a/docs/src/snippets/configs/poolside/laguna-s21.jsx +++ b/docs/src/snippets/configs/poolside/laguna-s21.jsx @@ -72,7 +72,7 @@ export const config = { --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`, accuracy: { gsm8k_pct: -`# pip install git+https://github.com/sgl-project/sgl-eval +`# pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 128`, @@ -80,7 +80,7 @@ sgl-eval run gsm8k \\ // Serve with a copy of the model's chat template whose enable_thinking default // is flipped to true. For BF16: use --max-tokens 131072 (see Configuration Tips). aime25_pct: -`# pip install git+https://github.com/sgl-project/sgl-eval +`# pip install sgl-eval # Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking). # For BF16: use --max-tokens 131072 (see Configuration Tips: BF16 reasoning length). sgl-eval run aime25 \\ diff --git a/docs/src/snippets/configs/poolside/laguna-xs21.jsx b/docs/src/snippets/configs/poolside/laguna-xs21.jsx index 8267b9d76..62573021a 100644 --- a/docs/src/snippets/configs/poolside/laguna-xs21.jsx +++ b/docs/src/snippets/configs/poolside/laguna-xs21.jsx @@ -112,7 +112,7 @@ export const config = { // GSM8K is the required accuracy sanity on every verified cell (cookbook_guide §3), via sgl-eval. accuracy: { gsm8k_pct: -`# pip install git+https://github.com/sgl-project/sgl-eval +`# pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 128`, @@ -121,7 +121,7 @@ sgl-eval run gsm8k \\ // AIME25 with thinking, serve with a copy of the model's chat template whose // enable_thinking default is flipped to true, passed via --chat-template. aime25_pct: -`# pip install git+https://github.com/sgl-project/sgl-eval +`# pip install sgl-eval # Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking). sgl-eval run aime25 \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ diff --git a/docs/src/snippets/configs/tencent/hy3.jsx b/docs/src/snippets/configs/tencent/hy3.jsx index c98e0cf1d..48a61afb5 100644 --- a/docs/src/snippets/configs/tencent/hy3.jsx +++ b/docs/src/snippets/configs/tencent/hy3.jsx @@ -62,12 +62,12 @@ export const config = { --warmup-requests 64`, accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32`, aime26_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run aime26 \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} --api-key \\ diff --git a/docs/src/snippets/configs/tencent/hy4-preview.jsx b/docs/src/snippets/configs/tencent/hy4-preview.jsx index cc2705732..4c4d2044f 100644 --- a/docs/src/snippets/configs/tencent/hy4-preview.jsx +++ b/docs/src/snippets/configs/tencent/hy4-preview.jsx @@ -72,7 +72,7 @@ export const config = { // GSM8K harness — keep these exact settings for comparability across runs. accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} \\ diff --git a/docs/src/snippets/configs/zai-org/glm-5.2.jsx b/docs/src/snippets/configs/zai-org/glm-5.2.jsx index d5ee97d6b..0f9606ce8 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.2.jsx @@ -63,12 +63,12 @@ export const config = { --warmup-requests 64 --flush-cache`, accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32`, aime25_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run aime25 \\ --model {{MODEL_NAME}} --api-key \\ --n-repeats 16 --max-tokens 64000 \\ diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx index 65845cca6..9b51a2c77 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx @@ -158,7 +158,7 @@ python3 -m sglang.bench_serving \\ numPromptsByConc: { 1: 16, 16: 80, 64: 320, 256: 1280, 1024: 5120 }, accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --model {{MODEL_NAME}} \\ diff --git a/docs/src/snippets/configs/zai-org/glm-5.3.jsx b/docs/src/snippets/configs/zai-org/glm-5.3.jsx index 16786241d..8b6ff04a0 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3.jsx @@ -61,12 +61,12 @@ export const config = { --warmup-requests 64 --flush-cache`, accuracy: { gsm8k_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 32`, aime26_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run aime26 \\ --model {{MODEL_NAME}} --api-key \\ --n-repeats 16 --max-tokens 64000 \\ @@ -74,7 +74,7 @@ sgl-eval run aime26 \\ --out-dir /sgl-workspace/logs \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`, aime25_pct: -`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +`# To install sgl-eval: pip install sgl-eval sgl-eval run aime25 \\ --model {{MODEL_NAME}} --api-key \\ --n-repeats 16 --max-tokens 64000 \\ diff --git a/python/pyproject.toml b/python/pyproject.toml index 6b6b0aa12..ac73d329f 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -172,11 +172,8 @@ test = [ "lm-eval[api]>=0.4.9.2", "matplotlib", # 4.9.3 is what sgl-eval's math_verify grader needs (latex2sympy2_extended - # raises ImportError on 4.7.x), so a `sglang[test]` environment is already - # compatible when sgl-eval is installed on top of it. - # Do NOT declare sgl-eval itself here: it is git-only, and PyPI rejects any - # uploaded distribution whose metadata carries a direct URL requirement. - # CUDA CI installs it in scripts/ci/cuda/ci_install_dependency.sh. + # raises ImportError on 4.7.x); pinned here so other test deps cannot pull + # it down. "antlr4-python3-runtime==4.9.3", "pandas", "parameterized", @@ -185,6 +182,10 @@ test = [ "pytest", "pytest-cov", "sentence_transformers", + # Pinned: a bump moves scoring for every eval test at once, so re-baseline + # MODEL_SCORE_THRESHOLDS in test/registered/eval/test_text_models_gsm8k_eval.py + # and the mmlu thresholds of run_eval's other callers before changing it. + "sgl-eval==0.1.0", "sglang[fastokens]", "tabulate", ] diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index 9cfbb2f21..571c89ef5 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -109,6 +109,7 @@ test = [ "pandas", "peft>=0.18.0", "sentence_transformers", + "sgl-eval==0.1.0", ] all = [] dev = ["sglang[test]"] diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index 5832cd90a..116a5e3e9 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -106,6 +106,7 @@ test = [ "peft>=0.18.0", "pytest", "sentence_transformers", + "sgl-eval==0.1.0", "tabulate", ] diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 91eac12b2..ac8106dfb 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -197,6 +197,7 @@ test = [ "peft>=0.18.0,<0.19.0", # Pin to <0.19.0 due to torchao incompatibility "pytest", "sentence_transformers", + "sgl-eval==0.1.0", "tabulate", ] diff --git a/python/sglang/test/kits/eval_accuracy_kit.py b/python/sglang/test/kits/eval_accuracy_kit.py index 399cea5f4..58b167200 100644 --- a/python/sglang/test/kits/eval_accuracy_kit.py +++ b/python/sglang/test/kits/eval_accuracy_kit.py @@ -111,7 +111,7 @@ def _run_sgl_eval( asserts the score meets ``score_threshold``, and checks the speculative accept length. ``thinking=True`` sends per-request ``chat_template_kwargs={"thinking": True}`` so the server separates reasoning from the final answer. Skips the test - if sgl-eval (git-only) is not installed. Returns the RunResult. + if sgl-eval is not installed. Returns the RunResult. """ assert ( score_threshold == score_threshold @@ -122,10 +122,7 @@ def _run_sgl_eval( from sgl_eval.sampler import ChatCompletionSampler from sgl_eval.types import GenConfig except ImportError: - test_case.skipTest( - "sgl-eval not installed; pip install " - "'sgl-eval @ git+https://github.com/sgl-project/sgl-eval'" - ) + test_case.skipTest("sgl-eval not installed; pip install 'sglang[test]'") base_url = test_case.base_url.rstrip("/") if not base_url.endswith("/v1"): diff --git a/scripts/ci/amd/amd_ci_install_dependency.sh b/scripts/ci/amd/amd_ci_install_dependency.sh index eec731e31..b65a5ffd1 100755 --- a/scripts/ci/amd/amd_ci_install_dependency.sh +++ b/scripts/ci/amd/amd_ci_install_dependency.sh @@ -187,10 +187,6 @@ else install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]" fi -# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh -source "$(dirname "${BASH_SOURCE[0]}")/../utils/sgl_eval_ref.sh" -install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache "$SGL_EVAL_SPEC" - if [[ -n "${SKIP_TT_DEPS}" ]]; then echo "Didn't build lmms_eval, human-eval, and others" else diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index de3dcd7b3..5c2d93e16 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -726,8 +726,6 @@ stabilize_flashinfer_jit_paths() { install_extra_deps() { MOONCAKE_VERSION="0.3.13" NIXL_VERSION="1.3.0" - # shellcheck source=scripts/ci/utils/sgl_eval_ref.sh - source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh" if [ "$CU_MAJOR" = "13" ]; then MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}" MOONCAKE_STALE_PKG="mooncake-transfer-engine" @@ -763,8 +761,6 @@ install_extra_deps() { --no-deps --force-reinstall $PIP_INSTALL_SUFFIX fi - $PIP_CMD install "$SGL_EVAL_SPEC" $PIP_INSTALL_SUFFIX - if [ "$IS_BLACKWELL" != "1" ]; then git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git $PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX diff --git a/scripts/ci/npu/npu_ci_install_dependency.sh b/scripts/ci/npu/npu_ci_install_dependency.sh index 89578cc77..192530517 100755 --- a/scripts/ci/npu/npu_ci_install_dependency.sh +++ b/scripts/ci/npu/npu_ci_install_dependency.sh @@ -79,8 +79,3 @@ rm -rf cann-custom-ops ### Install SGLang rm -rf python/pyproject.toml && mv python/pyproject_npu.toml python/pyproject.toml ${UV_PIP_INSTALL} -v -e "python[dev_npu]" - -### Install sgl-eval -# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh -source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh" -${UV_PIP_INSTALL} "$SGL_EVAL_SPEC" diff --git a/scripts/ci/utils/sgl_eval_ref.sh b/scripts/ci/utils/sgl_eval_ref.sh deleted file mode 100644 index 1e4044516..000000000 --- a/scripts/ci/utils/sgl_eval_ref.sh +++ /dev/null @@ -1,13 +0,0 @@ -# Single source of truth for the sgl-eval commit every CI variant installs. -# Meant to be sourced, not executed -- each variant then installs -# "$SGL_EVAL_SPEC" with its own pip invocation, since those differ (uv pip on -# CUDA/CPU, `docker exec ... pip` on AMD, `python3 -m pip` on NPU). -# -# sgl-eval is git-only and cannot be declared in python/pyproject.toml (see the -# note there). Every eval that shells out to the `sgl-eval` CLI fails without -# it, and a bump moves scoring for all of them at once -- so re-baseline -# MODEL_SCORE_THRESHOLDS in -# test/registered/eval/test_text_models_gsm8k_eval.py, and the mmlu thresholds -# of run_eval's other callers, before changing this. -SGL_EVAL_REF="a231b7a439b235090ff7baa30778fa2b514309ae" -SGL_EVAL_SPEC="sgl-eval@git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}"