[CI] Route mmlu and GB300 MMMU-Pro evals through sgl-eval (#34477)

This commit is contained in:
Liangsheng Yin
2026-08-12 19:33:21 -07:00
committed by GitHub
parent cfdfd31826
commit 50cc1aa241
25 changed files with 182 additions and 503 deletions
@@ -134,6 +134,10 @@ else
install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]"
fi
# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
source "$(dirname "${BASH_SOURCE[0]}")/../utils/sgl_eval_ref.sh"
install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache "$SGL_EVAL_SPEC"
if [[ -n "${SKIP_TT_DEPS}" ]]; then
echo "Didn't build lmms_eval, human-eval, and others"
else
+3 -7
View File
@@ -682,12 +682,8 @@ stabilize_flashinfer_jit_paths() {
install_extra_deps() {
MOONCAKE_VERSION="0.3.12.post1"
NIXL_VERSION="1.3.0"
# sgl-eval is git-only and cannot be declared in python/pyproject.toml (see
# the note there). The nightly GSM8K eval shells out to the sgl-eval CLI and
# fails without it. Bumping the SHA can change zero-shot \boxed{} grading, so
# re-baseline MODEL_SCORE_THRESHOLDS in
# test/registered/eval/test_text_models_gsm8k_eval.py first.
SGL_EVAL_REF="b2a2703c42cae379bbcb8b7ff092df6601a61694"
# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh"
if [ "$CU_MAJOR" = "13" ]; then
MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
MOONCAKE_STALE_PKG="mooncake-transfer-engine"
@@ -723,7 +719,7 @@ install_extra_deps() {
--no-deps --force-reinstall $PIP_INSTALL_SUFFIX
fi
$PIP_CMD install "sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}" $PIP_INSTALL_SUFFIX
$PIP_CMD install "$SGL_EVAL_SPEC" $PIP_INSTALL_SUFFIX
if [ "$IS_BLACKWELL" != "1" ]; then
git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
@@ -84,3 +84,8 @@ rm -rf cann-custom-ops
### Install SGLang
rm -rf python/pyproject.toml && mv python/pyproject_npu.toml python/pyproject.toml
${UV_PIP_INSTALL} -v -e "python[dev_npu]"
### Install sgl-eval
# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh"
${UV_PIP_INSTALL} "$SGL_EVAL_SPEC"
+13
View File
@@ -0,0 +1,13 @@
# Single source of truth for the sgl-eval commit every CI variant installs.
# Meant to be sourced, not executed -- each variant then installs
# "$SGL_EVAL_SPEC" with its own pip invocation, since those differ (uv pip on
# CUDA/CPU, `docker exec ... pip` on AMD, `python3 -m pip` on NPU).
#
# sgl-eval is git-only and cannot be declared in python/pyproject.toml (see the
# note there). Every eval that shells out to the `sgl-eval` CLI fails without
# it, and a bump moves scoring for all of them at once -- so re-baseline
# MODEL_SCORE_THRESHOLDS in
# test/registered/eval/test_text_models_gsm8k_eval.py, and the mmlu thresholds
# of run_eval's other callers, before changing this.
SGL_EVAL_REF="6690895609dcbc5df1e7b00dd57c9502b868ec4d"
SGL_EVAL_SPEC="sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}"