[CI] Route mmlu and GB300 MMMU-Pro evals through sgl-eval (#34477)
This commit is contained in:
@@ -134,6 +134,10 @@ else
|
||||
install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]"
|
||||
fi
|
||||
|
||||
# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/../utils/sgl_eval_ref.sh"
|
||||
install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache "$SGL_EVAL_SPEC"
|
||||
|
||||
if [[ -n "${SKIP_TT_DEPS}" ]]; then
|
||||
echo "Didn't build lmms_eval, human-eval, and others"
|
||||
else
|
||||
|
||||
@@ -682,12 +682,8 @@ stabilize_flashinfer_jit_paths() {
|
||||
install_extra_deps() {
|
||||
MOONCAKE_VERSION="0.3.12.post1"
|
||||
NIXL_VERSION="1.3.0"
|
||||
# sgl-eval is git-only and cannot be declared in python/pyproject.toml (see
|
||||
# the note there). The nightly GSM8K eval shells out to the sgl-eval CLI and
|
||||
# fails without it. Bumping the SHA can change zero-shot \boxed{} grading, so
|
||||
# re-baseline MODEL_SCORE_THRESHOLDS in
|
||||
# test/registered/eval/test_text_models_gsm8k_eval.py first.
|
||||
SGL_EVAL_REF="b2a2703c42cae379bbcb8b7ff092df6601a61694"
|
||||
# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
|
||||
source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh"
|
||||
if [ "$CU_MAJOR" = "13" ]; then
|
||||
MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
|
||||
MOONCAKE_STALE_PKG="mooncake-transfer-engine"
|
||||
@@ -723,7 +719,7 @@ install_extra_deps() {
|
||||
--no-deps --force-reinstall $PIP_INSTALL_SUFFIX
|
||||
fi
|
||||
|
||||
$PIP_CMD install "sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}" $PIP_INSTALL_SUFFIX
|
||||
$PIP_CMD install "$SGL_EVAL_SPEC" $PIP_INSTALL_SUFFIX
|
||||
|
||||
if [ "$IS_BLACKWELL" != "1" ]; then
|
||||
git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
|
||||
|
||||
@@ -84,3 +84,8 @@ rm -rf cann-custom-ops
|
||||
### Install SGLang
|
||||
rm -rf python/pyproject.toml && mv python/pyproject_npu.toml python/pyproject.toml
|
||||
${UV_PIP_INSTALL} -v -e "python[dev_npu]"
|
||||
|
||||
### Install sgl-eval
|
||||
# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
|
||||
source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh"
|
||||
${UV_PIP_INSTALL} "$SGL_EVAL_SPEC"
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Single source of truth for the sgl-eval commit every CI variant installs.
|
||||
# Meant to be sourced, not executed -- each variant then installs
|
||||
# "$SGL_EVAL_SPEC" with its own pip invocation, since those differ (uv pip on
|
||||
# CUDA/CPU, `docker exec ... pip` on AMD, `python3 -m pip` on NPU).
|
||||
#
|
||||
# sgl-eval is git-only and cannot be declared in python/pyproject.toml (see the
|
||||
# note there). Every eval that shells out to the `sgl-eval` CLI fails without
|
||||
# it, and a bump moves scoring for all of them at once -- so re-baseline
|
||||
# MODEL_SCORE_THRESHOLDS in
|
||||
# test/registered/eval/test_text_models_gsm8k_eval.py, and the mmlu thresholds
|
||||
# of run_eval's other callers, before changing this.
|
||||
SGL_EVAL_REF="6690895609dcbc5df1e7b00dd57c9502b868ec4d"
|
||||
SGL_EVAL_SPEC="sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}"
|
||||
Reference in New Issue
Block a user