minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] slash handler: lookup runs_on from runner_configs.yml (#25394)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-05-15 13:59:00 -07:00
committed by GitHub
parent 54221dd998
commit f9caf43095
33 changed files with 261 additions and 309 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-states.yml
+1 -1
View File
@@ -63,7 +63,7 @@ jobs:
const missingCIText = ':x: **Missing `run-ci` label** — add it to run CI tests.';
const peBlockedByCIText = ':x: **Blocked** — `run-ci` is required first.';
const notExtraEnabledText = ':warning: **Not enabled** — add `run-ci-extra` label to opt in.';
const stalePushText = ':warning: **Not run on latest push** — push again or use `/rerun-failed-ci` to dispatch.';
const stalePushText = ':warning: **Not run on latest push** — push again to dispatch.';
const ptText = !hasCI
? missingCIText
: (isReal(ptRun) ? `[Run #${ptRun.id}](${ptRun.html_url})` : '_Not run yet_');
.github/workflows/rerun-test.yml
+110 -62
View File
@@ -4,46 +4,43 @@ run-name: ${{ inputs.pr_head_sha && format('[rerun-test] {0} {1}', inputs.test_c
on:
workflow_dispatch:
inputs:
mode:
description: "Dispatch mode: cuda | multimodal_gen | cpu"
required: true
type: choice
options:
- cuda
- multimodal_gen
- cpu
test_command:
description: "Test command(s) to run, one per line (e.g. 'registered/core/test_srt_endpoint.py TestSRTEndpoint.test_simple_decode')"
required: true
type: string
runner_label:
description: "Runner label"
required: true
type: choice
options:
- 1-gpu-h100
- 1-gpu-5090
- 2-gpu-h100
- 4-gpu-h100
- 4-gpu-a10
- 4-gpu-b200
- 8-gpu-h200
- 8-gpu-h200-deepep
- 8-gpu-h20
- 8-gpu-b200
- ubuntu-latest
runs_on:
description: "GHA runner label (cuda/multimodal_gen only; ignored for cpu)"
required: false
type: string
default: ""
install_script:
description: "Install script path (cuda only). E.g. scripts/ci/cuda/ci_install_dependency.sh"
required: false
type: string
default: ""
install_timeout:
description: "Install-step timeout minutes (cuda only)"
required: false
type: string
default: "20"
rdma_devices:
description: "SGLANG_CI_RDMA_ALL_DEVICES csv (cuda only; empty = unset)"
required: false
type: string
default: ""
pr_head_sha:
description: "PR head SHA to checkout (for /rerun-test on fork PRs)"
required: false
type: string
default: ""
use_deepep:
description: "Use ci_install_deepep.sh instead of ci_install_dependency.sh"
required: false
type: string
default: "false"
is_cpu:
description: "Run as CPU-only test (uses ubuntu-latest with uv pip install)"
required: false
type: string
default: "false"
install_diffusion:
description: "Install diffusion dependencies (for multimodal gen tests)"
required: false
type: string
default: "false"
reply_comment_id:
description: "Reply comment ID to write back result to"
required: false
@@ -69,15 +66,15 @@ permissions:
jobs:
rerun-test-cuda:
if: inputs.is_cpu != 'true'
runs-on: ${{ inputs.runner_label }}
if: inputs.mode == 'cuda'
runs-on: ${{ inputs.runs_on }}
timeout-minutes: 120
permissions:
contents: read
issues: write
env:
RUNNER_LABELS: ${{ inputs.runner_label }}
SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.runner_label == '8-gpu-h20' && 'mlx5_1,mlx5_2,mlx5_3,mlx5_4' || '' }}
RUNNER_LABELS: ${{ inputs.runs_on }}
SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.rdma_devices }}
steps:
- name: Checkout code
uses: actions/checkout@v4
@@ -90,7 +87,7 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then
if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
source /etc/profile.d/sglang-ci.sh
fi
python3 scripts/ci/utils/update_rerun_test_status.py \
@@ -102,26 +99,19 @@ jobs:
- uses: ./.github/actions/check-maintenance
- name: Install dependencies
timeout-minutes: 20
timeout-minutes: ${{ fromJson(inputs.install_timeout) }}
run: |
if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then
if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
source /etc/profile.d/sglang-ci.sh
fi
if [[ "${{ inputs.use_deepep }}" == "true" ]]; then
bash scripts/ci/cuda/ci_install_deepep.sh
elif [[ "${{ inputs.install_diffusion }}" == "true" ]]; then
bash scripts/ci/cuda/ci_install_dependency.sh diffusion
else
bash scripts/ci/cuda/ci_install_dependency.sh
fi
bash ${{ inputs.install_script }}
- name: Run test
timeout-minutes: 60
run: |
if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then
if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
source /etc/profile.d/sglang-ci.sh
fi
# Collect non-empty commands into an array for counting.
cmds=()
while IFS= read -r cmd; do
[ -z "$cmd" ] && continue
@@ -134,17 +124,74 @@ jobs:
cmd="${cmds[$idx]}"
echo ""
echo "."
if [[ "${{ inputs.install_diffusion }}" == "true" ]]; then
echo "Begin ($i/$total): python3 -m pytest $cmd -x"
echo "."
file_start=$SECONDS
python3 -m pytest $cmd -x || exit 1
else
echo "Begin ($i/$total): python3 $cmd"
echo "."
file_start=$SECONDS
(cd test/ && python3 $cmd -f) || exit 1
fi
echo "Begin ($i/$total): python3 $cmd"
echo "."
file_start=$SECONDS
(cd test/ && python3 $cmd -f) || exit 1
elapsed=$(( SECONDS - file_start ))
echo "."
echo "End ($i/$total): elapsed=${elapsed}s"
echo "."
echo ""
done
total_elapsed=$(( SECONDS - suite_start ))
echo "All $total test(s) passed in ${total_elapsed}s"
- uses: ./.github/actions/upload-cuda-coredumps
if: failure()
rerun-test-multimodal-gen:
if: inputs.mode == 'multimodal_gen'
runs-on: ${{ inputs.runs_on }}
timeout-minutes: 120
permissions:
contents: read
issues: write
env:
RUNNER_LABELS: ${{ inputs.runs_on }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.pr_head_sha || github.sha }}
- name: Mark runner picked up
if: inputs.reply_comment_id != '' && inputs.reply_marker != ''
continue-on-error: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
python3 scripts/ci/utils/update_rerun_test_status.py \
--comment-id "${{ inputs.reply_comment_id }}" \
--marker "${{ inputs.reply_marker }}" \
--status running \
--repo "${{ github.repository }}"
- uses: ./.github/actions/check-maintenance
- name: Install dependencies (diffusion)
timeout-minutes: 20
run: bash scripts/ci/cuda/ci_install_dependency.sh diffusion
- name: Run test
timeout-minutes: 60
run: |
cmds=()
while IFS= read -r cmd; do
[ -z "$cmd" ] && continue
cmds+=("$cmd")
done <<< "${{ inputs.test_command }}"
total=${#cmds[@]}
suite_start=$SECONDS
for idx in "${!cmds[@]}"; do
i=$((idx + 1))
cmd="${cmds[$idx]}"
echo ""
echo "."
echo "Begin ($i/$total): python3 -m pytest $cmd -x"
echo "."
file_start=$SECONDS
python3 -m pytest $cmd -x || exit 1
elapsed=$(( SECONDS - file_start ))
echo "."
echo "End ($i/$total): elapsed=${elapsed}s"
@@ -158,7 +205,7 @@ jobs:
if: failure()
rerun-test-cpu:
if: inputs.is_cpu == 'true'
if: inputs.mode == 'cpu'
runs-on: ubuntu-latest
timeout-minutes: 120
permissions:
@@ -214,7 +261,6 @@ jobs:
timeout-minutes: 60
run: |
cd test/
# Collect non-empty commands into an array for counting.
cmds=()
while IFS= read -r cmd; do
[ -z "$cmd" ] && continue
@@ -241,7 +287,7 @@ jobs:
echo "All $total test(s) passed in ${total_elapsed}s"
write-back-result:
needs: [rerun-test-cuda, rerun-test-cpu]
needs: [rerun-test-cuda, rerun-test-multimodal-gen, rerun-test-cpu]
if: always() && inputs.reply_comment_id != '' && inputs.reply_marker != ''
runs-on: ubuntu-latest
permissions:
@@ -263,7 +309,9 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
if [[ "${{ needs.rerun-test-cuda.result }}" == "success" || "${{ needs.rerun-test-cpu.result }}" == "success" ]]; then
if [[ "${{ needs.rerun-test-cuda.result }}" == "success" \
|| "${{ needs.rerun-test-multimodal-gen.result }}" == "success" \
|| "${{ needs.rerun-test-cpu.result }}" == "success" ]]; then
STATUS=success
else
STATUS=failure
.github/workflows/slash-command-handler.yml
+1 -1
View File
@@ -86,7 +86,7 @@ jobs:
- name: Install dependencies
run: |
pip install PyGithub
pip install PyGithub PyYAML
- name: Handle Slash Command
env:
scripts/ci/utils/slash_command_handler.py
+142 -137
View File
@@ -9,6 +9,21 @@ from datetime import datetime, timezone
import requests
from github import Auth, Github
# Import scripts/ci/runner_configs.py (sibling-up dir) for runner_config -> runs_on lookup.
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
import runner_configs as _runner_configs # noqa: E402
# rerun-test workflow doesn't build sgl-kernel, so b200 stages always use the
# non-kernel pool when resolving the `$b200_runner` sentinel from runner_configs.yml.
_B200_DEFAULT_RUNNER = "4-gpu-b200"
# install_script values from runner_configs.yml are passed verbatim into a
# `bash ${{ inputs.install_script }}` step in rerun-test.yml. GHA expression
# substitution happens before bash parses, so shell metacharacters in the
# string would inject. Restrict the allowed shape to `scripts/ci/cuda/*.sh`
# (single path component under that dir, no whitespace/operators).
_ALLOWED_INSTALL_SCRIPT = re.compile(r"^scripts/ci/cuda/[\w.-]+\.sh$")
# Configuration
PERMISSIONS_FILE_PATH = ".github/CI_PERMISSIONS.json"
@@ -388,49 +403,6 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr
return False
CUDA_SUITE_TO_RUNNER = {
# PR test suites
"stage-a-test-1-gpu-small": "1-gpu-5090",
"stage-a-test-cpu": "ubuntu-latest",
"stage-b-test-1-gpu-small": "1-gpu-5090",
"stage-b-test-1-gpu-large": "1-gpu-h100",
"stage-b-test-2-gpu-large": "2-gpu-h100",
"stage-b-test-4-gpu-b200": "4-gpu-b200",
"stage-c-test-4-gpu-h100": "4-gpu-h100",
"stage-c-test-8-gpu-h200": "8-gpu-h200",
"stage-c-test-8-gpu-h20": "8-gpu-h20",
"stage-c-test-4-gpu-b200": "4-gpu-b200",
"stage-c-test-deepep-4-gpu-h100": "4-gpu-h100",
"stage-c-test-deepep-8-gpu-h200": "8-gpu-h200-deepep",
"stage-c-test-dsv4-4-gpu-b200": "4-gpu-b200",
"stage-c-test-dsv4-8-gpu-h200": "8-gpu-h200",
# Nightly test suites (NVIDIA)
"nightly-1-gpu": "1-gpu-h100",
"nightly-4-gpu": "4-gpu-h100",
"nightly-4-gpu-b200": "4-gpu-b200",
"nightly-8-gpu-common": "8-gpu-h200",
"nightly-8-gpu-h200": "8-gpu-h200",
"nightly-8-gpu-h20": "8-gpu-h20",
"nightly-8-gpu-b200": "8-gpu-b200",
"nightly-eval-text-2-gpu": "2-gpu-h100",
"nightly-eval-vlm-2-gpu": "2-gpu-h100",
"nightly-perf-text-2-gpu": "2-gpu-h100",
"nightly-perf-vlm-2-gpu": "2-gpu-h100",
"nightly-kernel-1-gpu": "1-gpu-h100",
"nightly-kernel-8-gpu-h200": "8-gpu-h200",
# Weekly test suites
"weekly-8-gpu-h200": "8-gpu-h200",
}
DEEPEP_SUITES = {
"stage-c-test-8-gpu-h20",
"stage-c-test-deepep-4-gpu-h100",
"stage-c-test-deepep-8-gpu-h200",
"stage-c-test-dsv4-4-gpu-b200",
"stage-c-test-dsv4-8-gpu-h200",
}
MULTIMODAL_TEST_DIR = "python/sglang/multimodal_gen/test"
MULTIMODAL_PATH_TO_RUNNER = {
@@ -574,71 +546,100 @@ def detect_multimodal_suite(file_path):
return MULTIMODAL_DEFAULT_RUNNER, None
def _extract_suite(content, func_name):
"""Pull a suite name out of a `register_{cuda,cpu}_ci(...)` call.
Two styles are supported:
1. legacy: register_cuda_ci(..., suite="stage-X-test-Y")
2. new: register_cuda_ci(..., stage="stage-X", runner_config="Y")
-> suite = f"{stage}-test-{runner_config}"
"""
legacy = re.search(
rf'^[^#\n]*{func_name}\([^)]*suite\s*=\s*["\']([^"\']+)["\']',
content,
re.MULTILINE,
)
if legacy:
return legacy.group(1)
args = re.search(rf"^[^#\n]*{func_name}\(([^)]*)\)", content, re.MULTILINE)
if args:
stage_m = re.search(r'stage\s*=\s*["\']([^"\']+)["\']', args.group(1))
rc_m = re.search(r'runner_config\s*=\s*["\']([^"\']+)["\']', args.group(1))
if stage_m and rc_m:
return f"{stage_m.group(1)}-test-{rc_m.group(1)}"
return None
def _extract_runner_config(content):
"""Pull `runner_config` and the args string from a `register_cuda_ci(...)` call."""
args = re.search(r"^[^#\n]*register_cuda_ci\s*\(([^)]*)\)", content, re.MULTILINE)
if not args:
return None, None
m = re.search(r'runner_config\s*=\s*["\']([^"\']+)["\']', args.group(1))
return (m.group(1), args.group(1)) if m else (None, None)
def detect_suite(file_path_from_test):
"""
Read a test file and extract the suite from register_cuda_ci or register_cpu_ci.
Read a test file and extract dispatch info from register_cuda_ci or
register_cpu_ci.
Returns (suite_name, runner_label, use_deepep, is_cpu, error_message).
CUDA tests must use `register_cuda_ci(stage=..., runner_config=...)`;
runner label, install script, timeout, and rdma_devices are all resolved
from scripts/ci/runner_configs.yml — the same single source of truth that
drives the main PR test pipeline.
CPU tests (`register_cpu_ci(...)`) dispatch to the CPU job (ubuntu-latest).
Returns dict with keys: suite, runner_label, install_script,
install_timeout, rdma_devices, is_cpu, error.
"""
full_path = f"test/{file_path_from_test}"
with open(full_path, "r") as f:
content = f.read()
suite = _extract_suite(content, "register_cuda_ci")
if suite:
runner = CUDA_SUITE_TO_RUNNER.get(suite)
if not runner:
known = ", ".join(f"`{s}`" for s in sorted(CUDA_SUITE_TO_RUNNER))
return (
suite,
None,
False,
False,
(
f"Unknown CUDA suite `{suite}` in `{full_path}`.\n\n"
f"Known suites: {known}"
),
def _err(suite, msg):
return {
"suite": suite,
"runner_label": None,
"install_script": "",
"install_timeout": "",
"rdma_devices": "",
"is_cpu": False,
"error": msg,
}
rc, args_str = _extract_runner_config(content)
if rc:
configs = _runner_configs.load()
cfg = configs.get(rc)
if cfg is None:
known = ", ".join(f"`{k}`" for k in sorted(configs))
return _err(
rc,
f"Unknown runner_config `{rc}` in `{full_path}` "
f"— not in scripts/ci/runner_configs.yml.\n\n"
f"Known runner_configs: {known}",
)
use_deepep = suite in DEEPEP_SUITES
return suite, runner, use_deepep, False, None
install_script = cfg["install"]
if not _ALLOWED_INSTALL_SCRIPT.match(install_script):
return _err(
rc,
f"Disallowed `install` value `{install_script}` for runner_config "
f"`{rc}` in scripts/ci/runner_configs.yml. The slash handler "
f"passes this string verbatim into a shell step, so it must "
f"match `scripts/ci/cuda/*.sh`.",
)
runs_on = cfg.get("runs_on")
# Resolve $b200_runner sentinel: rerun-test never builds sgl-kernel,
# so always pick the non-kernel b200 pool.
if runs_on == "$b200_runner":
runs_on = _B200_DEFAULT_RUNNER
stage_m = re.search(r'stage\s*=\s*["\']([^"\']+)["\']', args_str)
suite = f"{stage_m.group(1)}-test-{rc}" if stage_m else rc
return {
"suite": suite,
"runner_label": runs_on,
"install_script": install_script,
"install_timeout": str(cfg["install_timeout"]),
"rdma_devices": cfg.get("rdma_devices", ""),
"is_cpu": False,
"error": None,
}
suite = _extract_suite(content, "register_cpu_ci")
if suite:
return suite, "ubuntu-latest", False, True, None
if re.search(r"^[^#\n]*register_cpu_ci\s*\(", content, re.MULTILINE):
return {
"suite": "cpu",
"runner_label": "ubuntu-latest",
"install_script": "",
"install_timeout": "",
"rdma_devices": "",
"is_cpu": True,
"error": None,
}
return (
return _err(
None,
None,
False,
False,
(
f"No `register_cuda_ci()` or `register_cpu_ci()` found in `{full_path}`.\n\n"
f"This file may not be a registered CI test."
),
f"No `register_cuda_ci(runner_config=...)` or `register_cpu_ci()` "
f"found in `{full_path}`. /rerun-test only supports tests registered "
f"via the new-style yml-driven API; nightly/weekly tests aren't "
f"dispatchable through this command.",
)
@@ -646,8 +647,8 @@ def _resolve_test_spec(test_spec):
"""
Resolve a single test spec into its components without dispatching.
Returns a dict with keys: spec, resolved_path, test_command, suite,
runner_label, use_deepep, is_cpu, error.
Returns a dict with keys: spec, test_command, mode, runs_on,
install_script, install_timeout, rdma_devices, error.
"""
if "::" in test_spec:
file_part, test_selector = test_spec.split("::", 1)
@@ -674,57 +675,61 @@ def _resolve_test_spec(test_spec):
test_command = f"{resolved_path}::{test_selector}"
print(
f"Resolved (multimodal): file={resolved_path}, selector={test_selector}, "
f"Resolved (multimodal_gen): file={resolved_path}, selector={test_selector}, "
f"runner={runner_label}, command='{test_command}'"
)
return {
"spec": test_spec,
"test_command": test_command,
"suite": "multimodal",
"runner_label": runner_label,
"use_deepep": False,
"is_cpu": False,
"install_diffusion": True,
"mode": "multimodal_gen",
"runs_on": runner_label,
"install_script": "",
"install_timeout": "",
"rdma_devices": "",
"error": None,
}
suite, runner_label, use_deepep, is_cpu, err = detect_suite(resolved_path)
if err:
return {"spec": test_spec, "error": err}
info = detect_suite(resolved_path)
if info["error"]:
return {"spec": test_spec, "error": info["error"]}
test_command = resolved_path
if test_selector:
test_command = f"{resolved_path} {test_selector}"
mode = "cpu" if info["is_cpu"] else "cuda"
print(
f"Resolved: file={resolved_path}, selector={test_selector}, "
f"suite={suite}, runner={runner_label}, deepep={use_deepep}, "
f"cpu={is_cpu}, command='{test_command}'"
f"suite={info['suite']}, mode={mode}, runs_on={info['runner_label']}, "
f"install={info['install_script']}, rdma={info['rdma_devices']}, "
f"command='{test_command}'"
)
return {
"spec": test_spec,
"test_command": test_command,
"suite": suite,
"runner_label": runner_label,
"use_deepep": use_deepep,
"is_cpu": is_cpu,
"install_diffusion": False,
"mode": mode,
"runs_on": info["runner_label"],
"install_script": info["install_script"],
"install_timeout": info["install_timeout"],
"rdma_devices": info["rdma_devices"],
"error": None,
}
def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker=""):
"""
Dispatch a single workflow run for a batch of resolved test specs
that share the same (runner_label, use_deepep, is_cpu).
Dispatch a single workflow run for a batch of resolved test specs that
share the same dispatch shape (mode + runs_on + install_script +
install_timeout + rdma_devices).
Returns a dict with keys: specs, success, test_commands, runner_label, run_url, error.
"""
test_commands = [r["test_command"] for r in batch]
runner_label = batch[0]["runner_label"]
use_deepep = batch[0]["use_deepep"]
is_cpu = batch[0]["is_cpu"]
install_diffusion = batch[0].get("install_diffusion", False)
mode = batch[0]["mode"]
runs_on = batch[0]["runs_on"]
install_script = batch[0]["install_script"]
install_timeout = batch[0]["install_timeout"]
rdma_devices = batch[0]["rdma_devices"]
# Join multiple commands with newlines for the workflow to iterate over
combined_command = "\n".join(test_commands)
@@ -751,11 +756,12 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker
pr_head_sha = None
inputs = {
"mode": mode,
"test_command": combined_command,
"runner_label": runner_label,
"use_deepep": str(use_deepep).lower(),
"is_cpu": str(is_cpu).lower(),
"install_diffusion": str(install_diffusion).lower(),
"runs_on": runs_on or "",
"install_script": install_script,
"install_timeout": install_timeout or "20",
"rdma_devices": rdma_devices,
"reply_comment_id": str(reply_comment_id) if reply_comment_id else "",
"reply_marker": reply_marker,
}
@@ -803,7 +809,8 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker
"specs": [r["spec"] for r in batch],
"success": True,
"test_commands": test_commands,
"runner_label": runner_label,
"mode": mode,
"runs_on": runs_on,
"run_url": run_url,
"reply_marker": reply_marker,
}
@@ -853,7 +860,8 @@ def handle_rerun_test(
):
"""
Handles the /rerun-test command. Resolves all test specs, groups them by
(runner_label, use_deepep, is_cpu), and dispatches one workflow per group.
dispatch shape (mode + runs_on + install_script + install_timeout +
rdma_devices), and dispatches one workflow per group.
"""
if not skip_permission_check and not _check_rerun_test_permissions(
gh_repo, pr, comment, user_perms, "rerun-test"
@@ -888,14 +896,15 @@ def handle_rerun_test(
else:
resolved.append(r)
# Phase 2: Group by (runner_label, use_deepep, is_cpu, install_diffusion)
# Phase 2: Group by dispatch shape.
groups = {}
for r in resolved:
key = (
r["runner_label"],
r["use_deepep"],
r["is_cpu"],
r.get("install_diffusion", False),
r["mode"],
r["runs_on"],
r["install_script"],
r["install_timeout"],
r["rdma_devices"],
)
groups.setdefault(key, []).append(r)
@@ -926,12 +935,7 @@ def handle_rerun_test(
lines = []
for dr in dispatch_results:
if dr["success"]:
install_diff = any(
r.get("install_diffusion", False)
for r in resolved
if r["spec"] in dr["specs"]
)
if install_diff:
if dr["mode"] == "multimodal_gen":
cmds = "\n".join(
f"python3 -m pytest {cmd} -x" for cmd in dr["test_commands"]
)
@@ -940,15 +944,16 @@ def handle_rerun_test(
f"cd test/ && python3 {cmd}" for cmd in dr["test_commands"]
)
marker = dr.get("reply_marker", "")
label = dr["runs_on"] or dr["mode"]
if dr.get("run_url"):
lines.append(
f"🚀 `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): "
f"🚀 `{label}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): "
f"⏳ [View workflow run]({dr['run_url']}) {marker}\n"
f"```\n{cmds}\n```"
)
else:
lines.append(
f"🚀 `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): ⏳ {marker}\n"
f"🚀 `{label}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): ⏳ {marker}\n"
f"```\n{cmds}\n```\n"
f"⚠️ Could not retrieve workflow run URL. "
f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress."
test/README.md
+3 -3
View File
@@ -66,12 +66,12 @@ Every CI-discovered test file must call a registration function at module level:
```python
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=80, suite="stage-b-test-1-gpu-small")
register_cuda_ci(est_time=80, stage="stage-b", runner_config="1-gpu-small")
```
Parameters: `est_time` (seconds), `suite` (target suite), `nightly=True` (nightly-only), `disabled="reason"` (temporarily disable).
Parameters: `est_time` (seconds), `stage` + `runner_config` (target stage and runner pool from `scripts/ci/runner_configs.yml`), `nightly=True` (nightly-only), `disabled="reason"` (temporarily disable).
Keep `est_time` and `suite` as **literal values** — `run_suite.py` collects them by AST parsing.
Keep `est_time`, `stage`, `runner_config` as **literal values** — `run_suite.py` collects them by AST parsing.
JIT kernel files live outside `test/registered/` but still use registration:
- Correctness tests: `python/sglang/jit_kernel/tests/test_*.py` → `stage-b-kernel-unit-1-gpu-large`
test/manual/4-gpu-models/test_qwen35_fp4_triton.py
-3
View File
@@ -1,7 +1,6 @@
import unittest
from sglang.test.accuracy_test_runner import AccuracyTestParams
from sglang.test.ci.ci_register import register_cuda_ci
# This eval harness applies the chat_template, which is critical for qwen3.5
# to get good accuracy on gsm8k
@@ -11,8 +10,6 @@ from sglang.test.test_utils import (
ModelLaunchSettings,
)
register_cuda_ci(est_time=720, suite="stage-c-test-4-gpu-b200")
QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4"
ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}}
test/manual/4-gpu-models/test_qwen3_next_models.py
-3
View File
@@ -1,13 +1,10 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin
from sglang.test.kits.prefix_cache_branching_kit import PrefixCacheBranchingMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
register_cuda_ci(est_time=142, suite="stage-c-test-4-gpu-h100")
QWEN3_NEXT_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct"
test/manual/8-gpu-models/test_deepseek_v3_basic.py
-3
View File
@@ -2,7 +2,6 @@ import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
@@ -15,8 +14,6 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=301, suite="stage-c-test-8-gpu-h200")
FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324"
test/manual/8-gpu-models/test_dsa_models_basic.py
-3
View File
@@ -2,7 +2,6 @@ import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
@@ -14,8 +13,6 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=1047, suite="stage-c-test-8-gpu-h200")
DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
GLM5_MODEL_PATH = "zai-org/GLM-5-FP8"
test/manual/attention/test_fa3.py
-3
View File
@@ -5,7 +5,6 @@ import requests
from sglang.srt.environ import envs
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_DRAFT_MODEL_EAGLE3,
@@ -20,8 +19,6 @@ from sglang.test.test_utils import (
# FlashAttention3 integration tests (requires SM 90+ / H100)
# Multiple test classes: FA3, FA3+MLA, FA3+SpecDecode variants
register_cuda_ci(est_time=551, suite="stage-b-test-1-gpu-large")
GSM_DATASET_PATH = None
# In case of some machine lack internet connection, we can set OFFLINE_MODE to True.
test/manual/attention/test_local_attn.py
+1 -4
View File
@@ -5,7 +5,6 @@ from types import SimpleNamespace
import requests
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION,
@@ -15,10 +14,8 @@ from sglang.test.test_utils import (
popen_launch_server,
)
# Local attention with FA3 (requires SM 90+ / H100, tp=4)
register_cuda_ci(est_time=217, suite="stage-c-test-4-gpu-h100")
@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher")
class TestFlashAttention3LocalAttn(CustomTestCase):
model = DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION
test/manual/core/test_gpt_oss_1gpu.py
-4
View File
@@ -1,11 +1,7 @@
import unittest
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.gpt_oss_common import BaseTestGptOss
register_cuda_ci(est_time=408, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=750, suite="stage-b-test-1-gpu-small-amd-mi35x")
class TestGptOss1Gpu(BaseTestGptOss):
def test_mxfp4_20b(self):
test/manual/distributed/test_dp_attention_large.py
-4
View File
@@ -5,7 +5,6 @@ import requests
from sglang.lang.chat_template import get_chat_template_by_model_path
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin
from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin
from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin
@@ -22,9 +21,6 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=245, suite="stage-c-test-4-gpu-h100")
register_amd_ci(est_time=350, suite="stage-c-test-4-gpu-amd")
@unittest.skipIf(
is_in_amd_ci(),
test/manual/eval/test_eval_accuracy_large.py
-4
View File
@@ -6,7 +6,6 @@ python -m unittest test_eval_accuracy_large.TestEvalAccuracyLarge.test_mmlu
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import HumanEvalMixin, MGSMEnMixin, MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -16,9 +15,6 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=496, suite="stage-b-test-1-gpu-small")
register_amd_ci(est_time=420, suite="stage-b-test-1-gpu-small-amd")
class TestEvalAccuracyLarge(CustomTestCase, MMLUMixin, HumanEvalMixin, MGSMEnMixin):
mmlu_score_threshold = 0.70
test/manual/lora/test_lora_backend.py
-8
View File
@@ -17,7 +17,6 @@ import os
import unittest
from typing import List
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.lora_utils import (
ALL_OTHER_LORA_MODELS,
BACKENDS,
@@ -29,13 +28,6 @@ from sglang.test.lora_utils import (
)
from sglang.test.test_utils import CustomTestCase, is_in_ci
register_cuda_ci(est_time=224, suite="stage-b-test-1-gpu-small")
register_amd_ci(
est_time=200,
suite="stage-b-test-1-gpu-small-amd",
disabled="see https://github.com/sgl-project/sglang/issues/13107",
)
class TestLoRABackend(CustomTestCase):
test/manual/mla/test_mla.py
+1 -5
View File
@@ -1,7 +1,6 @@
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
@@ -11,11 +10,8 @@ from sglang.test.test_utils import (
popen_launch_server,
)
# MLA attention test with MGSM evaluation
register_cuda_ci(est_time=181, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd")
class TestMLA(CustomTestCase, MGSMEnMixin):
mgsm_en_score_threshold = 0.8
test/manual/mla/test_mla_deepseek_v3.py
-9
View File
@@ -5,7 +5,6 @@ from types import SimpleNamespace
import requests
from sglang.srt.utils import is_cuda, is_hip, kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -15,14 +14,6 @@ from sglang.test.test_utils import (
popen_launch_server,
)
# DeepSeek-V3 MLA tests with torch compile, FA3, and MTP speculative decoding
register_cuda_ci(est_time=543, suite="stage-b-test-1-gpu-large")
register_amd_ci(
est_time=221,
suite="stage-b-test-1-gpu-small-amd",
disabled="see https://github.com/sgl-project/sglang/issues/12574",
)
class TestMLADeepseekV3(CustomTestCase):
@classmethod
test/manual/models/test_nvidia_nemotron_nano_v2.py
-3
View File
@@ -1,12 +1,9 @@
import unittest
from sglang.srt.utils import is_blackwell
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
register_cuda_ci(est_time=249, suite="stage-b-test-2-gpu-large")
class TestNvidiaNemotronNanoV2BF16(GSM8KMixin, DefaultServerBase):
model = "nvidia/NVIDIA-Nemotron-Nano-9B-v2"
test/manual/models/test_nvidia_nemotron_nano_v2_vl.py
-3
View File
@@ -1,6 +1,5 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.mmmu_vlm_kit import MMMUMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
@@ -10,8 +9,6 @@ from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
# GSM8k + MMMU evaluation
register_cuda_ci(est_time=256, suite="stage-b-test-1-gpu-large")
MODEL = "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16"
test/manual/models/test_qwen_models.py
-4
View File
@@ -4,7 +4,6 @@ import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -13,9 +12,6 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=108, suite="stage-b-test-1-gpu-small")
register_amd_ci(est_time=130, suite="stage-b-test-1-gpu-small-amd")
class TestQwen2(CustomTestCase):
@classmethod
test/manual/openai_server/function_call/test_tool_choice.py
-4
View File
@@ -14,7 +14,6 @@ import openai
from sglang.srt.utils import kill_process_tree
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -22,9 +21,6 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=204, stage="stage-b", runner_config="1-gpu-small")
register_amd_ci(est_time=258, suite="stage-b-test-1-gpu-small-amd")
class TestToolChoiceLlama32(CustomTestCase):
test/manual/perf/test_bench_one_batch_1gpu.py
-4
View File
@@ -5,7 +5,6 @@ import unittest
import numpy as np
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
@@ -16,9 +15,6 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=95, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=120, suite="stage-b-test-1-gpu-large-amd")
class TestBenchOneBatch1GPU(CustomTestCase):
test/manual/prefill_only/test_cross_encoder_models.py
-4
View File
@@ -4,16 +4,12 @@ import unittest
import torch
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.runners import TEST_RERANK_QUERY_DOCS, HFRunner, SRTRunner
from sglang.test.test_utils import CustomTestCase, is_in_ci
# Cross encoder model tests
register_cuda_ci(est_time=125, suite="stage-b-test-1-gpu-small")
register_amd_ci(est_time=150, suite="stage-b-test-1-gpu-small-amd")
MODELS = [
("cross-encoder/ms-marco-MiniLM-L6-v2", 1, 1e-2),
("BAAI/bge-reranker-v2-m3", 1, 1e-2),
test/manual/prefill_only/test_encoder_embedding_models.py
-3
View File
@@ -6,7 +6,6 @@ import unittest
import torch
from transformers import AutoConfig, AutoTokenizer
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner
from sglang.test.test_utils import CustomTestCase, get_similarities, is_in_ci
@@ -29,8 +28,6 @@ from sglang.test.test_utils import CustomTestCase, get_similarities, is_in_ci
# python -m unittest test_encoder_embedding_models.TestEncoderEmbeddingModels.test_prefill_logits
register_cuda_ci(est_time=444, suite="stage-b-test-1-gpu-small")
MODELS = [("BAAI/bge-small-en", 1, 1e-5), ("BAAI/bge-m3", 1, 1e-5)]
ATTENTION_BACKEND = ["torch_native", "triton", "flashinfer"]
test/manual/quant/test_autoround.py
-3
View File
@@ -7,7 +7,6 @@ import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST,
@@ -17,8 +16,6 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=99, suite="stage-b-test-1-gpu-large")
class TestAutoRound(CustomTestCase):
@classmethod
test/manual/quant/test_deepseek_v32_fp4_4gpu.py
-3
View File
@@ -2,7 +2,6 @@ import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
@@ -13,8 +12,6 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=874, suite="stage-c-test-4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4"
SERVER_LAUNCH_TIMEOUT = 1200
test/manual/quant/test_eval_fp8_accuracy.py
-4
View File
@@ -2,7 +2,6 @@ import unittest
from types import SimpleNamespace
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8,
@@ -14,9 +13,6 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=351, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=600, suite="stage-b-test-1-gpu-small-amd")
class TestEvalFP8Accuracy(CustomTestCase):
@classmethod
test/manual/quant/test_quantization.py
-3
View File
@@ -4,7 +4,6 @@ import warnings
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1,
@@ -16,8 +15,6 @@ from sglang.test.test_utils import (
write_results_to_json,
)
register_cuda_ci(est_time=460, suite="stage-b-test-1-gpu-large")
MODEL_SCORE_THRESHOLDS = {
# Baselines observed with gsm8k 5-shot concatenated format via chat API,
# which scores lower than reported benchmarks using proper CoT format.
test/manual/scheduler/test_no_chunked_prefill.py
-4
View File
@@ -1,6 +1,5 @@
import unittest
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
CustomTestCase,
@@ -8,9 +7,6 @@ from sglang.test.test_utils import (
run_mmlu_test,
)
register_cuda_ci(est_time=131, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=108, suite="stage-b-test-1-gpu-small-amd")
class TestNoChunkedPrefill(CustomTestCase):
test/manual/scheduler/test_no_overlap_scheduler.py
-4
View File
@@ -6,12 +6,8 @@ python3 test_overlap_schedule.py
import unittest
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import CustomTestCase, run_mmlu_test
register_cuda_ci(est_time=267, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=275, suite="stage-b-test-1-gpu-small-amd")
class TestOverlapSchedule(CustomTestCase):
def test_no_radix_attention_chunked_prefill(self):
test/manual/spec/eagle/test_eagle3_basic.py
-4
View File
@@ -4,7 +4,6 @@ from types import SimpleNamespace
import requests
from sglang.srt.utils import is_hip
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.server_fixtures.eagle_fixture import EagleServerBase
from sglang.test.test_utils import (
@@ -12,9 +11,6 @@ from sglang.test.test_utils import (
DEFAULT_TARGET_MODEL_EAGLE3,
)
register_cuda_ci(est_time=88, suite="stage-b-test-1-gpu-small")
register_amd_ci(est_time=50, suite="stage-b-test-1-gpu-small")
_is_hip = is_hip()
test/registered/dsv4/test_deepseek_v4_flash_fp4_megamoe_b200.py
+1 -1
View File
@@ -21,7 +21,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_cuda_ci(est_time=900, suite="stage-c-test-dsv4-4-gpu-b200")
register_cuda_ci(est_time=900, stage="stage-c", runner_config="dsv4-4-gpu-b200")
MODEL = "deepseek-ai/DeepSeek-V4-Flash"
SERVER_LAUNCH_TIMEOUT = 3600
test/registered/unit/managers/test_priority_scheduling_disaggregation.py
+1 -1
View File
@@ -14,7 +14,7 @@ from sglang.srt.managers.schedule_batch import FINISH_ABORT # noqa: E402
from sglang.srt.managers.scheduler import Scheduler # noqa: E402
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=5, stage="stage-a", runner_config="1-gpu-small")
register_cuda_ci(est_time=5, stage="stage-b", runner_config="1-gpu-small")
class TestDisaggregationPriorityQueueing(unittest.TestCase):