minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add GB300 base C CI suite (#27427)

Browse Source
This commit is contained in:
Baizhou Zhang
2026-06-06 02:27:30 -07:00
committed by GitHub
parent 42fe025280
commit 9a48bf75f5
9 changed files with 89 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/_pr-test-stage.yml
+2
View File
@@ -119,6 +119,8 @@ jobs:
- name: Install dependencies
timeout-minutes: ${{ fromJson(steps.rc.outputs.install_timeout) }}
env:
GRACE_BLACKWELL: ${{ steps.rc.outputs.grace_blackwell || '0' }}
run: |
CUSTOM_BUILD_SGL_KERNEL=${{ fromJson(inputs.check_changes).sgl_kernel }} bash ${{ steps.rc.outputs.install }}
.github/workflows/pr-test.yml
+15 -3
View File
@@ -59,8 +59,6 @@ env:
SGLANG_CUDA_COREDUMP: "1"
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
SKIP_PR_TEST_HEALTH_CHECK: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }}
# TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main.
FORCE_REBUILD_DEEPEP: '1'
# Schedule / main-branch dispatch / workflow_call from main use refs/heads/main; PR events use refs/pull/*/merge
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
USE_VENV: false
@@ -515,6 +513,20 @@ jobs:
timeout_per_file: '1800'
secrets: inherit
base-c-test-4-gpu-gb300:
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/_pr-test-stage.yml
with:
self_name: base-c-test-4-gpu-gb300
runner_config: 4-gpu-gb300
check_changes: ${{ toJson(needs.check-changes.outputs) }}
caller_inputs: ${{ toJson(inputs) }}
partitions: ${{ needs.check-changes.outputs.partitions }}
run_timeout_minutes: '30'
timeout_per_file: '1800'
secrets: inherit
pr-test-finish:
needs:
[
@@ -545,7 +557,7 @@ jobs:
base-c-test-deepep-4-gpu-b200,
base-c-test-deepep-8-gpu-h200,
base-c-test-4-gpu-b200,
# base-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner
base-c-test-4-gpu-gb300,
]
if: always()
runs-on: ubuntu-latest
.github/workflows/rerun-test.yml
+7
View File
@@ -31,6 +31,11 @@ on:
required: false
type: string
default: "20"
grace_blackwell:
description: "Set GRACE_BLACKWELL for the install step (cuda only)"
required: false
type: string
default: "0"
rdma_devices:
description: "SGLANG_CI_RDMA_ALL_DEVICES csv (cuda only; empty = unset)"
required: false
@@ -100,6 +105,8 @@ jobs:
- name: Install dependencies
timeout-minutes: ${{ fromJson(inputs.install_timeout) }}
env:
GRACE_BLACKWELL: ${{ inputs.grace_blackwell }}
run: |
if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
source /etc/profile.d/sglang-ci.sh
scripts/ci/runner_configs.yml
+3
View File
@@ -8,6 +8,8 @@
# - artifact_version: actions/download-artifact major version
# - install_timeout: install-step wall-clock cap (minutes), enforced via
# `timeout-minutes:` on the install step in _pr-test-stage.yml
# - grace_blackwell (optional): exported as GRACE_BLACKWELL for the install
# step. Used by GB300 DeePEP setup.
# - runs_on: GHA runner label for the stage's `runs-on:`. The literal
# `$b200_runner` is substituted at workflow-load time with the dynamic
# b200 runner tag from check-changes (see runner_configs.py --map).
@@ -23,6 +25,7 @@ runner_configs:
1-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-h100 }
2-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 2-gpu-h100 }
4-gpu-b200: { install: *default, artifact_version: v6, install_timeout: "20", runs_on: $b200_runner }
4-gpu-gb300: { install: *deepep, artifact_version: v6, install_timeout: "20", grace_blackwell: "1", runs_on: 4-gpu-gb300 }
4-gpu-h100: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 }
8-gpu-h200: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200 }
8-gpu-b200: { install: *default, artifact_version: v6, install_timeout: "20", runs_on: 8-gpu-b200 }
scripts/ci/utils/slash_command_handler.py
+19 -9
View File
@@ -709,8 +709,9 @@ def _extract_legacy_suites(content):
# matches the runner the nightly/weekly pipeline actually uses (see
# .github/workflows/{nightly,weekly}-test-nvidia.yml), so /rerun-test can still
# dispatch a single nightly/weekly test. The runner label, install script,
# timeout and rdma_devices are then resolved from runner_configs.yml as usual,
# keeping that file the single source of truth for runner details.
# timeout, grace_blackwell, and rdma_devices are then resolved from
# runner_configs.yml as usual, keeping that file the single source of truth for
# runner details.
#
# Suites on hardware with no matching runner_config (e.g. nightly-4-gpu-gb300)
# and non-CUDA suites (npu/amd) are intentionally absent and stay
@@ -740,6 +741,7 @@ def _dispatch_err(suite, msg):
"runner_label": None,
"install_script": "",
"install_timeout": "",
"grace_blackwell": "0",
"rdma_devices": "",
"is_cpu": False,
"error": msg,
@@ -780,6 +782,7 @@ def _resolve_runner_config(rc, full_path, suite):
"runner_label": runs_on,
"install_script": install_script,
"install_timeout": str(cfg["install_timeout"]),
"grace_blackwell": str(cfg.get("grace_blackwell", "0")),
"rdma_devices": cfg.get("rdma_devices", ""),
"is_cpu": False,
"error": None,
@@ -793,9 +796,9 @@ def detect_suite(file_path_from_test):
A CUDA file can carry multiple `register_cuda_ci(...)` calls — one per
pool it should run on — so this returns a *list* of dispatch dicts, one
per registration. Runner label, install script, timeout, and rdma_devices
are all resolved from scripts/ci/runner_configs.yml — the same single
source of truth that drives the main PR test pipeline.
per registration. Runner label, install script, timeout, grace_blackwell,
and rdma_devices are all resolved from scripts/ci/runner_configs.yml — the
same single source of truth that drives the main PR test pipeline.
Legacy nightly/weekly CUDA suites (single-string `suite=`) are dispatchable
too: each suite name is mapped to the matching runner_config via
@@ -806,7 +809,7 @@ def detect_suite(file_path_from_test):
`error` set.
Each dict has keys: suite, runner_label, install_script,
install_timeout, rdma_devices, is_cpu, error.
install_timeout, grace_blackwell, rdma_devices, is_cpu, error.
"""
full_path = f"test/{file_path_from_test}"
with open(full_path, "r") as f:
@@ -838,6 +841,7 @@ def detect_suite(file_path_from_test):
"runner_label": "ubuntu-latest",
"install_script": "",
"install_timeout": "",
"grace_blackwell": "0",
"rdma_devices": "",
"is_cpu": True,
"error": None,
@@ -912,6 +916,7 @@ def _resolve_test_spec(test_spec):
"runs_on": runner_label,
"install_script": "",
"install_timeout": "",
"grace_blackwell": "0",
"rdma_devices": "",
"error": None,
}
@@ -930,7 +935,8 @@ def _resolve_test_spec(test_spec):
print(
f"Resolved: file={resolved_path}, selector={test_selector}, "
f"suite={info['suite']}, mode={mode}, runs_on={info['runner_label']}, "
f"install={info['install_script']}, rdma={info['rdma_devices']}, "
f"install={info['install_script']}, grace_blackwell={info['grace_blackwell']}, "
f"rdma={info['rdma_devices']}, "
f"command='{test_command}'"
)
out.append(
@@ -941,6 +947,7 @@ def _resolve_test_spec(test_spec):
"runs_on": info["runner_label"],
"install_script": info["install_script"],
"install_timeout": info["install_timeout"],
"grace_blackwell": info["grace_blackwell"],
"rdma_devices": info["rdma_devices"],
"error": None,
}
@@ -952,7 +959,7 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker
"""
Dispatch a single workflow run for a batch of resolved test specs that
share the same dispatch shape (mode + runs_on + install_script +
install_timeout + rdma_devices).
install_timeout + grace_blackwell + rdma_devices).
Returns a dict with keys: specs, success, test_commands, runner_label, run_url, error.
"""
@@ -961,6 +968,7 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker
runs_on = batch[0]["runs_on"]
install_script = batch[0]["install_script"]
install_timeout = batch[0]["install_timeout"]
grace_blackwell = batch[0]["grace_blackwell"]
rdma_devices = batch[0]["rdma_devices"]
# Join multiple commands with newlines for the workflow to iterate over
@@ -993,6 +1001,7 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker
"runs_on": runs_on or "",
"install_script": install_script,
"install_timeout": install_timeout or "20",
"grace_blackwell": grace_blackwell or "0",
"rdma_devices": rdma_devices,
"reply_comment_id": str(reply_comment_id) if reply_comment_id else "",
"reply_marker": reply_marker,
@@ -1100,7 +1109,7 @@ def handle_rerun_test(
"""
Handles the /rerun-test command. Resolves all test specs, groups them by
dispatch shape (mode + runs_on + install_script + install_timeout +
rdma_devices), and dispatches one workflow per group.
grace_blackwell + rdma_devices), and dispatches one workflow per group.
"""
if not skip_permission_check and not _check_rerun_test_permissions(
gh_repo, pr, comment, user_perms, "rerun-test"
@@ -1195,6 +1204,7 @@ def handle_rerun_test(
r["runs_on"],
r["install_script"],
r["install_timeout"],
r["grace_blackwell"],
r["rdma_devices"],
)
groups.setdefault(key, []).append(r)
test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py
+1 -1
View File
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
try_cached_model,
)
register_cuda_ci(est_time=1800, stage="base-c", runner_config="4-gpu-gb200")
register_cuda_ci(est_time=1800, stage="base-c", runner_config="4-gpu-gb300")
class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase):
test/registered/disaggregation/test_disaggregation_aarch64.py
+4 -3
View File
@@ -8,12 +8,13 @@ from sglang.test.server_fixtures.disaggregation_fixture import (
PDDisaggregationServerBase,
)
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
popen_launch_pd_server,
)
register_cuda_ci(est_time=300, stage="base-c", runner_config="4-gpu-gb200")
QWEN3_8B_MODEL_PATH = "Qwen/Qwen3-8B"
register_cuda_ci(est_time=300, stage="base-c", runner_config="4-gpu-gb300")
class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase):
@@ -22,7 +23,7 @@ class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase):
super().setUpClass()
os.environ["SGLANG_MOONCAKE_CUSTOM_MEM_POOL"] = "true"
os.environ["MC_FORCE_MNNVL"] = "true"
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.model = QWEN3_8B_MODEL_PATH
# Non blocking start servers
cls.start_prefill()
test/registered/utils/test_numa_utils.py
+37 -31
View File
@@ -10,8 +10,8 @@ from sglang.srt.utils.numa_utils import (
from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
register_cpu_ci(est_time=7, suite="base-a-test-cpu")
register_cuda_ci(est_time=10, stage="base-c", runner_config="4-gpu-gb200")
register_cuda_ci(est_time=10, stage="base-c", runner_config="8-gpu-b200")
register_cuda_ci(est_time=10, stage="base-c", runner_config="4-gpu-gb300")
register_cuda_ci(est_time=10, stage="base-c", runner_config="4-gpu-b200")
class TestIsNumaAvailable(unittest.TestCase):
@@ -249,36 +249,44 @@ class TestGetNumaNodeIfAvailable(unittest.TestCase):
_mock_gpu.assert_not_called()
def _get_gpu_name():
def _get_gpu_info():
try:
import pynvml
pynvml.nvmlInit()
handle = pynvml.nvmlDeviceGetHandleByIndex(0)
name = pynvml.nvmlDeviceGetName(handle)
if isinstance(name, bytes):
name = name.decode()
count = pynvml.nvmlDeviceGetCount()
pynvml.nvmlShutdown()
return name
return name, count
except Exception:
return ""
return "", 0
_gpu_name = _get_gpu_name()
_gpu_name, _gpu_count = _get_gpu_info()
@unittest.skipUnless("GB200" in _gpu_name, "Requires GB200 hardware")
class TestGB200NumaTopology(unittest.TestCase):
"""Hardware test validating expected NUMA topology on GB200 (2 NUMA nodes, 4 GPUs)."""
def _query_single_numa_node_for_gpu(gpu_id: int):
nodes = _query_numa_node_for_gpu(gpu_id)
if len(nodes) != 1:
raise AssertionError(f"GPU {gpu_id}: expected one NUMA node, got {nodes}")
return nodes[0]
def _make_server_args(self):
args = MagicMock()
args.numa_node = None
return args
@unittest.skipUnless(
("GB200" in _gpu_name or "GB300" in _gpu_name) and _gpu_count == 4,
"Requires 4-GPU Grace Blackwell hardware",
)
class TestGraceBlackwellNumaTopology(unittest.TestCase):
"""Hardware test validating expected NUMA topology on 4-GPU GB200/GB300."""
def test_gpu_numa_mapping(self):
self.assertEqual(_gpu_count, 4)
expected = {0: 0, 1: 0, 2: 1, 3: 1}
args = self._make_server_args()
for gpu_id, expected_node in expected.items():
result = get_numa_node_if_available(args, gpu_id)
result = _query_single_numa_node_for_gpu(gpu_id)
self.assertEqual(
result,
expected_node,
@@ -286,25 +294,23 @@ class TestGB200NumaTopology(unittest.TestCase):
)
@unittest.skipUnless("B200" in _gpu_name, "Requires B200 hardware")
@unittest.skipUnless(
"B200" in _gpu_name and _gpu_count == 4,
"Requires 4-GPU B200 hardware",
)
class TestB200NumaTopology(unittest.TestCase):
"""Hardware test validating expected NUMA topology on B200 (2 NUMA nodes, 8 GPUs)."""
def _make_server_args(self):
args = MagicMock()
args.numa_node = None
return args
"""Hardware test validating expected NUMA topology on 4-GPU B200."""
def test_gpu_numa_mapping(self):
expected = {0: 0, 1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1}
args = self._make_server_args()
for gpu_id, expected_node in expected.items():
result = get_numa_node_if_available(args, gpu_id)
self.assertEqual(
result,
expected_node,
f"GPU {gpu_id}: expected NUMA node {expected_node}, got {result}",
)
self.assertEqual(_gpu_count, 4)
numa_nodes = {
_query_single_numa_node_for_gpu(gpu_id) for gpu_id in range(_gpu_count)
}
self.assertEqual(
len(numa_nodes),
1,
f"Expected all visible 4-GPU B200 devices on one NUMA node, got {numa_nodes}",
)
if __name__ == "__main__":
test/run_suite.py
+1 -1
View File
@@ -54,7 +54,7 @@ PER_COMMIT_SUITES = {
"base-b-kernel-benchmark-1-gpu-large",
"base-c-test-4-gpu-h100",
"base-c-test-4-gpu-b200",
"base-c-test-4-gpu-gb200",
"base-c-test-4-gpu-gb300",
"base-c-test-8-gpu-h20",
"base-c-test-8-gpu-h200",
"base-c-test-8-gpu-b200",