From 9a48bf75f537140f712758014c373cef2ee7a1b4 Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Sat, 6 Jun 2026 02:27:30 -0700 Subject: [PATCH] Add GB300 base C CI suite (#27427) --- .github/workflows/_pr-test-stage.yml | 2 + .github/workflows/pr-test.yml | 18 ++++- .github/workflows/rerun-test.yml | 7 ++ scripts/ci/runner_configs.yml | 3 + scripts/ci/utils/slash_command_handler.py | 28 +++++--- .../test_deepseek_v3_cutedsl_4gpu.py | 2 +- .../test_disaggregation_aarch64.py | 7 +- test/registered/utils/test_numa_utils.py | 68 ++++++++++--------- test/run_suite.py | 2 +- 9 files changed, 89 insertions(+), 48 deletions(-) diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index 0b4e12ddc..a45b69c4f 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -119,6 +119,8 @@ jobs: - name: Install dependencies timeout-minutes: ${{ fromJson(steps.rc.outputs.install_timeout) }} + env: + GRACE_BLACKWELL: ${{ steps.rc.outputs.grace_blackwell || '0' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{ fromJson(inputs.check_changes).sgl_kernel }} bash ${{ steps.rc.outputs.install }} diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 1631aa39a..d92163c84 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -59,8 +59,6 @@ env: SGLANG_CUDA_COREDUMP: "1" SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true SKIP_PR_TEST_HEALTH_CHECK: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }} - # TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main. - FORCE_REBUILD_DEEPEP: '1' # Schedule / main-branch dispatch / workflow_call from main use refs/heads/main; PR events use refs/pull/*/merge PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }} USE_VENV: false @@ -515,6 +513,20 @@ jobs: timeout_per_file: '1800' secrets: inherit + base-c-test-4-gpu-gb300: + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: base-c-test-4-gpu-gb300 + runner_config: 4-gpu-gb300 + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '30' + timeout_per_file: '1800' + secrets: inherit + pr-test-finish: needs: [ @@ -545,7 +557,7 @@ jobs: base-c-test-deepep-4-gpu-b200, base-c-test-deepep-8-gpu-h200, base-c-test-4-gpu-b200, - # base-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner + base-c-test-4-gpu-gb300, ] if: always() runs-on: ubuntu-latest diff --git a/.github/workflows/rerun-test.yml b/.github/workflows/rerun-test.yml index f903a8983..dba7dc124 100644 --- a/.github/workflows/rerun-test.yml +++ b/.github/workflows/rerun-test.yml @@ -31,6 +31,11 @@ on: required: false type: string default: "20" + grace_blackwell: + description: "Set GRACE_BLACKWELL for the install step (cuda only)" + required: false + type: string + default: "0" rdma_devices: description: "SGLANG_CI_RDMA_ALL_DEVICES csv (cuda only; empty = unset)" required: false @@ -100,6 +105,8 @@ jobs: - name: Install dependencies timeout-minutes: ${{ fromJson(inputs.install_timeout) }} + env: + GRACE_BLACKWELL: ${{ inputs.grace_blackwell }} run: | if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then source /etc/profile.d/sglang-ci.sh diff --git a/scripts/ci/runner_configs.yml b/scripts/ci/runner_configs.yml index 0e8c60ef5..605a79d12 100644 --- a/scripts/ci/runner_configs.yml +++ b/scripts/ci/runner_configs.yml @@ -8,6 +8,8 @@ # - artifact_version: actions/download-artifact major version # - install_timeout: install-step wall-clock cap (minutes), enforced via # `timeout-minutes:` on the install step in _pr-test-stage.yml +# - grace_blackwell (optional): exported as GRACE_BLACKWELL for the install +# step. Used by GB300 DeePEP setup. # - runs_on: GHA runner label for the stage's `runs-on:`. The literal # `$b200_runner` is substituted at workflow-load time with the dynamic # b200 runner tag from check-changes (see runner_configs.py --map). @@ -23,6 +25,7 @@ runner_configs: 1-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-h100 } 2-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 2-gpu-h100 } 4-gpu-b200: { install: *default, artifact_version: v6, install_timeout: "20", runs_on: $b200_runner } + 4-gpu-gb300: { install: *deepep, artifact_version: v6, install_timeout: "20", grace_blackwell: "1", runs_on: 4-gpu-gb300 } 4-gpu-h100: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 } 8-gpu-h200: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200 } 8-gpu-b200: { install: *default, artifact_version: v6, install_timeout: "20", runs_on: 8-gpu-b200 } diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index 86f4cecaf..57ea48455 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -709,8 +709,9 @@ def _extract_legacy_suites(content): # matches the runner the nightly/weekly pipeline actually uses (see # .github/workflows/{nightly,weekly}-test-nvidia.yml), so /rerun-test can still # dispatch a single nightly/weekly test. The runner label, install script, -# timeout and rdma_devices are then resolved from runner_configs.yml as usual, -# keeping that file the single source of truth for runner details. +# timeout, grace_blackwell, and rdma_devices are then resolved from +# runner_configs.yml as usual, keeping that file the single source of truth for +# runner details. # # Suites on hardware with no matching runner_config (e.g. nightly-4-gpu-gb300) # and non-CUDA suites (npu/amd) are intentionally absent and stay @@ -740,6 +741,7 @@ def _dispatch_err(suite, msg): "runner_label": None, "install_script": "", "install_timeout": "", + "grace_blackwell": "0", "rdma_devices": "", "is_cpu": False, "error": msg, @@ -780,6 +782,7 @@ def _resolve_runner_config(rc, full_path, suite): "runner_label": runs_on, "install_script": install_script, "install_timeout": str(cfg["install_timeout"]), + "grace_blackwell": str(cfg.get("grace_blackwell", "0")), "rdma_devices": cfg.get("rdma_devices", ""), "is_cpu": False, "error": None, @@ -793,9 +796,9 @@ def detect_suite(file_path_from_test): A CUDA file can carry multiple `register_cuda_ci(...)` calls — one per pool it should run on — so this returns a *list* of dispatch dicts, one - per registration. Runner label, install script, timeout, and rdma_devices - are all resolved from scripts/ci/runner_configs.yml — the same single - source of truth that drives the main PR test pipeline. + per registration. Runner label, install script, timeout, grace_blackwell, + and rdma_devices are all resolved from scripts/ci/runner_configs.yml — the + same single source of truth that drives the main PR test pipeline. Legacy nightly/weekly CUDA suites (single-string `suite=`) are dispatchable too: each suite name is mapped to the matching runner_config via @@ -806,7 +809,7 @@ def detect_suite(file_path_from_test): `error` set. Each dict has keys: suite, runner_label, install_script, - install_timeout, rdma_devices, is_cpu, error. + install_timeout, grace_blackwell, rdma_devices, is_cpu, error. """ full_path = f"test/{file_path_from_test}" with open(full_path, "r") as f: @@ -838,6 +841,7 @@ def detect_suite(file_path_from_test): "runner_label": "ubuntu-latest", "install_script": "", "install_timeout": "", + "grace_blackwell": "0", "rdma_devices": "", "is_cpu": True, "error": None, @@ -912,6 +916,7 @@ def _resolve_test_spec(test_spec): "runs_on": runner_label, "install_script": "", "install_timeout": "", + "grace_blackwell": "0", "rdma_devices": "", "error": None, } @@ -930,7 +935,8 @@ def _resolve_test_spec(test_spec): print( f"Resolved: file={resolved_path}, selector={test_selector}, " f"suite={info['suite']}, mode={mode}, runs_on={info['runner_label']}, " - f"install={info['install_script']}, rdma={info['rdma_devices']}, " + f"install={info['install_script']}, grace_blackwell={info['grace_blackwell']}, " + f"rdma={info['rdma_devices']}, " f"command='{test_command}'" ) out.append( @@ -941,6 +947,7 @@ def _resolve_test_spec(test_spec): "runs_on": info["runner_label"], "install_script": info["install_script"], "install_timeout": info["install_timeout"], + "grace_blackwell": info["grace_blackwell"], "rdma_devices": info["rdma_devices"], "error": None, } @@ -952,7 +959,7 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker """ Dispatch a single workflow run for a batch of resolved test specs that share the same dispatch shape (mode + runs_on + install_script + - install_timeout + rdma_devices). + install_timeout + grace_blackwell + rdma_devices). Returns a dict with keys: specs, success, test_commands, runner_label, run_url, error. """ @@ -961,6 +968,7 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker runs_on = batch[0]["runs_on"] install_script = batch[0]["install_script"] install_timeout = batch[0]["install_timeout"] + grace_blackwell = batch[0]["grace_blackwell"] rdma_devices = batch[0]["rdma_devices"] # Join multiple commands with newlines for the workflow to iterate over @@ -993,6 +1001,7 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker "runs_on": runs_on or "", "install_script": install_script, "install_timeout": install_timeout or "20", + "grace_blackwell": grace_blackwell or "0", "rdma_devices": rdma_devices, "reply_comment_id": str(reply_comment_id) if reply_comment_id else "", "reply_marker": reply_marker, @@ -1100,7 +1109,7 @@ def handle_rerun_test( """ Handles the /rerun-test command. Resolves all test specs, groups them by dispatch shape (mode + runs_on + install_script + install_timeout + - rdma_devices), and dispatches one workflow per group. + grace_blackwell + rdma_devices), and dispatches one workflow per group. """ if not skip_permission_check and not _check_rerun_test_permissions( gh_repo, pr, comment, user_perms, "rerun-test" @@ -1195,6 +1204,7 @@ def handle_rerun_test( r["runs_on"], r["install_script"], r["install_timeout"], + r["grace_blackwell"], r["rdma_devices"], ) groups.setdefault(key, []).append(r) diff --git a/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py b/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py index dd50fdf2d..b5ce3d47a 100644 --- a/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py +++ b/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=1800, stage="base-c", runner_config="4-gpu-gb200") +register_cuda_ci(est_time=1800, stage="base-c", runner_config="4-gpu-gb300") class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase): diff --git a/test/registered/disaggregation/test_disaggregation_aarch64.py b/test/registered/disaggregation/test_disaggregation_aarch64.py index 5c8630282..68889645d 100644 --- a/test/registered/disaggregation/test_disaggregation_aarch64.py +++ b/test/registered/disaggregation/test_disaggregation_aarch64.py @@ -8,12 +8,13 @@ from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, popen_launch_pd_server, ) -register_cuda_ci(est_time=300, stage="base-c", runner_config="4-gpu-gb200") +QWEN3_8B_MODEL_PATH = "Qwen/Qwen3-8B" + +register_cuda_ci(est_time=300, stage="base-c", runner_config="4-gpu-gb300") class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase): @@ -22,7 +23,7 @@ class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase): super().setUpClass() os.environ["SGLANG_MOONCAKE_CUSTOM_MEM_POOL"] = "true" os.environ["MC_FORCE_MNNVL"] = "true" - cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.model = QWEN3_8B_MODEL_PATH # Non blocking start servers cls.start_prefill() diff --git a/test/registered/utils/test_numa_utils.py b/test/registered/utils/test_numa_utils.py index a0863a62d..2653dd81f 100644 --- a/test/registered/utils/test_numa_utils.py +++ b/test/registered/utils/test_numa_utils.py @@ -10,8 +10,8 @@ from sglang.srt.utils.numa_utils import ( from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci register_cpu_ci(est_time=7, suite="base-a-test-cpu") -register_cuda_ci(est_time=10, stage="base-c", runner_config="4-gpu-gb200") -register_cuda_ci(est_time=10, stage="base-c", runner_config="8-gpu-b200") +register_cuda_ci(est_time=10, stage="base-c", runner_config="4-gpu-gb300") +register_cuda_ci(est_time=10, stage="base-c", runner_config="4-gpu-b200") class TestIsNumaAvailable(unittest.TestCase): @@ -249,36 +249,44 @@ class TestGetNumaNodeIfAvailable(unittest.TestCase): _mock_gpu.assert_not_called() -def _get_gpu_name(): +def _get_gpu_info(): try: import pynvml pynvml.nvmlInit() handle = pynvml.nvmlDeviceGetHandleByIndex(0) name = pynvml.nvmlDeviceGetName(handle) + if isinstance(name, bytes): + name = name.decode() + count = pynvml.nvmlDeviceGetCount() pynvml.nvmlShutdown() - return name + return name, count except Exception: - return "" + return "", 0 -_gpu_name = _get_gpu_name() +_gpu_name, _gpu_count = _get_gpu_info() -@unittest.skipUnless("GB200" in _gpu_name, "Requires GB200 hardware") -class TestGB200NumaTopology(unittest.TestCase): - """Hardware test validating expected NUMA topology on GB200 (2 NUMA nodes, 4 GPUs).""" +def _query_single_numa_node_for_gpu(gpu_id: int): + nodes = _query_numa_node_for_gpu(gpu_id) + if len(nodes) != 1: + raise AssertionError(f"GPU {gpu_id}: expected one NUMA node, got {nodes}") + return nodes[0] - def _make_server_args(self): - args = MagicMock() - args.numa_node = None - return args + +@unittest.skipUnless( + ("GB200" in _gpu_name or "GB300" in _gpu_name) and _gpu_count == 4, + "Requires 4-GPU Grace Blackwell hardware", +) +class TestGraceBlackwellNumaTopology(unittest.TestCase): + """Hardware test validating expected NUMA topology on 4-GPU GB200/GB300.""" def test_gpu_numa_mapping(self): + self.assertEqual(_gpu_count, 4) expected = {0: 0, 1: 0, 2: 1, 3: 1} - args = self._make_server_args() for gpu_id, expected_node in expected.items(): - result = get_numa_node_if_available(args, gpu_id) + result = _query_single_numa_node_for_gpu(gpu_id) self.assertEqual( result, expected_node, @@ -286,25 +294,23 @@ class TestGB200NumaTopology(unittest.TestCase): ) -@unittest.skipUnless("B200" in _gpu_name, "Requires B200 hardware") +@unittest.skipUnless( + "B200" in _gpu_name and _gpu_count == 4, + "Requires 4-GPU B200 hardware", +) class TestB200NumaTopology(unittest.TestCase): - """Hardware test validating expected NUMA topology on B200 (2 NUMA nodes, 8 GPUs).""" - - def _make_server_args(self): - args = MagicMock() - args.numa_node = None - return args + """Hardware test validating expected NUMA topology on 4-GPU B200.""" def test_gpu_numa_mapping(self): - expected = {0: 0, 1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1} - args = self._make_server_args() - for gpu_id, expected_node in expected.items(): - result = get_numa_node_if_available(args, gpu_id) - self.assertEqual( - result, - expected_node, - f"GPU {gpu_id}: expected NUMA node {expected_node}, got {result}", - ) + self.assertEqual(_gpu_count, 4) + numa_nodes = { + _query_single_numa_node_for_gpu(gpu_id) for gpu_id in range(_gpu_count) + } + self.assertEqual( + len(numa_nodes), + 1, + f"Expected all visible 4-GPU B200 devices on one NUMA node, got {numa_nodes}", + ) if __name__ == "__main__": diff --git a/test/run_suite.py b/test/run_suite.py index 551a36645..0fef1b85c 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -54,7 +54,7 @@ PER_COMMIT_SUITES = { "base-b-kernel-benchmark-1-gpu-large", "base-c-test-4-gpu-h100", "base-c-test-4-gpu-b200", - "base-c-test-4-gpu-gb200", + "base-c-test-4-gpu-gb300", "base-c-test-8-gpu-h20", "base-c-test-8-gpu-h200", "base-c-test-8-gpu-b200",