Add GB300 base C CI suite (#27427)

.github/workflows/_pr-test-stage.yml

@@ -119,6 +119,8 @@ jobs:
 
       - name: Install dependencies
         timeout-minutes: ${{ fromJson(steps.rc.outputs.install_timeout) }}
+        env:
+          GRACE_BLACKWELL: ${{ steps.rc.outputs.grace_blackwell || '0' }}
         run: |
           CUSTOM_BUILD_SGL_KERNEL=${{ fromJson(inputs.check_changes).sgl_kernel }} bash ${{ steps.rc.outputs.install }}
 

.github/workflows/pr-test.yml

@@ -59,8 +59,6 @@ env:
   SGLANG_CUDA_COREDUMP: "1"
   SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
   SKIP_PR_TEST_HEALTH_CHECK: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }}
-  # TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main.
-  FORCE_REBUILD_DEEPEP: '1'
   # Schedule / main-branch dispatch / workflow_call from main use refs/heads/main; PR events use refs/pull/*/merge
   PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
   USE_VENV: false
@@ -515,6 +513,20 @@ jobs:
       timeout_per_file: '1800'
     secrets: inherit
 
+  base-c-test-4-gpu-gb300:
+    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
+    if: ${{ !failure() && !cancelled() }}
+    uses: ./.github/workflows/_pr-test-stage.yml
+    with:
+      self_name: base-c-test-4-gpu-gb300
+      runner_config: 4-gpu-gb300
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
+      timeout_per_file: '1800'
+    secrets: inherit
+
   pr-test-finish:
     needs:
       [
@@ -545,7 +557,7 @@ jobs:
         base-c-test-deepep-4-gpu-b200,
         base-c-test-deepep-8-gpu-h200,
         base-c-test-4-gpu-b200,
-        # base-c-test-4-gpu-gb200,  # Temporarily disabled — no GB200 runner
+        base-c-test-4-gpu-gb300,
       ]
     if: always()
     runs-on: ubuntu-latest

.github/workflows/rerun-test.yml

@@ -31,6 +31,11 @@ on:
         required: false
         type: string
         default: "20"
+      grace_blackwell:
+        description: "Set GRACE_BLACKWELL for the install step (cuda only)"
+        required: false
+        type: string
+        default: "0"
       rdma_devices:
         description: "SGLANG_CI_RDMA_ALL_DEVICES csv (cuda only; empty = unset)"
         required: false
@@ -100,6 +105,8 @@ jobs:
 
       - name: Install dependencies
         timeout-minutes: ${{ fromJson(inputs.install_timeout) }}
+        env:
+          GRACE_BLACKWELL: ${{ inputs.grace_blackwell }}
         run: |
           if [[ "${{ inputs.runs_on }}" == "1-gpu-5090" ]]; then
             source /etc/profile.d/sglang-ci.sh

scripts/ci/runner_configs.yml

@@ -8,6 +8,8 @@
 #   - artifact_version: actions/download-artifact major version
 #   - install_timeout: install-step wall-clock cap (minutes), enforced via
 #     `timeout-minutes:` on the install step in _pr-test-stage.yml
+#   - grace_blackwell (optional): exported as GRACE_BLACKWELL for the install
+#     step. Used by GB300 DeePEP setup.
 #   - runs_on: GHA runner label for the stage's `runs-on:`. The literal
 #     `$b200_runner` is substituted at workflow-load time with the dynamic
 #     b200 runner tag from check-changes (see runner_configs.py --map).
@@ -23,6 +25,7 @@ runner_configs:
   1-gpu-large:       { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-h100 }
   2-gpu-large:       { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 2-gpu-h100 }
   4-gpu-b200:        { install: *default, artifact_version: v6, install_timeout: "20", runs_on: $b200_runner }
+  4-gpu-gb300:       { install: *deepep,  artifact_version: v6, install_timeout: "20", grace_blackwell: "1", runs_on: 4-gpu-gb300 }
   4-gpu-h100:        { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 }
   8-gpu-h200:        { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200 }
   8-gpu-b200:        { install: *default, artifact_version: v6, install_timeout: "20", runs_on: 8-gpu-b200 }

scripts/ci/utils/slash_command_handler.py

@@ -709,8 +709,9 @@ def _extract_legacy_suites(content):
 # matches the runner the nightly/weekly pipeline actually uses (see
 # .github/workflows/{nightly,weekly}-test-nvidia.yml), so /rerun-test can still
 # dispatch a single nightly/weekly test. The runner label, install script,
-# timeout and rdma_devices are then resolved from runner_configs.yml as usual,
+# timeout, grace_blackwell, and rdma_devices are then resolved from
-# keeping that file the single source of truth for runner details.
+# runner_configs.yml as usual, keeping that file the single source of truth for
+# runner details.
 #
 # Suites on hardware with no matching runner_config (e.g. nightly-4-gpu-gb300)
 # and non-CUDA suites (npu/amd) are intentionally absent and stay
@@ -740,6 +741,7 @@ def _dispatch_err(suite, msg):
         "runner_label": None,
         "install_script": "",
         "install_timeout": "",
+        "grace_blackwell": "0",
         "rdma_devices": "",
         "is_cpu": False,
         "error": msg,
@@ -780,6 +782,7 @@ def _resolve_runner_config(rc, full_path, suite):
         "runner_label": runs_on,
         "install_script": install_script,
         "install_timeout": str(cfg["install_timeout"]),
+        "grace_blackwell": str(cfg.get("grace_blackwell", "0")),
         "rdma_devices": cfg.get("rdma_devices", ""),
         "is_cpu": False,
         "error": None,
@@ -793,9 +796,9 @@ def detect_suite(file_path_from_test):
 
     A CUDA file can carry multiple `register_cuda_ci(...)` calls — one per
     pool it should run on — so this returns a *list* of dispatch dicts, one
-    per registration. Runner label, install script, timeout, and rdma_devices
+    per registration. Runner label, install script, timeout, grace_blackwell,
-    are all resolved from scripts/ci/runner_configs.yml — the same single
+    and rdma_devices are all resolved from scripts/ci/runner_configs.yml — the
-    source of truth that drives the main PR test pipeline.
+    same single source of truth that drives the main PR test pipeline.
 
     Legacy nightly/weekly CUDA suites (single-string `suite=`) are dispatchable
     too: each suite name is mapped to the matching runner_config via
@@ -806,7 +809,7 @@ def detect_suite(file_path_from_test):
     `error` set.
 
     Each dict has keys: suite, runner_label, install_script,
-    install_timeout, rdma_devices, is_cpu, error.
+    install_timeout, grace_blackwell, rdma_devices, is_cpu, error.
     """
     full_path = f"test/{file_path_from_test}"
     with open(full_path, "r") as f:
@@ -838,6 +841,7 @@ def detect_suite(file_path_from_test):
                 "runner_label": "ubuntu-latest",
                 "install_script": "",
                 "install_timeout": "",
+                "grace_blackwell": "0",
                 "rdma_devices": "",
                 "is_cpu": True,
                 "error": None,
@@ -912,6 +916,7 @@ def _resolve_test_spec(test_spec):
                 "runs_on": runner_label,
                 "install_script": "",
                 "install_timeout": "",
+                "grace_blackwell": "0",
                 "rdma_devices": "",
                 "error": None,
             }
@@ -930,7 +935,8 @@ def _resolve_test_spec(test_spec):
         print(
             f"Resolved: file={resolved_path}, selector={test_selector}, "
             f"suite={info['suite']}, mode={mode}, runs_on={info['runner_label']}, "
-            f"install={info['install_script']}, rdma={info['rdma_devices']}, "
+            f"install={info['install_script']}, grace_blackwell={info['grace_blackwell']}, "
+            f"rdma={info['rdma_devices']}, "
             f"command='{test_command}'"
         )
         out.append(
@@ -941,6 +947,7 @@ def _resolve_test_spec(test_spec):
                 "runs_on": info["runner_label"],
                 "install_script": info["install_script"],
                 "install_timeout": info["install_timeout"],
+                "grace_blackwell": info["grace_blackwell"],
                 "rdma_devices": info["rdma_devices"],
                 "error": None,
             }
@@ -952,7 +959,7 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker
     """
     Dispatch a single workflow run for a batch of resolved test specs that
     share the same dispatch shape (mode + runs_on + install_script +
-    install_timeout + rdma_devices).
+    install_timeout + grace_blackwell + rdma_devices).
 
     Returns a dict with keys: specs, success, test_commands, runner_label, run_url, error.
     """
@@ -961,6 +968,7 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker
     runs_on = batch[0]["runs_on"]
     install_script = batch[0]["install_script"]
     install_timeout = batch[0]["install_timeout"]
+    grace_blackwell = batch[0]["grace_blackwell"]
     rdma_devices = batch[0]["rdma_devices"]
 
     # Join multiple commands with newlines for the workflow to iterate over
@@ -993,6 +1001,7 @@ def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker
             "runs_on": runs_on or "",
             "install_script": install_script,
             "install_timeout": install_timeout or "20",
+            "grace_blackwell": grace_blackwell or "0",
             "rdma_devices": rdma_devices,
             "reply_comment_id": str(reply_comment_id) if reply_comment_id else "",
             "reply_marker": reply_marker,
@@ -1100,7 +1109,7 @@ def handle_rerun_test(
     """
     Handles the /rerun-test command. Resolves all test specs, groups them by
     dispatch shape (mode + runs_on + install_script + install_timeout +
-    rdma_devices), and dispatches one workflow per group.
+    grace_blackwell + rdma_devices), and dispatches one workflow per group.
     """
     if not skip_permission_check and not _check_rerun_test_permissions(
         gh_repo, pr, comment, user_perms, "rerun-test"
@@ -1195,6 +1204,7 @@ def handle_rerun_test(
             r["runs_on"],
             r["install_script"],
             r["install_timeout"],
+            r["grace_blackwell"],
             r["rdma_devices"],
         )
         groups.setdefault(key, []).append(r)

test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py

@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
     try_cached_model,
 )
 
-register_cuda_ci(est_time=1800, stage="base-c", runner_config="4-gpu-gb200")
+register_cuda_ci(est_time=1800, stage="base-c", runner_config="4-gpu-gb300")
 
 
 class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase):

test/registered/disaggregation/test_disaggregation_aarch64.py

@@ -8,12 +8,13 @@ from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
 )
 from sglang.test.test_utils import (
-    DEFAULT_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     popen_launch_pd_server,
 )
 
-register_cuda_ci(est_time=300, stage="base-c", runner_config="4-gpu-gb200")
+QWEN3_8B_MODEL_PATH = "Qwen/Qwen3-8B"
+
+register_cuda_ci(est_time=300, stage="base-c", runner_config="4-gpu-gb300")
 
 
 class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase):
@@ -22,7 +23,7 @@ class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase):
         super().setUpClass()
         os.environ["SGLANG_MOONCAKE_CUSTOM_MEM_POOL"] = "true"
         os.environ["MC_FORCE_MNNVL"] = "true"
-        cls.model = DEFAULT_MODEL_NAME_FOR_TEST
+        cls.model = QWEN3_8B_MODEL_PATH
 
         # Non blocking start servers
         cls.start_prefill()

test/registered/utils/test_numa_utils.py

@@ -10,8 +10,8 @@ from sglang.srt.utils.numa_utils import (
 from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
 
 register_cpu_ci(est_time=7, suite="base-a-test-cpu")
-register_cuda_ci(est_time=10, stage="base-c", runner_config="4-gpu-gb200")
+register_cuda_ci(est_time=10, stage="base-c", runner_config="4-gpu-gb300")
-register_cuda_ci(est_time=10, stage="base-c", runner_config="8-gpu-b200")
+register_cuda_ci(est_time=10, stage="base-c", runner_config="4-gpu-b200")
 
 
 class TestIsNumaAvailable(unittest.TestCase):
@@ -249,36 +249,44 @@ class TestGetNumaNodeIfAvailable(unittest.TestCase):
         _mock_gpu.assert_not_called()
 
 
-def _get_gpu_name():
+def _get_gpu_info():
     try:
         import pynvml
 
         pynvml.nvmlInit()
         handle = pynvml.nvmlDeviceGetHandleByIndex(0)
         name = pynvml.nvmlDeviceGetName(handle)
+        if isinstance(name, bytes):
+            name = name.decode()
+        count = pynvml.nvmlDeviceGetCount()
         pynvml.nvmlShutdown()
-        return name
+        return name, count
     except Exception:
-        return ""
+        return "", 0
 
 
-_gpu_name = _get_gpu_name()
+_gpu_name, _gpu_count = _get_gpu_info()
 
 
-@unittest.skipUnless("GB200" in _gpu_name, "Requires GB200 hardware")
+def _query_single_numa_node_for_gpu(gpu_id: int):
-class TestGB200NumaTopology(unittest.TestCase):
+    nodes = _query_numa_node_for_gpu(gpu_id)
-    """Hardware test validating expected NUMA topology on GB200 (2 NUMA nodes, 4 GPUs)."""
+    if len(nodes) != 1:
+        raise AssertionError(f"GPU {gpu_id}: expected one NUMA node, got {nodes}")
+    return nodes[0]
 
-    def _make_server_args(self):
+
-        args = MagicMock()
+@unittest.skipUnless(
-        args.numa_node = None
+    ("GB200" in _gpu_name or "GB300" in _gpu_name) and _gpu_count == 4,
-        return args
+    "Requires 4-GPU Grace Blackwell hardware",
+)
+class TestGraceBlackwellNumaTopology(unittest.TestCase):
+    """Hardware test validating expected NUMA topology on 4-GPU GB200/GB300."""
 
     def test_gpu_numa_mapping(self):
+        self.assertEqual(_gpu_count, 4)
         expected = {0: 0, 1: 0, 2: 1, 3: 1}
-        args = self._make_server_args()
         for gpu_id, expected_node in expected.items():
-            result = get_numa_node_if_available(args, gpu_id)
+            result = _query_single_numa_node_for_gpu(gpu_id)
             self.assertEqual(
                 result,
                 expected_node,
@@ -286,25 +294,23 @@ class TestGB200NumaTopology(unittest.TestCase):
             )
 
 
-@unittest.skipUnless("B200" in _gpu_name, "Requires B200 hardware")
+@unittest.skipUnless(
+    "B200" in _gpu_name and _gpu_count == 4,
+    "Requires 4-GPU B200 hardware",
+)
 class TestB200NumaTopology(unittest.TestCase):
-    """Hardware test validating expected NUMA topology on B200 (2 NUMA nodes, 8 GPUs)."""
+    """Hardware test validating expected NUMA topology on 4-GPU B200."""
-
-    def _make_server_args(self):
-        args = MagicMock()
-        args.numa_node = None
-        return args
 
     def test_gpu_numa_mapping(self):
-        expected = {0: 0, 1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1}
+        self.assertEqual(_gpu_count, 4)
-        args = self._make_server_args()
+        numa_nodes = {
-        for gpu_id, expected_node in expected.items():
+            _query_single_numa_node_for_gpu(gpu_id) for gpu_id in range(_gpu_count)
-            result = get_numa_node_if_available(args, gpu_id)
+        }
-            self.assertEqual(
+        self.assertEqual(
-                result,
+            len(numa_nodes),
-                expected_node,
+            1,
-                f"GPU {gpu_id}: expected NUMA node {expected_node}, got {result}",
+            f"Expected all visible 4-GPU B200 devices on one NUMA node, got {numa_nodes}",
-            )
+        )
 
 
 if __name__ == "__main__":

test/run_suite.py

@@ -54,7 +54,7 @@ PER_COMMIT_SUITES = {
         "base-b-kernel-benchmark-1-gpu-large",
         "base-c-test-4-gpu-h100",
         "base-c-test-4-gpu-b200",
-        "base-c-test-4-gpu-gb200",
+        "base-c-test-4-gpu-gb300",
         "base-c-test-8-gpu-h20",
         "base-c-test-8-gpu-h200",
         "base-c-test-8-gpu-b200",