diff --git a/.github/workflows/_pr-test-check-changes.yml b/.github/workflows/_pr-test-check-changes.yml index 61cccb599..366a55286 100644 --- a/.github/workflows/_pr-test-check-changes.yml +++ b/.github/workflows/_pr-test-check-changes.yml @@ -29,8 +29,8 @@ on: value: ${{ jobs.run.outputs.partitions }} partition_model_sha: value: ${{ jobs.run.outputs.partition_model_sha }} - b200_runner: - value: ${{ jobs.run.outputs.b200_runner }} + runs_on_map: + value: ${{ jobs.run.outputs.runs_on_map }} enable_retry: value: ${{ jobs.run.outputs.enable_retry }} continue_on_error: @@ -47,7 +47,7 @@ jobs: multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} partitions: ${{ steps.partitions.outputs.partitions }} partition_model_sha: ${{ steps.partition-model-sha.outputs.sha }} - b200_runner: ${{ steps.set-runner.outputs.b200_runner }} + runs_on_map: ${{ steps.runner-map.outputs.runs_on_map }} enable_retry: ${{ steps.set-retry.outputs.enable_retry }} continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }} steps: @@ -174,6 +174,13 @@ jobs: echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT fi + - name: Build runs_on_map (resolves $b200_runner sentinel) + id: runner-map + run: | + python3 scripts/ci/runner_configs.py --map \ + '${{ steps.set-runner.outputs.b200_runner }}' \ + >> "$GITHUB_OUTPUT" + - name: Enable retry for CI id: set-retry run: | diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index 2c6e24829..90f606658 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -15,15 +15,11 @@ on: type: string required: true runner_config: - description: 'Key in scripts/ci/runner_configs.yml (install script / artifact version / install timeout).' - type: string - required: true - runs_on: - description: 'GHA runner label. B200 stages pass needs.check-changes.outputs.b200_runner for dynamic selection.' + description: 'Key in scripts/ci/runner_configs.yml. Resolves install script, artifact version, install timeout, runs-on label, and rdma_devices.' type: string required: true check_changes: - description: 'toJson(needs.check-changes.outputs). Read via fromJson(...).main_package / sgl_kernel / continue_on_error etc.' + description: 'toJson(needs.check-changes.outputs). Read via fromJson(...).main_package / sgl_kernel / continue_on_error / runs_on_map etc.' type: string required: true caller_inputs: @@ -79,11 +75,11 @@ jobs: always() && ((github.event_name == 'schedule' || fromJson(inputs.caller_inputs).test_parallel_dispatch == true) || (!failure() && !cancelled())) && (fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true') - runs-on: ${{ inputs.runs_on }} + # runs-on resolved from runs_on_map; check-changes already substituted + # $b200_runner (see runner_configs.py --map). rdma_devices is exported + # below in a setup step via $GITHUB_ENV. + runs-on: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }} timeout-minutes: 240 - env: - # Only stage-c-test-8-gpu-h20 needs the RDMA device list. - SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.runner_config == '8-gpu-h20' && 'mlx5_1,mlx5_2,mlx5_3,mlx5_4' || '' }} strategy: fail-fast: false max-parallel: ${{ fromJson(inputs.partitions)[inputs.self_name].max_parallel }} @@ -98,6 +94,9 @@ jobs: id: rc run: python3 scripts/ci/runner_configs.py '${{ inputs.runner_config }}' >> "$GITHUB_OUTPUT" + - name: Export rdma_devices to job env + run: echo "SGLANG_CI_RDMA_ALL_DEVICES=${{ steps.rc.outputs.rdma_devices || '' }}" >> "$GITHUB_ENV" + - uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-maintenance diff --git a/.github/workflows/pr-test-extra.yml b/.github/workflows/pr-test-extra.yml index 4039c7e6c..12212f894 100644 --- a/.github/workflows/pr-test-extra.yml +++ b/.github/workflows/pr-test-extra.yml @@ -107,7 +107,6 @@ jobs: with: self_name: extra-a-test-1-gpu-small runner_config: 1-gpu-small - runs_on: 1-gpu-5090 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -121,7 +120,6 @@ jobs: with: self_name: extra-a-test-1-gpu-large runner_config: 1-gpu-large - runs_on: 1-gpu-h100 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -136,7 +134,6 @@ jobs: with: self_name: extra-a-test-2-gpu-large runner_config: 2-gpu-large - runs_on: 2-gpu-h100 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -151,7 +148,6 @@ jobs: with: self_name: extra-b-test-4-gpu-h100 runner_config: 4-gpu-h100 - runs_on: 4-gpu-h100 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -165,7 +161,6 @@ jobs: with: self_name: extra-b-test-4-gpu-b200 runner_config: 4-gpu-b200 - runs_on: ${{ needs.check-changes.outputs.b200_runner }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -180,7 +175,6 @@ jobs: with: self_name: extra-b-test-8-gpu-h200 runner_config: 8-gpu-h200 - runs_on: 8-gpu-h200 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -194,7 +188,6 @@ jobs: with: self_name: extra-b-test-deepep-8-gpu-h200 runner_config: deepep-8-gpu-h200 - runs_on: 8-gpu-h200-deepep check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} diff --git a/.github/workflows/pr-test-jit-kernel.yml b/.github/workflows/pr-test-jit-kernel.yml index 74c0b90e4..9e3e597aa 100644 --- a/.github/workflows/pr-test-jit-kernel.yml +++ b/.github/workflows/pr-test-jit-kernel.yml @@ -9,7 +9,10 @@ on: sgl_kernel: required: true type: string - b200_runner: + runner_config: + required: true + type: string + runs_on_map: required: true type: string git_ref: @@ -157,7 +160,7 @@ jobs: if: | github.event_name != 'schedule' && inputs.test_parallel_dispatch != 'true' - runs-on: ${{ inputs.b200_runner }} + runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }} timeout-minutes: 240 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index 18cf49387..4ebdb2cd4 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -9,7 +9,10 @@ on: sgl_kernel: required: true type: string - b200_runner: + runner_config: + required: true + type: string + runs_on_map: required: true type: string continue_on_error: @@ -264,7 +267,7 @@ jobs: if: | ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) && inputs.multimodal_gen == 'true' - runs-on: ${{ inputs.b200_runner }} + runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }} timeout-minutes: 240 steps: - name: Checkout code diff --git a/.github/workflows/pr-test-sgl-kernel.yml b/.github/workflows/pr-test-sgl-kernel.yml index 54cfd3736..af1552954 100644 --- a/.github/workflows/pr-test-sgl-kernel.yml +++ b/.github/workflows/pr-test-sgl-kernel.yml @@ -6,7 +6,10 @@ on: sgl_kernel: required: true type: string - b200_runner: + runner_config: + required: true + type: string + runs_on_map: required: true type: string git_ref: @@ -110,7 +113,7 @@ jobs: echo "All benchmark tests completed!" sgl-kernel-b200-test: - runs-on: ${{ inputs.b200_runner }} + runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }} timeout-minutes: 240 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index a11db0c3a..d5f4d824c 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -209,8 +209,9 @@ jobs: needs.check-changes.outputs.sgl_kernel == 'true' uses: ./.github/workflows/pr-test-sgl-kernel.yml with: + runner_config: 4-gpu-b200 + runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }} sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} - b200_runner: ${{ needs.check-changes.outputs.b200_runner }} git_ref: ${{ inputs.git_ref || '' }} skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} secrets: inherit @@ -227,9 +228,10 @@ jobs: needs.check-changes.outputs.jit_kernel == 'true' uses: ./.github/workflows/pr-test-jit-kernel.yml with: + runner_config: 4-gpu-b200 + runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }} jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }} sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} - b200_runner: ${{ needs.check-changes.outputs.b200_runner }} git_ref: ${{ inputs.git_ref || '' }} test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} @@ -245,7 +247,6 @@ jobs: with: self_name: stage-a-test-1-gpu-small runner_config: 1-gpu-small - runs_on: 1-gpu-5090 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -325,7 +326,6 @@ jobs: with: self_name: stage-b-test-1-gpu-small runner_config: 1-gpu-small - runs_on: 1-gpu-5090 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -340,7 +340,6 @@ jobs: with: self_name: stage-b-test-1-gpu-large runner_config: 1-gpu-large - runs_on: 1-gpu-h100 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -355,7 +354,6 @@ jobs: with: self_name: stage-b-test-2-gpu-large runner_config: 2-gpu-large - runs_on: 2-gpu-h100 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -369,7 +367,6 @@ jobs: with: self_name: stage-b-test-4-gpu-b200 runner_config: 4-gpu-b200 - runs_on: ${{ needs.check-changes.outputs.b200_runner }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -386,9 +383,10 @@ jobs: needs.check-changes.outputs.multimodal_gen == 'true' uses: ./.github/workflows/pr-test-multimodal-gen.yml with: + runner_config: 4-gpu-b200 + runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }} multimodal_gen: ${{ needs.check-changes.outputs.multimodal_gen }} sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} - b200_runner: ${{ needs.check-changes.outputs.b200_runner }} continue_on_error: ${{ needs.check-changes.outputs.continue_on_error }} git_ref: ${{ inputs.git_ref || '' }} test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} @@ -403,7 +401,6 @@ jobs: with: self_name: stage-c-test-4-gpu-h100 runner_config: 4-gpu-h100 - runs_on: 4-gpu-h100 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -417,7 +414,6 @@ jobs: with: self_name: stage-c-test-8-gpu-h200 runner_config: 8-gpu-h200 - runs_on: 8-gpu-h200 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -438,7 +434,6 @@ jobs: with: self_name: stage-c-test-8-gpu-h20 runner_config: 8-gpu-h20 - runs_on: 8-gpu-h20 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -452,7 +447,6 @@ jobs: with: self_name: stage-c-test-deepep-4-gpu-h100 runner_config: deepep-4-gpu-h100 - runs_on: 4-gpu-h100 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -468,7 +462,6 @@ jobs: with: self_name: stage-c-test-4-gpu-b200 runner_config: 4-gpu-b200 - runs_on: ${{ needs.check-changes.outputs.b200_runner }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -483,7 +476,6 @@ jobs: with: self_name: stage-c-test-dsv4-4-gpu-b200 runner_config: dsv4-4-gpu-b200 - runs_on: ${{ needs.check-changes.outputs.b200_runner }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} @@ -498,7 +490,6 @@ jobs: with: self_name: stage-c-test-dsv4-8-gpu-h200 runner_config: dsv4-8-gpu-h200 - runs_on: 8-gpu-h200 check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} diff --git a/scripts/ci/runner_configs.py b/scripts/ci/runner_configs.py index 227dcf73b..26d9b115d 100644 --- a/scripts/ci/runner_configs.py +++ b/scripts/ci/runner_configs.py @@ -1,14 +1,24 @@ -"""Emit a runner_config's setup details (install / artifact_version / -install_timeout) in $GITHUB_OUTPUT format. Reads scripts/ci/runner_configs.yml. -Called by .github/workflows/_pr-test-stage.yml. +"""Emit runner_config setup for GitHub Actions $GITHUB_OUTPUT. + +runner_configs.py + Per-field `key=value` lines (install / artifact_version / + install_timeout / rdma_devices). `runs_on` is intentionally omitted — + it carries the `$b200_runner` sentinel and is resolved via --map. + Called per stage by _pr-test-stage.yml. + +runner_configs.py --map + `runs_on_map={json}` — flat dict {runner_config: runs_on}, with + `$b200_runner` substituted. Called once by _pr-test-check-changes.yml. """ +import json import os import sys import yaml _YAML_PATH = os.path.join(os.path.dirname(__file__), "runner_configs.yml") +_B200_SENTINEL = "$b200_runner" def load() -> dict: @@ -16,12 +26,35 @@ def load() -> dict: return yaml.safe_load(f)["runner_configs"] -if __name__ == "__main__": - if len(sys.argv) != 2: - sys.exit("usage: runner_configs.py ") - rc = sys.argv[1] - config = load().get(rc) - if config is None: +def _emit_single(rc: str) -> None: + # runs_on goes through --map (resolves $b200_runner). Suppress it here so a + # consumer can't accidentally read the raw sentinel value. + cfg = load().get(rc) + if cfg is None: sys.exit(f"unknown runner_config: {rc!r}") - for key, value in config.items(): + for key, value in cfg.items(): + if key == "runs_on": + continue print(f"{key}={value}") + + +def _emit_map(b200_runner: str) -> None: + runs_on = { + name: (b200_runner if cfg.get("runs_on") == _B200_SENTINEL else cfg["runs_on"]) + for name, cfg in load().items() + } + print(f"runs_on_map={json.dumps(runs_on, separators=(',', ':'))}") + + +if __name__ == "__main__": + args = sys.argv[1:] + if len(args) == 1: + _emit_single(args[0]) + elif len(args) == 2 and args[0] == "--map": + _emit_map(args[1]) + else: + sys.exit( + "usage:\n" + " runner_configs.py \n" + " runner_configs.py --map " + ) diff --git a/scripts/ci/runner_configs.yml b/scripts/ci/runner_configs.yml index fde798da3..2cccf432d 100644 --- a/scripts/ci/runner_configs.yml +++ b/scripts/ci/runner_configs.yml @@ -3,9 +3,16 @@ # scripts/ci/runner_configs.py (CLI wrapper), which is in turn called by # .github/workflows/_pr-test-stage.yml. # -# Each runner_config carries install script, actions/download-artifact major -# version, and install-step wall-clock cap (minutes, enforced via -# `timeout-minutes:` on the install step in _pr-test-stage.yml). +# Each runner_config carries: +# - install: install script path +# - artifact_version: actions/download-artifact major version +# - install_timeout: install-step wall-clock cap (minutes), enforced via +# `timeout-minutes:` on the install step in _pr-test-stage.yml +# - runs_on: GHA runner label for the stage's `runs-on:`. The literal +# `$b200_runner` is substituted at workflow-load time with the dynamic +# b200 runner tag from check-changes (see runner_configs.py --map). +# - rdma_devices (optional): exported as SGLANG_CI_RDMA_ALL_DEVICES env +# to the stage job; absent means unset. _anchors: default_install: &default scripts/ci/cuda/ci_install_dependency.sh @@ -13,14 +20,14 @@ _anchors: dsv4_install: &dsv4 scripts/ci/cuda/ci_install_dsv4_dep.sh runner_configs: - 1-gpu-small: { install: *default, artifact_version: v4, install_timeout: "20" } - 1-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20" } - 2-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20" } - 4-gpu-b200: { install: *default, artifact_version: v6, install_timeout: "20" } - 4-gpu-h100: { install: *default, artifact_version: v4, install_timeout: "20" } - 8-gpu-h200: { install: *default, artifact_version: v4, install_timeout: "20" } - 8-gpu-h20: { install: *deepep, artifact_version: v4, install_timeout: "20" } - deepep-4-gpu-h100: { install: *deepep, artifact_version: v4, install_timeout: "20" } - deepep-8-gpu-h200: { install: *deepep, artifact_version: v4, install_timeout: "20" } - dsv4-4-gpu-b200: { install: *dsv4, artifact_version: v6, install_timeout: "30" } - dsv4-8-gpu-h200: { install: *dsv4, artifact_version: v4, install_timeout: "30" } + 1-gpu-small: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-5090 } + 1-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-h100 } + 2-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 2-gpu-h100 } + 4-gpu-b200: { install: *default, artifact_version: v6, install_timeout: "20", runs_on: $b200_runner } + 4-gpu-h100: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 } + 8-gpu-h200: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200 } + 8-gpu-h20: { install: *deepep, artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h20, rdma_devices: "mlx5_1,mlx5_2,mlx5_3,mlx5_4" } + deepep-4-gpu-h100: { install: *deepep, artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 } + deepep-8-gpu-h200: { install: *deepep, artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200-deepep } + dsv4-4-gpu-b200: { install: *dsv4, artifact_version: v6, install_timeout: "30", runs_on: $b200_runner } + dsv4-8-gpu-h200: { install: *dsv4, artifact_version: v4, install_timeout: "30", runs_on: 8-gpu-h200 }