From 54989b1fd025d40c51774f47acf866e17a3fc3fd Mon Sep 17 00:00:00 2001 From: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Date: Fri, 12 Jun 2026 15:12:56 -0700 Subject: [PATCH] [AMD] ci: add label-gated extra-a tier (kv_canary + mock_model unit tests) (#27822) --- .github/workflows/pr-test-amd-extra.yml | 174 ++++++++++++++++++ .github/workflows/pr-test-amd-rocm720.yml | 20 ++ .github/workflows/pr-test-amd.yml | 19 ++ .../kv_canary/test_self_unit_buffer_alloc.py | 3 +- .../kv_canary/test_self_unit_endpoint.py | 3 +- .../kv_canary/test_self_unit_future_tensor.py | 3 +- .../kv_canary/test_self_unit_perturb.py | 3 +- .../kv_canary/test_self_unit_plan_input.py | 3 +- .../kv_canary/test_self_unit_pool_patcher.py | 3 +- .../test_self_unit_pool_patcher_utils.py | 3 +- .../kv_canary/test_self_unit_radix_walker.py | 3 +- ..._unit_req_to_expected_token_ids_manager.py | 3 +- .../kv_canary/test_self_unit_runner_health.py | 3 +- .../test_self_unit_runner_per_forward.py | 3 +- .../test_self_unit_runner_swa_divergence.py | 3 +- .../kv_canary/test_self_unit_runner_sweep.py | 3 +- .../test_self_unit_sweep_plan_builder.py | 3 +- .../kv_canary/test_self_unit_token_oracle.py | 3 +- .../kv_canary/test_self_unit_violation.py | 3 +- .../test_self_unit_canary_mock_wiring.py | 3 +- .../mock_model/test_self_unit_install.py | 3 +- .../mock_model/test_self_unit_oracle.py | 3 +- .../test_self_unit_oracle_torch_vs_ref.py | 3 +- .../test_self_unit_sampler_hookpoint.py | 3 +- test/run_suite.py | 8 + 25 files changed, 263 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/pr-test-amd-extra.yml diff --git a/.github/workflows/pr-test-amd-extra.yml b/.github/workflows/pr-test-amd-extra.yml new file mode 100644 index 000000000..ff86d6dd4 --- /dev/null +++ b/.github/workflows/pr-test-amd-extra.yml @@ -0,0 +1,174 @@ +name: PR Test Extra (AMD) +# Label-gated AMD extra CI workflow — the AMD mirror of pr-test-extra.yml. +# +# Adds AMD runtime to a PR only when the author opts in: the PR must carry +# BOTH `run-ci` (basic-CI prerequisite) and `run-ci-extra` (explicit opt-in). +# The label check happens at runtime in pr-gate.yml via a live +# `gh pr view`-style fetch, so reruns after adding the labels (e.g. via a +# slash command) pick up the new label set — a workflow-level `if` would read +# the frozen event payload, which never updates on rerun. The job graph also +# runs unconditionally on workflow_dispatch / workflow_call so it can be +# triggered manually or chained from the AMD scheduler. +# +# Stage: extra-a (1-gpu-small-amd). The job mirrors the container bring-up of +# pr-test-amd.yml and dispatches `run_suite.py --hw amd --suite +# extra-a-test-1-gpu-small-amd`. Only the mock-model / kv_canary *unit* tests +# are onboarded so far; the canary *e2e* tests (which would land in +# 1-/2-gpu-large) need the canary JIT kernel ported to ROCm first, so those +# suites are intentionally not registered for AMD yet. + +on: + pull_request: + # `labeled` lets the workflow re-fire when `run-ci-extra` (or `run-ci`) + # is added after the latest push. See call-gate.if for the matching guard + # that prevents unrelated label additions from dispatching a full run. + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + runner_arch: + description: 'AMD runner pool to dispatch GPU jobs to' + required: false + type: choice + default: mi325 + options: + - mi300 + - mi325 + rocm_version: + description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)' + required: false + type: choice + default: '' + options: + - '' + - rocm720 + aiter_ref: + description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' + required: false + type: string + default: '' + continue_on_error: + description: 'Continue on error (do not fail the workflow on test failures)' + required: false + type: boolean + default: false + workflow_call: + inputs: + ref: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + required: false + type: string + default: '' + rocm_version: + description: 'ROCm container variant (empty = Dockerfile default; rocm720 = ROCm 7.2.0)' + required: false + type: string + default: '' + aiter_ref: + description: 'Override AITER commit (optional, leave empty to use Dockerfile default)' + required: false + type: string + default: '' + continue_on_error: + description: 'Continue on error (do not fail the workflow on test failures)' + required: false + type: boolean + default: false + +env: + AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }} + DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }} + DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }} + +concurrency: + group: pr-test-amd-extra-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.ref || 'all' }} + cancel-in-progress: ${{ github.event_name != 'workflow_call' }} + +permissions: + actions: write + contents: read + issues: read + pull-requests: read + +jobs: + # =============================================== PR Gate ==================================================== + # Runtime live-fetch label gate (mirrors pr-test-extra.yml's call-gate): + # requires both `run-ci` and `run-ci-extra`. A failure here cascades to + # every test job via `needs`, so a PR without the labels ends in one red + # ~30s gate job plus a row of skipped jobs instead of consuming AMD runners. + # + # The job-level `if` only filters the `labeled` event type so that adding an + # unrelated label doesn't dispatch a full run; the actual label-presence + # gate is enforced at runtime inside pr-gate.yml. + call-gate: + if: | + github.event_name != 'pull_request' || + github.event.action != 'labeled' || + github.event.label.name == 'run-ci' || + github.event.label.name == 'run-ci-extra' + uses: ./.github/workflows/pr-gate.yml + with: + require-run-ci: true + require-run-ci-extra: true + secrets: inherit + + # =============================================== extra-a (1-gpu-small) =============================================== + # Single unpartitioned job: the 21 onboarded unit tests total ~233s, so the + # expensive per-job setup (container bring-up + sgl-kernel ROCm build + dep + # install, several minutes) dominates. Partitioning would multiply that + # setup across scarce AMD GPUs to shave only a couple minutes of test time, + # so one GPU running the whole suite sequentially is the better trade. + extra-a-test-1-gpu-small-amd: + name: ${{ format('extra-a-test-1-gpu-small-amd{0} (linux-{1}-1gpu-sglang)', inputs.rocm_version && format('-{0}', inputs.rocm_version) || '', inputs.runner_arch || 'mi325') }} + needs: [call-gate] + if: ${{ !cancelled() && needs.call-gate.result == 'success' }} + runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || 'mi325') }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Start CI container + # `rocm_version` (e.g. rocm720) selects an alternate ROCm container; empty uses the Dockerfile default. + run: bash scripts/ci/amd/amd_ci_start_container.sh ${{ inputs.rocm_version && format('--rocm-version {0}', inputs.rocm_version) || '' }} + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 45 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite extra-a-test-1-gpu-small-amd --timeout-per-file 1800 ${{ inputs.continue_on_error == true && '--continue-on-error' || '' }} + + # =============================================== aggregator ==================================================== + # Single fan-in job so branch protection / notifications depend on one job + # rather than every matrix leg. Fails if any dependent failed or was + # cancelled; `skipped` (e.g. PR without the opt-in labels) is not a failure. + pr-test-amd-extra-finish: + needs: + [ + call-gate, + extra-a-test-1-gpu-small-amd, + ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + json_needs='${{ toJson(needs) }}' + job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]') + for job in $job_names; do + result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result') + echo "$job: $result" + if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then + echo "The above jobs failed." + exit 1 + fi + done + echo "All jobs completed successfully" + exit 0 diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 482d0b440..0348212f9 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -203,6 +203,26 @@ jobs: - "python/pyproject_rocm.toml" - "python/pyproject_other.toml" + # =============================================== extra (scheduled) ==================================================== + # ROCm 7.2 mirror of pr-test-amd.yml's `call-pr-test-amd-extra`: chain the + # label-gated AMD extra tier into this workflow's daily schedule, but in a + # ROCm 7.2 container (`rocm_version: rocm720`). On `schedule` (and + # run_all_tests dispatch) the extra suite runs on `main` without the + # `run-ci-extra` label (pr-gate.yml only enforces labels on pull_request + # events). Targeted dispatches (target_stage set) are excluded. Not wired + # into any finish aggregator so the base rocm720 run never depends on it. + call-pr-test-amd-extra-rocm720: + if: | + (github.event_name == 'schedule' || inputs.run_all_tests == true) && + !(inputs.target_stage || inputs.target_stage_select) + uses: ./.github/workflows/pr-test-amd-extra.yml + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || '' }} + rocm_version: rocm720 + aiter_ref: ${{ inputs.aiter_ref }} + continue_on_error: true + secrets: inherit + # =============================================== sgl-kernel ==================================================== sgl-kernel-unit-test-amd-rocm720: needs: [check-changes] diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index e5991f614..23a38df39 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -187,6 +187,25 @@ jobs: - "python/pyproject_rocm.toml" - "python/pyproject_other.toml" + # =============================================== extra (scheduled) ==================================================== + # Chain the label-gated AMD extra tier into the scheduled run, mirroring + # pr-test.yml's `call-pr-test-extra`. On `schedule` (and run_all_tests + # dispatch) the extra suite runs on `main` without needing the + # `run-ci-extra` label (pr-gate.yml only enforces labels on pull_request + # events). Targeted /rerun-stage dispatches (target_stage set) are excluded. + # Not added to `pr-test-amd-finish` so the base AMD gate never depends on + # the opt-in extra suite. + call-pr-test-amd-extra: + if: | + (github.event_name == 'schedule' || inputs.run_all_tests == true) && + !(inputs.target_stage || inputs.target_stage_select) + uses: ./.github/workflows/pr-test-amd-extra.yml + with: + ref: ${{ inputs.pr_head_sha || inputs.ref || '' }} + aiter_ref: ${{ inputs.aiter_ref }} + continue_on_error: true + secrets: inherit + # =============================================== sgl-kernel ==================================================== sgl-kernel-unit-test-amd: name: ${{ format('sgl-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || 'mi325') }} diff --git a/test/registered/kv_canary/test_self_unit_buffer_alloc.py b/test/registered/kv_canary/test_self_unit_buffer_alloc.py index 97d5d8435..27c3cf55f 100644 --- a/test/registered/kv_canary/test_self_unit_buffer_alloc.py +++ b/test/registered/kv_canary/test_self_unit_buffer_alloc.py @@ -12,10 +12,11 @@ from sglang.srt.kv_canary.pool_patcher.buffer_alloc import ( make_row_source, resolve_real_kv_read_bytes, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=10, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=10, suite="extra-a-test-1-gpu-small-amd") def _config(mode: RealKvHashMode) -> CanaryConfig: diff --git a/test/registered/kv_canary/test_self_unit_endpoint.py b/test/registered/kv_canary/test_self_unit_endpoint.py index b301b27b5..8adcded2b 100644 --- a/test/registered/kv_canary/test_self_unit_endpoint.py +++ b/test/registered/kv_canary/test_self_unit_endpoint.py @@ -21,11 +21,12 @@ from sglang.srt.kv_canary.expected_inputs import ExpectedInputs from sglang.srt.kv_canary.state import ( ViolationLog, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import DEFAULT_DEVICE from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=20, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=20, suite="extra-a-test-1-gpu-small-amd") def _make_endpoint(*, device, kernel_kind=CanaryLaunchTag.HEAD_K_FULL, swa_lut=None): diff --git a/test/registered/kv_canary/test_self_unit_future_tensor.py b/test/registered/kv_canary/test_self_unit_future_tensor.py index e593cd6d0..a0256378f 100644 --- a/test/registered/kv_canary/test_self_unit_future_tensor.py +++ b/test/registered/kv_canary/test_self_unit_future_tensor.py @@ -6,10 +6,11 @@ from typing import cast import torch from sglang.srt.kv_canary.runner.future_tensor import FutureTensors -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=20, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=20, suite="extra-a-test-1-gpu-small-amd") class _FakeEvent: diff --git a/test/registered/kv_canary/test_self_unit_perturb.py b/test/registered/kv_canary/test_self_unit_perturb.py index 5f3d4ed62..8e9a240f7 100644 --- a/test/registered/kv_canary/test_self_unit_perturb.py +++ b/test/registered/kv_canary/test_self_unit_perturb.py @@ -27,7 +27,7 @@ from sglang.srt.kv_canary.perturb.utils import ( flip_first_byte_in_source, pick_target_group, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import ( DEFAULT_DEVICE, make_buffer_group, @@ -41,6 +41,7 @@ if TYPE_CHECKING: from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache register_cuda_ci(est_time=10, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=10, suite="extra-a-test-1-gpu-small-amd") class TestParseTargetGroupKind(CustomTestCase): diff --git a/test/registered/kv_canary/test_self_unit_plan_input.py b/test/registered/kv_canary/test_self_unit_plan_input.py index 937b7221b..7449b0947 100644 --- a/test/registered/kv_canary/test_self_unit_plan_input.py +++ b/test/registered/kv_canary/test_self_unit_plan_input.py @@ -6,7 +6,7 @@ from types import SimpleNamespace import torch from sglang.srt.kv_canary.plan_input import PlanInput -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import ( DEFAULT_DEVICE, make_forward_batch, @@ -14,6 +14,7 @@ from sglang.test.kv_canary.fixtures import ( from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=30, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=30, suite="extra-a-test-1-gpu-small-amd") def _make_static_plan_input(*, bs_capacity: int, device) -> PlanInput: diff --git a/test/registered/kv_canary/test_self_unit_pool_patcher.py b/test/registered/kv_canary/test_self_unit_pool_patcher.py index 859914721..19f0934e7 100644 --- a/test/registered/kv_canary/test_self_unit_pool_patcher.py +++ b/test/registered/kv_canary/test_self_unit_pool_patcher.py @@ -15,7 +15,7 @@ from sglang.jit_kernel.kv_canary.verify import ( ) from sglang.srt.kv_canary.buffer_group import PoolKind from sglang.srt.kv_canary.pool_patcher.api import attach_canary_buffers -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import ( DEFAULT_DEVICE, make_base_config, @@ -25,6 +25,7 @@ from sglang.test.kv_canary.fixtures import ( from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=45, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=45, suite="extra-a-test-1-gpu-small-amd") class PoolPatcherHelper: diff --git a/test/registered/kv_canary/test_self_unit_pool_patcher_utils.py b/test/registered/kv_canary/test_self_unit_pool_patcher_utils.py index cfb1019dc..eefa62bab 100644 --- a/test/registered/kv_canary/test_self_unit_pool_patcher_utils.py +++ b/test/registered/kv_canary/test_self_unit_pool_patcher_utils.py @@ -3,10 +3,11 @@ from __future__ import annotations import unittest from sglang.srt.kv_canary.pool_patcher.utils import wrap_method -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=10, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=10, suite="extra-a-test-1-gpu-small-amd") class _FakeObj: diff --git a/test/registered/kv_canary/test_self_unit_radix_walker.py b/test/registered/kv_canary/test_self_unit_radix_walker.py index 6ef7dce93..4caef8b8c 100644 --- a/test/registered/kv_canary/test_self_unit_radix_walker.py +++ b/test/registered/kv_canary/test_self_unit_radix_walker.py @@ -6,11 +6,12 @@ import torch from sglang.srt.kv_canary.radix_cache_walker import walk_radix_cache_for_canary from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache, TreeNode -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import DEFAULT_DEVICE, make_radix_cache from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=30, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=30, suite="extra-a-test-1-gpu-small-amd") class TestSelfUnitRadixWalker(CustomTestCase): diff --git a/test/registered/kv_canary/test_self_unit_req_to_expected_token_ids_manager.py b/test/registered/kv_canary/test_self_unit_req_to_expected_token_ids_manager.py index 518c4fbb9..a11db78c5 100644 --- a/test/registered/kv_canary/test_self_unit_req_to_expected_token_ids_manager.py +++ b/test/registered/kv_canary/test_self_unit_req_to_expected_token_ids_manager.py @@ -10,11 +10,12 @@ from sglang.srt.kv_canary.req_to_expected_token_ids_manager import ( compute_req_all_ids_info, populate_req_to_expected_token_ids, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import DEFAULT_DEVICE, make_forward_batch from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=15, suite="extra-a-test-1-gpu-small-amd") def _make_req(*, origin: list[int], output: list[int]) -> SimpleNamespace: diff --git a/test/registered/kv_canary/test_self_unit_runner_health.py b/test/registered/kv_canary/test_self_unit_runner_health.py index 7255ba60c..b033a0e55 100644 --- a/test/registered/kv_canary/test_self_unit_runner_health.py +++ b/test/registered/kv_canary/test_self_unit_runner_health.py @@ -11,7 +11,7 @@ from sglang.srt.kv_canary.config import CanaryConfig from sglang.srt.kv_canary.runner import stats_logger as stats_logger_module from sglang.srt.kv_canary.runner.health_checker import KernelRunCounterHealthChecker from sglang.srt.kv_canary.state import CanaryDeviceState -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.runner_test_base import ( CanaryManagerTestCase, make_config, @@ -20,6 +20,7 @@ from sglang.test.kv_canary.runner_test_base import ( from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=45, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=45, suite="extra-a-test-1-gpu-small-amd") class TestSelfUnitManagerHealth(CanaryManagerTestCase): diff --git a/test/registered/kv_canary/test_self_unit_runner_per_forward.py b/test/registered/kv_canary/test_self_unit_runner_per_forward.py index de5ebcf32..d99ac6eb1 100644 --- a/test/registered/kv_canary/test_self_unit_runner_per_forward.py +++ b/test/registered/kv_canary/test_self_unit_runner_per_forward.py @@ -12,7 +12,7 @@ from sglang.srt.kv_canary import endpoint as endpoint_module from sglang.srt.kv_canary.expected_inputs import ExpectedInputs from sglang.srt.kv_canary.runner import kernel_launcher as kernel_launcher_module from sglang.srt.kv_canary.state import ViolationLog -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import make_buffer_group, make_forward_batch from sglang.test.kv_canary.runner_test_base import ( CanaryManagerTestCase, @@ -21,6 +21,7 @@ from sglang.test.kv_canary.runner_test_base import ( ) register_cuda_ci(est_time=45, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=45, suite="extra-a-test-1-gpu-small-amd") class TestManagerPerForward(CanaryManagerTestCase): diff --git a/test/registered/kv_canary/test_self_unit_runner_swa_divergence.py b/test/registered/kv_canary/test_self_unit_runner_swa_divergence.py index 7d0ef0059..0e3dc80c7 100644 --- a/test/registered/kv_canary/test_self_unit_runner_swa_divergence.py +++ b/test/registered/kv_canary/test_self_unit_runner_swa_divergence.py @@ -15,12 +15,13 @@ from sglang.srt.kv_canary.runner.swa_divergence import ( SwaDivergenceReporter, compute_swa_full_idx_divergence, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import make_buffer_group from sglang.test.kv_canary.runner_test_base import CanaryManagerTestCase, make_manager from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=45, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=45, suite="extra-a-test-1-gpu-small-amd") _DEVICE = torch.device("cuda") diff --git a/test/registered/kv_canary/test_self_unit_runner_sweep.py b/test/registered/kv_canary/test_self_unit_runner_sweep.py index a2d02e1c6..49e55f018 100644 --- a/test/registered/kv_canary/test_self_unit_runner_sweep.py +++ b/test/registered/kv_canary/test_self_unit_runner_sweep.py @@ -4,7 +4,7 @@ import unittest from unittest.mock import patch from sglang.srt.kv_canary import endpoint as endpoint_module -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import ( make_forward_batch, make_radix_cache, @@ -17,6 +17,7 @@ from sglang.test.kv_canary.runner_test_base import ( ) register_cuda_ci(est_time=45, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=45, suite="extra-a-test-1-gpu-small-amd") def _run_one_cycle(manager, forward_batch) -> None: diff --git a/test/registered/kv_canary/test_self_unit_sweep_plan_builder.py b/test/registered/kv_canary/test_self_unit_sweep_plan_builder.py index b9d5c0731..2cab72e1f 100644 --- a/test/registered/kv_canary/test_self_unit_sweep_plan_builder.py +++ b/test/registered/kv_canary/test_self_unit_sweep_plan_builder.py @@ -5,7 +5,7 @@ import unittest import torch from sglang.srt.kv_canary.sweep_plan_builder import build_verify_plan_radix_sweep -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import ( DEFAULT_DEVICE, make_radix_cache, @@ -14,6 +14,7 @@ from sglang.test.kv_canary.fixtures import ( from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=30, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=30, suite="extra-a-test-1-gpu-small-amd") class TestSelfUnitSweepPlanBuilder(CustomTestCase): diff --git a/test/registered/kv_canary/test_self_unit_token_oracle.py b/test/registered/kv_canary/test_self_unit_token_oracle.py index f8866473e..fb0ebb3d4 100644 --- a/test/registered/kv_canary/test_self_unit_token_oracle.py +++ b/test/registered/kv_canary/test_self_unit_token_oracle.py @@ -9,11 +9,12 @@ from sglang.srt.kv_canary.expected_inputs import ExpectedInputs from sglang.srt.kv_canary.token_oracle.oracle import HashOracle from sglang.srt.kv_canary.token_oracle.oracle_manager import TokenOracleManager from sglang.srt.model_executor.forward_batch_info import ForwardMode -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kv_canary.fixtures import DEFAULT_DEVICE from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=1, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=1, suite="extra-a-test-1-gpu-small-amd") class TestTokenOracleManager(CustomTestCase): diff --git a/test/registered/kv_canary/test_self_unit_violation.py b/test/registered/kv_canary/test_self_unit_violation.py index 6a9ff778a..7245586a0 100644 --- a/test/registered/kv_canary/test_self_unit_violation.py +++ b/test/registered/kv_canary/test_self_unit_violation.py @@ -15,10 +15,11 @@ from sglang.srt.kv_canary.runner.violation_reporter import ( ViolationReporter, _format_violation, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=5, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=5, suite="extra-a-test-1-gpu-small-amd") def _make_row( diff --git a/test/registered/mock_model/test_self_unit_canary_mock_wiring.py b/test/registered/mock_model/test_self_unit_canary_mock_wiring.py index 7916867d7..2b3fbd78e 100644 --- a/test/registered/mock_model/test_self_unit_canary_mock_wiring.py +++ b/test/registered/mock_model/test_self_unit_canary_mock_wiring.py @@ -12,11 +12,12 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardMode, _stable_hash_str_to_i64, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.mock_model.utils import mock_model_server_args, mock_model_server_env from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=60, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=60, suite="extra-a-test-1-gpu-small-amd") @dataclasses.dataclass diff --git a/test/registered/mock_model/test_self_unit_install.py b/test/registered/mock_model/test_self_unit_install.py index 28f655b52..0bcc35feb 100644 --- a/test/registered/mock_model/test_self_unit_install.py +++ b/test/registered/mock_model/test_self_unit_install.py @@ -9,10 +9,11 @@ os.environ["SGLANG_KV_CANARY_ENABLE_TOKEN_ORACLE"] = "1" from sglang.srt.kv_canary.token_oracle.install import install_token_oracle_from_env from sglang.srt.kv_canary.token_oracle.oracle import HashOracle from sglang.srt.layers.sampler import _CUSTOM_SAMPLER_FACTORIES -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=60, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=60, suite="extra-a-test-1-gpu-small-amd") def _make_server_args(*, sampling_backend: str) -> SimpleNamespace: diff --git a/test/registered/mock_model/test_self_unit_oracle.py b/test/registered/mock_model/test_self_unit_oracle.py index fe7f02e6c..c28b723e2 100644 --- a/test/registered/mock_model/test_self_unit_oracle.py +++ b/test/registered/mock_model/test_self_unit_oracle.py @@ -10,10 +10,11 @@ from sglang.srt.kv_canary.token_oracle.oracle import ( HashOracle, _splitmix64_tensor, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=60, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=60, suite="extra-a-test-1-gpu-small-amd") _U64_MASK: int = (1 << 64) - 1 diff --git a/test/registered/mock_model/test_self_unit_oracle_torch_vs_ref.py b/test/registered/mock_model/test_self_unit_oracle_torch_vs_ref.py index 3043e66ec..c880a8d4f 100644 --- a/test/registered/mock_model/test_self_unit_oracle_torch_vs_ref.py +++ b/test/registered/mock_model/test_self_unit_oracle_torch_vs_ref.py @@ -7,10 +7,11 @@ import torch from sglang.jit_kernel.kv_canary.verify_ref import splitmix64 from sglang.srt.kv_canary.token_oracle.oracle import HashOracle -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=30, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=30, suite="extra-a-test-1-gpu-small-amd") class TestHashOracleTorchVsRef(CustomTestCase): diff --git a/test/registered/mock_model/test_self_unit_sampler_hookpoint.py b/test/registered/mock_model/test_self_unit_sampler_hookpoint.py index 2d4e2c44a..c46c07954 100644 --- a/test/registered/mock_model/test_self_unit_sampler_hookpoint.py +++ b/test/registered/mock_model/test_self_unit_sampler_hookpoint.py @@ -18,10 +18,11 @@ from sglang.srt.kv_canary.token_oracle.oracle import HashOracle from sglang.srt.kv_canary.token_oracle.sampler import install_oracle_sampler from sglang.srt.layers.sampler import _CUSTOM_SAMPLER_FACTORIES from sglang.srt.server_args import SAMPLING_BACKEND_CHOICES -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=60, stage="extra-a", runner_config="1-gpu-small") +register_amd_ci(est_time=60, suite="extra-a-test-1-gpu-small-amd") class TestInstallOracleSampler(CustomTestCase): diff --git a/test/run_suite.py b/test/run_suite.py index d9c139e64..a7a0a951b 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -43,6 +43,14 @@ PER_COMMIT_SUITES = { "stage-c-test-4-gpu-amd", "stage-c-test-large-8-gpu-amd", "stage-c-test-large-8-gpu-amd-mi35x", + # extra-a: label-gated PR opt-in suite in pr-test-amd-extra.yml + # (mirror of CUDA extra-a; tests stay tagged per-commit but only + # dispatch when the PR carries the `run-ci-extra` label). Only the + # 1-gpu-small mock-model / kv_canary *unit* tests are onboarded so + # far; the canary *e2e* tests (1-/2-gpu-large) need the canary JIT + # kernel ported to ROCm first, so those suites are intentionally + # not yet registered for AMD. + "extra-a-test-1-gpu-small-amd", ], HWBackend.MUSA: [], HWBackend.CUDA: [