From 653f6735a028ee20034ae4ce267d876b512dff9b Mon Sep 17 00:00:00 2001 From: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Date: Fri, 19 Jun 2026 19:46:34 -0700 Subject: [PATCH] [AMD] add nightly kv_canary JIT benchmark suite (#28611) --- .../workflows/nightly-test-amd-rocm720.yml | 31 ++++++++++++++++++ .github/workflows/nightly-test-amd.yml | 32 +++++++++++++++++++ .../jit/benchmark/kv_canary/bench_plan.py | 4 ++- .../kv_canary/bench_scatter_req_token_ids.py | 6 +++- .../jit/benchmark/kv_canary/bench_verify.py | 6 +++- .../jit/benchmark/kv_canary/bench_write.py | 6 +++- test/run_suite.py | 1 + 7 files changed, 82 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index a4845fb00..98ce7a342 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -29,6 +29,7 @@ on: - 'all' # 1-GPU Unit Tests (MI30x + MI35x) - nightly-test-1-gpu-unit-rocm720 + - nightly-test-1-gpu-kernel-rocm720 - nightly-test-1-gpu-mi35x-rocm720 # 2-GPU and 4-GPU Tests (MI30x + MI35x) - nightly-accuracy-2-gpu-rocm720 @@ -168,6 +169,36 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + nightly-test-1-gpu-kernel-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel-rocm720,')) + runs-on: linux-mi325-1gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + - name: Nightly Kernel Benchmark ROCm 7.2 (1-GPU) + timeout-minutes: 60 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-kernel-1-gpu --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-test-1-gpu-mi35x-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x-rocm720,')) runs-on: linux-mi35x-gpu-1 diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index f34c27257..8b2a46c1d 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -29,6 +29,7 @@ on: - 'all' # 1-GPU Unit Tests (MI30x + MI35x) - nightly-test-1-gpu-unit + - nightly-test-1-gpu-kernel - nightly-test-1-gpu-mi35x # 2-GPU and 4-GPU Tests (MI30x + MI35x) - nightly-accuracy-2-gpu @@ -165,6 +166,37 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + nightly-test-1-gpu-kernel: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-kernel,')) + runs-on: linux-mi325-1gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Ensure VRAM is clear + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd/amd_ci_install_dependency.sh + + - name: Nightly Kernel Benchmark (1-GPU) + timeout-minutes: 60 + run: | + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-kernel-1-gpu --nightly --timeout-per-file 900 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + nightly-test-1-gpu-mi35x: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-test-1-gpu-mi35x,')) runs-on: linux-mi35x-gpu-1 diff --git a/test/registered/jit/benchmark/kv_canary/bench_plan.py b/test/registered/jit/benchmark/kv_canary/bench_plan.py index c130f3799..5c8712e0e 100644 --- a/test/registered/jit/benchmark/kv_canary/bench_plan.py +++ b/test/registered/jit/benchmark/kv_canary/bench_plan.py @@ -23,9 +23,11 @@ from sglang.jit_kernel.benchmark.utils import ( from sglang.jit_kernel.kv_canary.plan import launch_canary_plan_kernels from sglang.jit_kernel.kv_canary.verify import VerifyPlan from sglang.jit_kernel.kv_canary.write import WritePlan -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=900, suite="nightly-kernel-1-gpu", nightly=True) +# AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). +register_amd_ci(est_time=900, suite="nightly-amd-kernel-1-gpu", nightly=True) _TOTAL_TOKENS_AXIS: list[int] = [256, 4096, 65536, 262144] diff --git a/test/registered/jit/benchmark/kv_canary/bench_scatter_req_token_ids.py b/test/registered/jit/benchmark/kv_canary/bench_scatter_req_token_ids.py index fcf88a112..eb4d688d2 100644 --- a/test/registered/jit/benchmark/kv_canary/bench_scatter_req_token_ids.py +++ b/test/registered/jit/benchmark/kv_canary/bench_scatter_req_token_ids.py @@ -14,9 +14,13 @@ from sglang.jit_kernel.benchmark.utils import ( from sglang.jit_kernel.kv_canary.scatter_req_token_ids import ( launch_scatter_req_token_ids_kernel, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=180, suite="nightly-kernel-1-gpu", nightly=True) +# AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). +# Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range +# (_BS_AXIS_CI/_SEQ_LEN_AXIS_CI via get_benchmark_range), same as CUDA nightly. +register_amd_ci(est_time=180, suite="nightly-amd-kernel-1-gpu", nightly=True) _BS_AXIS_FULL: list[int] = [1, 8, 64, 256] diff --git a/test/registered/jit/benchmark/kv_canary/bench_verify.py b/test/registered/jit/benchmark/kv_canary/bench_verify.py index a027d2a98..46fa62094 100644 --- a/test/registered/jit/benchmark/kv_canary/bench_verify.py +++ b/test/registered/jit/benchmark/kv_canary/bench_verify.py @@ -30,9 +30,13 @@ from sglang.jit_kernel.kv_canary.verify import ( VerifyPlan, launch_canary_verify_kernel, ) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=900, suite="nightly-kernel-1-gpu", nightly=True) +# AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). +# Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range +# (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly. +register_amd_ci(est_time=900, suite="nightly-amd-kernel-1-gpu", nightly=True) _X_NAMES = [ diff --git a/test/registered/jit/benchmark/kv_canary/bench_write.py b/test/registered/jit/benchmark/kv_canary/bench_write.py index 8dfd28d09..2aa5f6f5b 100644 --- a/test/registered/jit/benchmark/kv_canary/bench_write.py +++ b/test/registered/jit/benchmark/kv_canary/bench_write.py @@ -28,9 +28,13 @@ from sglang.jit_kernel.kv_canary.verify import ( VerifyOrWriteContext, ) from sglang.jit_kernel.kv_canary.write import WritePlan, launch_canary_write_kernel -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci(est_time=900, suite="nightly-kernel-1-gpu", nightly=True) +# AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). +# Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range +# (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly. +register_amd_ci(est_time=900, suite="nightly-amd-kernel-1-gpu", nightly=True) _X_NAMES = [ diff --git a/test/run_suite.py b/test/run_suite.py index de7d3247f..3f732711b 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -129,6 +129,7 @@ NIGHTLY_SUITES = { HWBackend.AMD: [ "nightly-amd", "nightly-amd-1-gpu", + "nightly-amd-kernel-1-gpu", "nightly-amd-1-gpu-mi35x", "nightly-amd-1-gpu-zimage-turbo", "nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2",