From bb298936891fb6390ee43e6f7ebf635b9954cc9b Mon Sep 17 00:00:00 2001 From: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com> Date: Thu, 26 Mar 2026 10:59:36 +0800 Subject: [PATCH] [Fix] Try to fix nvcc compilation error (#21246) --- .github/workflows/nightly-test-nvidia.yml | 33 +++++++++++++++++++ .github/workflows/pr-test-jit-kernel.yml | 27 +++++++++++++++ python/sglang/jit_kernel/all_reduce.py | 2 ++ .../distributed/custom_all_reduce.cuh | 4 +-- .../tests/test_custom_all_reduce.py | 10 +++--- test/README.md | 6 +++- test/run_suite.py | 2 ++ 7 files changed, 75 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index cffd45834..f839a3072 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -27,6 +27,7 @@ on: - 'nightly-test-perf-8-gpu-b200' - 'nightly-test-kernel-1-gpu-h100' - 'nightly-test-diffusion-comparison' + - 'nightly-test-kernel-8-gpu-h200' workflow_call: inputs: ref: @@ -114,6 +115,38 @@ jobs: - uses: ./.github/actions/upload-cuda-coredumps if: always() + nightly-test-kernel-8-gpu-h200: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-kernel-8-gpu-h200') + runs-on: 8-gpu-h200 + timeout-minutes: 240 + env: + SGLANG_JIT_KERNEL_RUN_FULL_TESTS: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true + SGLANG_PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + + - name: Install dependencies + timeout-minutes: 20 + run: | + bash scripts/ci/cuda/ci_install_dependency.sh + + - name: Run multi-GPU jit kernel nightly suite + timeout-minutes: 90 + run: | + cd test + python3 run_suite.py --hw cuda --suite nightly-kernel-8-gpu-h200 --nightly --continue-on-error + + - uses: ./.github/actions/upload-cuda-coredumps + if: always() + # General tests - 4 GPU H100 nightly-test-general-4-gpu-h100: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-4-gpu-h100') diff --git a/.github/workflows/pr-test-jit-kernel.yml b/.github/workflows/pr-test-jit-kernel.yml index 4ee64e4f4..15a5df59b 100644 --- a/.github/workflows/pr-test-jit-kernel.yml +++ b/.github/workflows/pr-test-jit-kernel.yml @@ -62,6 +62,33 @@ jobs: cd test/ python3 run_suite.py --hw cuda --suite stage-b-kernel-unit-1-gpu-large + jit-kernel-multigpu-unit-test: + if: | + github.event_name != 'schedule' && + inputs.test_parallel_dispatch != 'true' && + !inputs.target_stage + runs-on: 8-gpu-h200 + timeout-minutes: 240 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + + - uses: ./.github/actions/check-maintenance + with: + github-token: ${{ github.token }} + + - name: Install dependencies + timeout-minutes: 20 + run: | + bash scripts/ci/cuda/ci_install_dependency.sh diffusion + + - name: Run multi-GPU test + timeout-minutes: 45 + run: | + cd test/ + python3 run_suite.py --hw cuda --suite stage-b-kernel-unit-8-gpu-h200 + jit-kernel-benchmark-test: if: | github.event_name != 'schedule' && diff --git a/python/sglang/jit_kernel/all_reduce.py b/python/sglang/jit_kernel/all_reduce.py index f913aa2b3..dd0210082 100644 --- a/python/sglang/jit_kernel/all_reduce.py +++ b/python/sglang/jit_kernel/all_reduce.py @@ -131,6 +131,8 @@ def get_custom_all_reduce_cls() -> type[CustomAllReduceObj]: @tvm_ffi.register_object("sgl.CustomAllReduce") class CustomAllReduceObjReal(tvm_ffi.Object): + __slots__ = ("__dict__",) + def __init__( self, rank: int, diff --git a/python/sglang/jit_kernel/include/sgl_kernel/distributed/custom_all_reduce.cuh b/python/sglang/jit_kernel/include/sgl_kernel/distributed/custom_all_reduce.cuh index aeaa58b84..7b627a9e9 100644 --- a/python/sglang/jit_kernel/include/sgl_kernel/distributed/custom_all_reduce.cuh +++ b/python/sglang/jit_kernel/include/sgl_kernel/distributed/custom_all_reduce.cuh @@ -194,8 +194,8 @@ struct CustomAllReduceBase : public tvm::ffi::Object { for (const auto j : irange(new_registered_count)) { /// NOTE: structural binding will cause intern compiler error... const auto elem = array[j]; - const auto offset = get<0>(elem); - const auto ipc_handle = get<1>(elem); + const auto offset = elem.get<0>(); + const auto ipc_handle = elem.get<1>(); data[j].input[i] = pointer::offset(open_cached(ipc_handle), offset); } } diff --git a/python/sglang/jit_kernel/tests/test_custom_all_reduce.py b/python/sglang/jit_kernel/tests/test_custom_all_reduce.py index bf0cfa38d..365761ddf 100644 --- a/python/sglang/jit_kernel/tests/test_custom_all_reduce.py +++ b/python/sglang/jit_kernel/tests/test_custom_all_reduce.py @@ -34,15 +34,13 @@ from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( - est_time=120, - suite="stage-b-kernel-unit-1-gpu-large", - disabled="requires multi-GPU distributed setup", + est_time=500, + suite="stage-b-kernel-unit-8-gpu-h200", ) register_cuda_ci( - est_time=120, - suite="nightly-kernel-1-gpu", + est_time=500, + suite="nightly-kernel-8-gpu-h200", nightly=True, - disabled="requires multi-GPU distributed setup", ) # --------------------------------------------------------------------------- diff --git a/test/README.md b/test/README.md index 374643017..26e00c45d 100644 --- a/test/README.md +++ b/test/README.md @@ -189,9 +189,11 @@ Use dedicated kernel suites: from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-large") +register_cuda_ci(est_time=120, suite="stage-b-kernel-unit-8-gpu-h200") register_cuda_ci(est_time=6, suite="stage-b-kernel-benchmark-1-gpu-large") # Optional nightly registration register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True) +register_cuda_ci(est_time=120, suite="nightly-kernel-8-gpu-h200", nightly=True) ``` Keep `est_time` and `suite` as literal values. `run_suite.py` collects them by statically parsing the file AST. @@ -210,6 +212,7 @@ You can find the available suites for each hardware backend at [`test/run_suite. | `stage-b-test-2-gpu-large` | `2-gpu-h100` | Two-GPU correctness and parallelism (TP/PP-style workloads) on H100 | | `stage-b-test-4-gpu-b200` | `4-gpu-b200` | Early Blackwell coverage (e.g. SM100+ paths) on four GPUs | | `stage-b-kernel-unit-1-gpu-large` | `1-gpu-h100` | JIT kernel correctness tests under `python/sglang/jit_kernel/tests/` | +| `stage-b-kernel-unit-8-gpu-h200` | `8-gpu-h200` | Multi-GPU JIT kernel correctness tests under `python/sglang/jit_kernel/tests/` | | `stage-b-kernel-benchmark-1-gpu-large` | `1-gpu-h100` | JIT kernel benchmark files under `python/sglang/jit_kernel/benchmark/` | | `stage-c-test-4-gpu-h100` | `4-gpu-h100` | Large 4-GPU H100 integration and scaling tests | | `stage-c-test-8-gpu-h200` | `8-gpu-h200` | Large 8-GPU H200 runs for big models and parallelism | @@ -242,6 +245,7 @@ Nightly registry suites are listed in `NIGHTLY_SUITES` in [`test/run_suite.py`]( - `nightly-1-gpu` (CUDA) - `nightly-kernel-1-gpu` (CUDA, JIT kernel full grids) +- `nightly-kernel-8-gpu-h200` (CUDA, multi-GPU JIT kernel nightly coverage) - `nightly-8-gpu-h200` (CUDA) - `nightly-eval-vlm-2-gpu` (CUDA) - `nightly-amd` (AMD) @@ -254,7 +258,7 @@ Use the lightest suite that still meets your test's needs. - Prefer the CPU suite (`stage-a-test-cpu`) when no GPU is required. - For most small GPU workloads that fit a 5090-class card in CI, use `stage-b-test-1-gpu-small`. Most tests should go here. - If you really need more GPU memory capacity or Hopper-specific features, use `stage-b-test-1-gpu-large`. -- For JIT kernel work under `python/sglang/jit_kernel/`, use `stage-b-kernel-unit-1-gpu-large` for correctness tests and `stage-b-kernel-benchmark-1-gpu-large` for benchmarks. +- For JIT kernel work under `python/sglang/jit_kernel/`, use `stage-b-kernel-unit-1-gpu-large` for single-GPU correctness tests, `stage-b-kernel-unit-8-gpu-h200` for multi-GPU correctness tests, and `stage-b-kernel-benchmark-1-gpu-large` for benchmarks. - Use multi-GPU suites only when the test actually needs multiple GPUs or other advanced multi-GPU behavior. In rare cases, if you need a new runner or custom setup, you might need to add a new suite. diff --git a/test/run_suite.py b/test/run_suite.py index 2f676cecb..b3e4b95db 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -43,6 +43,7 @@ PER_COMMIT_SUITES = { "stage-b-test-2-gpu-large", "stage-b-test-4-gpu-b200", "stage-b-kernel-unit-1-gpu-large", + "stage-b-kernel-unit-8-gpu-h200", "stage-b-kernel-benchmark-1-gpu-large", "stage-c-test-4-gpu-h100", "stage-c-test-4-gpu-b200", @@ -77,6 +78,7 @@ NIGHTLY_SUITES = { "nightly-8-gpu-b200-basic", # Basic tests for large models on B200 "nightly-8-gpu-common", # Common tests that run on both H200 and B200 "nightly-kernel-1-gpu", + "nightly-kernel-8-gpu-h200", # Eval and perf suites (2-gpu) "nightly-eval-text-2-gpu", "nightly-eval-vlm-2-gpu",