diff --git a/.claude/skills/add-jit-kernel/SKILL.md b/.claude/skills/add-jit-kernel/SKILL.md index e43f37064..ce6116a54 100644 --- a/.claude/skills/add-jit-kernel/SKILL.md +++ b/.claude/skills/add-jit-kernel/SKILL.md @@ -456,7 +456,8 @@ def test_scale_unsupported_dtype(): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + import sys + sys.exit(pytest.main([__file__, "-v", "-s"])) ``` --- diff --git a/.claude/skills/add-sgl-kernel/SKILL.md b/.claude/skills/add-sgl-kernel/SKILL.md index 662df875b..8f6c4639b 100644 --- a/.claude/skills/add-sgl-kernel/SKILL.md +++ b/.claude/skills/add-sgl-kernel/SKILL.md @@ -233,7 +233,8 @@ def test_scale_cpu_input(): if __name__ == "__main__": - pytest.main([__file__, "-q"]) + import sys + sys.exit(pytest.main([__file__, "-q"])) ``` --- diff --git a/.claude/skills/write-sglang-test/SKILL.md b/.claude/skills/write-sglang-test/SKILL.md index a35ddd6e1..7ae3fc29e 100644 --- a/.claude/skills/write-sglang-test/SKILL.md +++ b/.claude/skills/write-sglang-test/SKILL.md @@ -1,10 +1,12 @@ --- name: write-sglang-test -description: Guide for writing SGLang CI/UT tests following project conventions. Covers CustomTestCase, CI registration, server fixtures, model selection, mock testing, and test placement. Use when creating new tests, adding CI test cases, writing unit tests, or when the user asks to add tests for SGLang features. +description: Guide for writing SGLang CI/UT tests. Covers CustomTestCase, CI registration, server fixtures, model selection, mock testing, and test placement. Always read test/README.md for the full CI layout, how to run tests, and extra tips. Use when creating new tests, adding CI test cases, writing unit tests, or when the user asks to add tests for SGLang features. --- # Writing SGLang CI / UT Tests +**Before or while applying this skill, read [`test/README.md`](../../../test/README.md)** in the repo root’s `test/` tree. It documents the three-stage CI pipeline, suite naming, how tests are discovered and executed, file layout, and practical tips that complement the sections below. + ## Core Rules 1. **Always use `CustomTestCase`** — never raw `unittest.TestCase` @@ -44,8 +46,8 @@ Defined in `python/sglang/test/test_utils.py`: | Suite | Runner | Scenario | |-------|--------|----------| | `stage-a-test-cpu` | CPU | CPU unit tests | -| `stage-b-test-1-gpu-small` | 1× 5090 (32GB) | Small model tests | -| `stage-b-test-1-gpu-large` | 1× H100 (80GB) | 8B model tests | +| `stage-b-test-1-gpu-small` | 1× 5090 (32GB) | Small model tests (1B, 8B) | +| `stage-b-test-1-gpu-large` | 1× H100 (80GB) | Large model tests or performance related | | `stage-b-test-2-gpu-large` | 2× H100 | TP=2 tests | | `stage-c-test-4-gpu-h100` | 4× H100 | TP=4 / EP tests | | `stage-c-test-8-gpu-h200` | 8× H200 | Large-scale multi-GPU | diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index dcc6b4925..d8efa2d74 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -182,7 +182,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -228,7 +228,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -270,7 +270,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -308,7 +308,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -348,7 +348,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -386,7 +386,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -424,7 +424,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -463,7 +463,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -502,7 +502,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -542,7 +542,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' @@ -672,7 +672,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' @@ -802,7 +802,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -847,7 +847,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 @@ -888,7 +888,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Check Host RDMA Environment id: rdma_detect diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 561d1075f..f6412c697 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -180,7 +180,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -227,7 +227,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -270,7 +270,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -309,7 +309,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -350,7 +350,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -389,7 +389,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -428,7 +428,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -468,7 +468,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -508,7 +508,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -548,7 +548,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' @@ -676,7 +676,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Download artifacts if: needs.check-changes.outputs.sgl_kernel == 'true' @@ -805,7 +805,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -851,7 +851,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Start CI container run: bash scripts/ci/amd/amd_ci_start_container.sh @@ -893,7 +893,7 @@ jobs: ref: ${{ inputs.pr_head_sha || inputs.ref || github.sha }} - name: Ensure VRAM is clear - run: bash scripts/ensure_vram_clear.sh rocm + run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - name: Check Host RDMA Environment id: rdma_detect diff --git a/README.md b/README.md index 77f647551..523383d94 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Its core features include: - **Fast Runtime**: Provides efficient serving with RadixAttention for prefix caching, a zero-overhead CPU scheduler, prefill-decode disaggregation, speculative decoding, continuous batching, paged attention, tensor/pipeline/expert/data parallelism, structured outputs, chunked prefill, quantization (FP4/FP8/INT4/AWQ/GPTQ), and multi-LoRA batching. - **Broad Model Support**: Supports a wide range of language models (Llama, Qwen, DeepSeek, Kimi, GLM, GPT, Gemma, Mistral, etc.), embedding models (e5-mistral, gte, mcdse), reward models (Skywork), and diffusion models (WAN, Qwen-Image), with easy extensibility for adding new models. Compatible with most Hugging Face models and OpenAI APIs. -- **Extensive Hardware Support**: Runs on NVIDIA GPUs (GB200/B300/H100/A100/Spark), AMD GPUs (MI355/MI300), Intel Xeon CPUs, Google TPUs, Ascend NPUs, and more. +- **Extensive Hardware Support**: Runs on NVIDIA GPUs (GB200/B300/H100/A100/Spark/5090), AMD GPUs (MI355/MI300), Intel Xeon CPUs, Google TPUs, Ascend NPUs, and more. - **Active Community**: SGLang is open-source and supported by a vibrant community with widespread industry adoption, powering over 400,000 GPUs worldwide. - **RL & Post-Training Backbone**: SGLang is a proven rollout backend used for training many frontier models, with native RL integrations and adoption by well-known post-training frameworks such as [**AReaL**](https://github.com/inclusionAI/AReaL), [**Miles**](https://github.com/radixark/miles), [**slime**](https://github.com/THUDM/slime), [**Tunix**](https://github.com/google/tunix), [**verl**](https://github.com/volcengine/verl) and more. @@ -73,10 +73,10 @@ Its core features include: - [Contribution Guide](https://docs.sglang.io/developer_guide/contribution_guide.html) ## Benchmark and Performance -Learn more in the release blogs: [v0.2 blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/), [v0.3 blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/), [v0.4 blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/), [Large-scale expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/), [GB200 rack-scale parallelism](https://lmsys.org/blog/2025-09-25-gb200-part-2/). +Learn more in the release blogs: [v0.2 blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/), [v0.3 blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/), [v0.4 blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/), [Large-scale expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/), [GB200 rack-scale parallelism](https://lmsys.org/blog/2025-09-25-gb200-part-2/), [GB300 long context](https://lmsys.org/blog/2026-02-19-gb300-longctx/). ## Adoption and Sponsorship -SGLang has been deployed at large scale, generating trillions of tokens in production each day. It is trusted and adopted by a wide range of leading enterprises and institutions, including xAI, AMD, NVIDIA, Intel, LinkedIn, Cursor, Oracle Cloud, Google Cloud, Microsoft Azure, AWS, Atlas Cloud, Voltage Park, Nebius, DataCrunch, Novita, InnoMatrix, MIT, UCLA, the University of Washington, Stanford, UC Berkeley, Tsinghua University, Jam & Tea Studios, Baseten, and other major technology organizations across North America and Asia. +SGLang has been deployed at large scale, generating trillions of tokens in production each day. It is trusted and adopted by a wide range of leading enterprises and institutions, including xAI, AMD, NVIDIA, Intel, LinkedIn, Cursor, Oracle Cloud, Google Cloud, Microsoft Azure, AWS, Atlas Cloud, Voltage Park, Nebius, DataCrunch, Novita, InnoMatrix, MIT, UCLA, the University of Washington, Stanford, UC Berkeley, Tsinghua University, Jam & Tea Studios, Baseten, and other major technology organizations. As an open-source LLM inference engine, SGLang has become the de facto industry standard, with deployments running on over 400,000 GPUs worldwide. SGLang is currently hosted under the non-profit open-source organization [LMSYS](https://lmsys.org/about/). diff --git a/docker/xpu.Dockerfile b/docker/xpu.Dockerfile index 3f707c282..90a2414da 100644 --- a/docker/xpu.Dockerfile +++ b/docker/xpu.Dockerfile @@ -48,7 +48,7 @@ RUN --mount=type=secret,id=github_token \ cd /home/sdp && \ . /home/sdp/miniforge3/bin/activate && \ conda activate py${PYTHON_VERSION} && \ - pip3 install torch==2.10.0+xpu torchao torchvision torchaudio triton-xpu==3.6.0 --index-url https://download.pytorch.org/whl/xpu + pip3 install torch==2.10.0+xpu torchao torchvision torchaudio==2.10.0+xpu triton-xpu==3.6.0 --index-url https://download.pytorch.org/whl/xpu RUN --mount=type=secret,id=github_token \ cd /home/sdp && \ diff --git a/docs/index.rst b/docs/index.rst index 60f52afc6..665942a48 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,7 +14,7 @@ Its core features include: - **Fast Runtime**: Provides efficient serving with RadixAttention for prefix caching, a zero-overhead CPU scheduler, prefill-decode disaggregation, speculative decoding, continuous batching, paged attention, tensor/pipeline/expert/data parallelism, structured outputs, chunked prefill, quantization (FP4/FP8/INT4/AWQ/GPTQ), and multi-LoRA batching. - **Broad Model Support**: Supports a wide range of language models (Llama, Qwen, DeepSeek, Kimi, GLM, GPT, Gemma, Mistral, etc.), embedding models (e5-mistral, gte, mcdse), reward models (Skywork), and diffusion models (WAN, Qwen-Image), with easy extensibility for adding new models. Compatible with most Hugging Face models and OpenAI APIs. -- **Extensive Hardware Support**: Runs on NVIDIA GPUs (GB200/B300/H100/A100/Spark), AMD GPUs (MI355/MI300), Intel Xeon CPUs, Google TPUs, Ascend NPUs, and more. +- **Extensive Hardware Support**: Runs on NVIDIA GPUs (GB200/B300/H100/A100/Spark/5090), AMD GPUs (MI355/MI300), Intel Xeon CPUs, Google TPUs, Ascend NPUs, and more. - **Active Community**: SGLang is open-source and supported by a vibrant community with widespread industry adoption, powering over 400,000 GPUs worldwide. - **RL & Post-Training Backbone**: SGLang is a proven rollout backend used for training many frontier models, with native RL integrations and adoption by well-known post-training frameworks such as AReaL, Miles, slime, Tunix, verl and more. diff --git a/python/sglang/jit_kernel/tests/test_add_constant.py b/python/sglang/jit_kernel/tests/test_add_constant.py index 36ea024ba..8379c18b1 100644 --- a/python/sglang/jit_kernel/tests/test_add_constant.py +++ b/python/sglang/jit_kernel/tests/test_add_constant.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch @@ -13,4 +15,4 @@ def test_add_constant(size: int, constant: int) -> None: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_awq_dequantize.py b/python/sglang/jit_kernel/tests/test_awq_dequantize.py index d2970e99b..d90fafa14 100644 --- a/python/sglang/jit_kernel/tests/test_awq_dequantize.py +++ b/python/sglang/jit_kernel/tests/test_awq_dequantize.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -161,4 +162,4 @@ def test_awq_dequantize_jit_vs_aot( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_awq_marlin_moe_repack.py b/python/sglang/jit_kernel/tests/test_awq_marlin_moe_repack.py index 197e11372..a7ff242df 100644 --- a/python/sglang/jit_kernel/tests/test_awq_marlin_moe_repack.py +++ b/python/sglang/jit_kernel/tests/test_awq_marlin_moe_repack.py @@ -1,3 +1,5 @@ +import sys + import numpy as np import pytest import torch @@ -116,4 +118,4 @@ def test_awq_marlin_moe_repack_shape( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_awq_marlin_repack.py b/python/sglang/jit_kernel/tests/test_awq_marlin_repack.py index 23f2720d4..a3ee38808 100644 --- a/python/sglang/jit_kernel/tests/test_awq_marlin_repack.py +++ b/python/sglang/jit_kernel/tests/test_awq_marlin_repack.py @@ -1,3 +1,5 @@ +import sys + import numpy as np import pytest import torch @@ -102,4 +104,4 @@ def test_awq_marlin_repack_correct(num_bits, k_tiles, n_tiles, group_size): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_clamp_position.py b/python/sglang/jit_kernel/tests/test_clamp_position.py index 4cabc7a4b..3fdc8c426 100644 --- a/python/sglang/jit_kernel/tests/test_clamp_position.py +++ b/python/sglang/jit_kernel/tests/test_clamp_position.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch @@ -37,4 +39,4 @@ class TestClampPosition: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_concat_mla.py b/python/sglang/jit_kernel/tests/test_concat_mla.py index cf1013c5e..45f35e248 100644 --- a/python/sglang/jit_kernel/tests/test_concat_mla.py +++ b/python/sglang/jit_kernel/tests/test_concat_mla.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -166,4 +167,4 @@ def test_concat_mla_absorb_q_jit_vs_aot(dim_0: int, dim_1: int) -> None: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_cutedsl_gdn.py b/python/sglang/jit_kernel/tests/test_cutedsl_gdn.py index 2b06ede6f..fc7a8e19b 100644 --- a/python/sglang/jit_kernel/tests/test_cutedsl_gdn.py +++ b/python/sglang/jit_kernel/tests/test_cutedsl_gdn.py @@ -1,5 +1,7 @@ """Tests for CuTe DSL fused sigmoid gating delta rule kernel (GDN).""" +import sys + import numpy as np import pytest import torch @@ -302,4 +304,4 @@ def test_cutedsl_gdn_performance(B: int): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_flash_attention_4.py b/python/sglang/jit_kernel/tests/test_flash_attention_4.py index 1540d4601..9785a2ec1 100644 --- a/python/sglang/jit_kernel/tests/test_flash_attention_4.py +++ b/python/sglang/jit_kernel/tests/test_flash_attention_4.py @@ -4,6 +4,7 @@ import itertools import math +import sys import pytest import torch @@ -1501,4 +1502,4 @@ def _generate_block_kvcache( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_fused_add_rmsnorm.py b/python/sglang/jit_kernel/tests/test_fused_add_rmsnorm.py index fb17dcd39..a16b2650c 100644 --- a/python/sglang/jit_kernel/tests/test_fused_add_rmsnorm.py +++ b/python/sglang/jit_kernel/tests/test_fused_add_rmsnorm.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -58,4 +59,4 @@ def test_fused_add_rmsnorm(batch_size: int, hidden_size: int) -> None: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_fused_metadata_copy.py b/python/sglang/jit_kernel/tests/test_fused_metadata_copy.py index 9cffd8d88..cc0f05904 100644 --- a/python/sglang/jit_kernel/tests/test_fused_metadata_copy.py +++ b/python/sglang/jit_kernel/tests/test_fused_metadata_copy.py @@ -8,6 +8,7 @@ This test suite verifies: 4. Performance benchmarks and speedup measurements """ +import sys import time import pytest @@ -1064,4 +1065,4 @@ def test_fused_metadata_copy_multi_large_batch(bs): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_fused_norm_scale_shift.py b/python/sglang/jit_kernel/tests/test_fused_norm_scale_shift.py index 592103b12..c54fcfd5a 100644 --- a/python/sglang/jit_kernel/tests/test_fused_norm_scale_shift.py +++ b/python/sglang/jit_kernel/tests/test_fused_norm_scale_shift.py @@ -1,3 +1,4 @@ +import sys from typing import Optional, Tuple import pytest @@ -233,4 +234,4 @@ class TestFusedScaleResidualNormScaleShift: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_fused_store_index_cache.py b/python/sglang/jit_kernel/tests/test_fused_store_index_cache.py index 00edb5819..4f38c558d 100644 --- a/python/sglang/jit_kernel/tests/test_fused_store_index_cache.py +++ b/python/sglang/jit_kernel/tests/test_fused_store_index_cache.py @@ -16,6 +16,7 @@ Design Notes: from __future__ import annotations +import sys from typing import Optional, Tuple import pytest @@ -450,4 +451,4 @@ def test_reference_writes_nonzero(): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_fused_verify_triton_gdn.py b/python/sglang/jit_kernel/tests/test_fused_verify_triton_gdn.py index a6e048a40..1545f3e6b 100644 --- a/python/sglang/jit_kernel/tests/test_fused_verify_triton_gdn.py +++ b/python/sglang/jit_kernel/tests/test_fused_verify_triton_gdn.py @@ -6,6 +6,8 @@ the reference two-step implementation: 2. o = fused_recurrent_gated_delta_rule_update(q, k, v, g, beta, ...) """ +import sys + import pytest import torch @@ -228,4 +230,4 @@ def test_mtp_single_step_decode(N: int): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_gptq_marlin.py b/python/sglang/jit_kernel/tests/test_gptq_marlin.py index c7cdb1e6c..ff2b2c13c 100644 --- a/python/sglang/jit_kernel/tests/test_gptq_marlin.py +++ b/python/sglang/jit_kernel/tests/test_gptq_marlin.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel.scalar_type import scalar_types @@ -96,4 +98,4 @@ def test_gptq_marlin_gemm( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_gptq_marlin_repack.py b/python/sglang/jit_kernel/tests/test_gptq_marlin_repack.py index 0c571dbff..71ab5aafa 100644 --- a/python/sglang/jit_kernel/tests/test_gptq_marlin_repack.py +++ b/python/sglang/jit_kernel/tests/test_gptq_marlin_repack.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel.scalar_type import scalar_types @@ -87,4 +89,4 @@ def test_gptq_marlin_repack( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_hadamard_jit.py b/python/sglang/jit_kernel/tests/test_hadamard_jit.py index c7fa1b491..daaa7addb 100644 --- a/python/sglang/jit_kernel/tests/test_hadamard_jit.py +++ b/python/sglang/jit_kernel/tests/test_hadamard_jit.py @@ -1,4 +1,5 @@ import math +import sys import numpy as np import pytest @@ -420,4 +421,4 @@ def test_hadamard_transform_40n(dim, dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/python/sglang/jit_kernel/tests/test_moe_lora_align_block_size.py b/python/sglang/jit_kernel/tests/test_moe_lora_align_block_size.py index 8a19baefa..415d2bf5a 100644 --- a/python/sglang/jit_kernel/tests/test_moe_lora_align_block_size.py +++ b/python/sglang/jit_kernel/tests/test_moe_lora_align_block_size.py @@ -1,5 +1,6 @@ # Temporarily adapted from https://github.com/vllm-project/vllm/blob/main/tests/lora/test_moe_lora_align_sum.py, will optimize in future refactor import random +import sys import pytest import torch @@ -163,4 +164,4 @@ def test_moe_lora_align_block_size( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/python/sglang/jit_kernel/tests/test_moe_wna16_marlin.py b/python/sglang/jit_kernel/tests/test_moe_wna16_marlin.py index d3dc541d1..90cb99a6f 100644 --- a/python/sglang/jit_kernel/tests/test_moe_wna16_marlin.py +++ b/python/sglang/jit_kernel/tests/test_moe_wna16_marlin.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -335,4 +336,4 @@ def test_moe_wna16_marlin_gemm( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_norm_jit.py b/python/sglang/jit_kernel/tests/test_norm_jit.py index 2df0ab8ff..1f8b41e6f 100644 --- a/python/sglang/jit_kernel/tests/test_norm_jit.py +++ b/python/sglang/jit_kernel/tests/test_norm_jit.py @@ -1,5 +1,7 @@ # Adapted from sgl-kernel/tests/test_norm.py +import sys + import pytest import torch @@ -135,4 +137,4 @@ def test_rmsnorm_rejects_unsupported_hidden_size(hidden_size): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/python/sglang/jit_kernel/tests/test_per_tensor_quant_fp8.py b/python/sglang/jit_kernel/tests/test_per_tensor_quant_fp8.py index b560127ae..ecc79d67b 100644 --- a/python/sglang/jit_kernel/tests/test_per_tensor_quant_fp8.py +++ b/python/sglang/jit_kernel/tests/test_per_tensor_quant_fp8.py @@ -1,4 +1,5 @@ import itertools +import sys from typing import Optional, Tuple import pytest @@ -83,4 +84,4 @@ def test_jit_per_tensor_quant_supports_3d(shape): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_per_token_group_quant_8bit.py b/python/sglang/jit_kernel/tests/test_per_token_group_quant_8bit.py index eebd49527..dde8ccc6d 100644 --- a/python/sglang/jit_kernel/tests/test_per_token_group_quant_8bit.py +++ b/python/sglang/jit_kernel/tests/test_per_token_group_quant_8bit.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -202,4 +203,4 @@ def test_per_token_group_quant_with_column_major( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_pos_enc.py b/python/sglang/jit_kernel/tests/test_pos_enc.py index fc973f9b9..2bfee5619 100644 --- a/python/sglang/jit_kernel/tests/test_pos_enc.py +++ b/python/sglang/jit_kernel/tests/test_pos_enc.py @@ -1,3 +1,4 @@ +import sys import time from typing import Optional, Tuple, Union @@ -486,4 +487,4 @@ def test_performance( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_qknorm.py b/python/sglang/jit_kernel/tests/test_qknorm.py index 72c574c71..5bd16b4f3 100644 --- a/python/sglang/jit_kernel/tests/test_qknorm.py +++ b/python/sglang/jit_kernel/tests/test_qknorm.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -93,4 +94,4 @@ def test_qknorm(batch_size: int, n_k: int, n_q: int, head_dim: int) -> None: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_qknorm_across_heads.py b/python/sglang/jit_kernel/tests/test_qknorm_across_heads.py index fb2123537..bf9e4991f 100644 --- a/python/sglang/jit_kernel/tests/test_qknorm_across_heads.py +++ b/python/sglang/jit_kernel/tests/test_qknorm_across_heads.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -75,4 +76,4 @@ def test_qknorm_across_heads(batch_size: int, hidden_dim: int) -> None: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_qwen_image_modulation.py b/python/sglang/jit_kernel/tests/test_qwen_image_modulation.py index 20ed90078..f536964f5 100644 --- a/python/sglang/jit_kernel/tests/test_qwen_image_modulation.py +++ b/python/sglang/jit_kernel/tests/test_qwen_image_modulation.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch import triton @@ -217,4 +219,4 @@ def test_fused_residual_layernorm_scale_shift_gate_select01( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_renorm.py b/python/sglang/jit_kernel/tests/test_renorm.py index a25a8b1f6..870b46db7 100644 --- a/python/sglang/jit_kernel/tests/test_renorm.py +++ b/python/sglang/jit_kernel/tests/test_renorm.py @@ -1,6 +1,8 @@ # Adapted from https://github.com/flashinfer-ai/flashinfer/blob/main/tests/test_sampling.py # and /sgl-workspace/sglang/sgl-kernel/tests/test_sampling.py +import sys + import pytest import sgl_kernel import torch @@ -115,4 +117,4 @@ def test_top_k_mask_logits(batch_size, vocab_size, k, neginf_input): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/python/sglang/jit_kernel/tests/test_resolve_future_token_ids.py b/python/sglang/jit_kernel/tests/test_resolve_future_token_ids.py index 1e10ceb72..ad3225720 100644 --- a/python/sglang/jit_kernel/tests/test_resolve_future_token_ids.py +++ b/python/sglang/jit_kernel/tests/test_resolve_future_token_ids.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch @@ -60,4 +62,4 @@ class TestResolveFutureTokenIds: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_rmsnorm.py b/python/sglang/jit_kernel/tests/test_rmsnorm.py index d7bdb9543..138fece21 100644 --- a/python/sglang/jit_kernel/tests/test_rmsnorm.py +++ b/python/sglang/jit_kernel/tests/test_rmsnorm.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -44,4 +45,4 @@ def test_rmsnorm(batch_size: int, hidden_size: int) -> None: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_rope.py b/python/sglang/jit_kernel/tests/test_rope.py index fa2ae97b5..abc482f5c 100644 --- a/python/sglang/jit_kernel/tests/test_rope.py +++ b/python/sglang/jit_kernel/tests/test_rope.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch import triton @@ -248,4 +250,4 @@ def test_fused_rope_store( if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_store_cache.py b/python/sglang/jit_kernel/tests/test_store_cache.py index 4f597a14a..cd6887042 100644 --- a/python/sglang/jit_kernel/tests/test_store_cache.py +++ b/python/sglang/jit_kernel/tests/test_store_cache.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -124,4 +125,4 @@ def test_can_use_store_cache() -> None: if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/jit_kernel/tests/test_timestep_embedding.py b/python/sglang/jit_kernel/tests/test_timestep_embedding.py index 945fa1f13..bc900291a 100644 --- a/python/sglang/jit_kernel/tests/test_timestep_embedding.py +++ b/python/sglang/jit_kernel/tests/test_timestep_embedding.py @@ -1,4 +1,5 @@ import os +import sys import numpy as np import pytest @@ -175,4 +176,4 @@ def test_timestep_embedding_perf(): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/add-cuda-kernel.md b/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/add-cuda-kernel.md index 48290a7dc..87d47a334 100644 --- a/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/add-cuda-kernel.md +++ b/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/add-cuda-kernel.md @@ -420,7 +420,8 @@ def test_rmsnorm_correctness(dtype, shape, has_weight): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + import sys + sys.exit(pytest.main([__file__, "-v", "-s"])) ``` --- diff --git a/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/add-triton-kernel.md b/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/add-triton-kernel.md index 92f105709..672c165cf 100644 --- a/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/add-triton-kernel.md +++ b/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/add-triton-kernel.md @@ -292,7 +292,8 @@ def test_fused_scale_shift_output_dtype_preserved(): if __name__ == "__main__": - pytest.main([__file__, "-v"]) + import sys + sys.exit(pytest.main([__file__, "-v"])) ``` Run: diff --git a/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/references/kernel-templates.md b/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/references/kernel-templates.md index 899a6347c..dec2acf52 100644 --- a/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/references/kernel-templates.md +++ b/python/sglang/multimodal_gen/.claude/skills/diffusion-kernel/references/kernel-templates.md @@ -502,7 +502,8 @@ def test__cpu_error(): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + import sys + sys.exit(pytest.main([__file__, "-v", "-s"])) ``` --- diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_flux_pipeline.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_flux_pipeline.py index 37d67ecdf..5b62c8385 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_flux_pipeline.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_flux_pipeline.py @@ -1,6 +1,7 @@ """Test for ComfyUIFluxPipeline with pass-through scheduler.""" import os +import sys import pytest import torch @@ -159,4 +160,4 @@ def test_comfyui_flux_pipeline_direct() -> None: if __name__ == "__main__": - pytest.main([__file__, "-v"]) + sys.exit(pytest.main([__file__, "-v"])) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_edit_pipeline.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_edit_pipeline.py index e36883c2a..98e8025e5 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_edit_pipeline.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_edit_pipeline.py @@ -1,6 +1,7 @@ """Test for ComfyUIQwenImageEditPipeline with pass-through scheduler (I2I/edit mode).""" import os +import sys import pytest import torch @@ -132,4 +133,4 @@ def test_comfyui_qwen_image_edit_pipeline_direct() -> None: if __name__ == "__main__": - pytest.main([__file__, "-v"]) + sys.exit(pytest.main([__file__, "-v"])) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_pipeline.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_pipeline.py index 43613fa0a..f3c12b2b0 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_pipeline.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_qwen_image_pipeline.py @@ -1,6 +1,7 @@ """Test for ComfyUIQwenImagePipeline with pass-through scheduler.""" import os +import sys import pytest import torch @@ -116,4 +117,4 @@ def test_comfyui_qwen_image_pipeline_direct() -> None: if __name__ == "__main__": - pytest.main([__file__, "-v"]) + sys.exit(pytest.main([__file__, "-v"])) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_zimage_pipeline.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_zimage_pipeline.py index 8ed1308f3..a928275fd 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_zimage_pipeline.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/test/test_zimage_pipeline.py @@ -1,6 +1,7 @@ """Test for ComfyUIZImagePipeline with pass-through scheduler.""" import os +import sys import pytest import torch @@ -118,4 +119,4 @@ def test_comfyui_zimage_pipeline_direct() -> None: if __name__ == "__main__": - pytest.main([__file__, "-v"]) + sys.exit(pytest.main([__file__, "-v"])) diff --git a/python/sglang/multimodal_gen/test/server/test_update_weights_from_disk.py b/python/sglang/multimodal_gen/test/server/test_update_weights_from_disk.py index ac3c8d7ff..69f34d075 100644 --- a/python/sglang/multimodal_gen/test/server/test_update_weights_from_disk.py +++ b/python/sglang/multimodal_gen/test/server/test_update_weights_from_disk.py @@ -139,6 +139,7 @@ import functools import os import random import shutil +import sys import tempfile import threading from collections.abc import Callable @@ -671,4 +672,4 @@ class TestUpdateWeightsFromDiskWithOffload(_UpdateWeightsApiMixin): if __name__ == "__main__": - pytest.main([__file__, "-v", "-s"]) + sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 65a08781d..a57cc863f 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -1094,13 +1094,6 @@ class ModelConfig: f"or model type {self.hf_config.model_type}. " "Please upgrade transformers to >= 5.0.0." ) - elif not needs_tf_v5: - logger.warning( - f"Transformers version {tf_version_str} is used for model type {self.hf_config.model_type}. " - "If you experience issues related to RoPE parameters, " - "they may be due to incompatibilities between Transformers >=5.0.0 and some models. " - "You can try downgrading to transformers==4.57.1 as a workaround." - ) def _get_hf_eos_token_id(self) -> Optional[Set[int]]: eos_ids = getattr(self.hf_config, "eos_token_id", None) diff --git a/python/sglang/srt/layers/quantization/fp8_kernel.py b/python/sglang/srt/layers/quantization/fp8_kernel.py index c5198eca4..24a1691d2 100644 --- a/python/sglang/srt/layers/quantization/fp8_kernel.py +++ b/python/sglang/srt/layers/quantization/fp8_kernel.py @@ -42,6 +42,7 @@ from sglang.srt.utils import ( log_info_on_rank0, ) from sglang.srt.utils.custom_op import register_custom_op +from sglang.srt.utils.patch_torch import register_fake_if_exists _is_hip = is_hip() _is_cuda = is_cuda() @@ -2058,7 +2059,7 @@ def triton_scaled_mm( if _is_cuda: if enable_sgl_per_token_group_quant_8bit: - @torch.library.register_fake("sgl_kernel::sgl_per_token_group_quant_8bit") + @register_fake_if_exists("sgl_kernel::sgl_per_token_group_quant_8bit") def _( input, output_q, output_s, group_size, eps, fp8_min, fp8_max, scale_ue8m0 ): @@ -2066,12 +2067,12 @@ if _is_cuda: else: - @torch.library.register_fake("sgl_kernel::sgl_per_token_group_quant_fp8") + @register_fake_if_exists("sgl_kernel::sgl_per_token_group_quant_fp8") def _( input, output_q, output_s, group_size, eps, fp8_min, fp8_max, scale_ue8m0 ): return - @torch.library.register_fake("sgl_kernel::sgl_per_token_quant_fp8") + @register_fake_if_exists("sgl_kernel::sgl_per_token_quant_fp8") def _(input, output_q, output_s): return diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 395549abe..3f26b736a 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -92,14 +92,16 @@ if _use_aiter: if _is_cuda: from sgl_kernel import fp8_blockwise_scaled_mm, fp8_scaled_mm - @torch.library.register_fake("sgl_kernel::fp8_scaled_mm") + from sglang.srt.utils.patch_torch import register_fake_if_exists + + @register_fake_if_exists("sgl_kernel::fp8_scaled_mm") def _fp8_scaled_mm_abstract(mat_a, mat_b, scales_a, scales_b, out_dtype, bias=None): # mat_a: [M, K], mat_b: [K, N] or [N, K] depending on callsite layout; output is [M, N]. M = mat_a.shape[-2] N = mat_b.shape[-1] return mat_a.new_empty((M, N), dtype=out_dtype) - @torch.library.register_fake("sgl_kernel::fp8_blockwise_scaled_mm") + @register_fake_if_exists("sgl_kernel::fp8_blockwise_scaled_mm") def _fp8_blockwise_scaled_mm_abstract(mat_a, mat_b, scales_a, scales_b, out_dtype): # mat_a: [M, K], mat_b: [K, N] or [N, K] depending on callsite layout; output is [M, N]. M = mat_a.shape[-2] diff --git a/python/sglang/srt/mem_cache/hisparse_memory_pool.py b/python/sglang/srt/mem_cache/hisparse_memory_pool.py index 4781fde76..5af8d257a 100644 --- a/python/sglang/srt/mem_cache/hisparse_memory_pool.py +++ b/python/sglang/srt/mem_cache/hisparse_memory_pool.py @@ -5,17 +5,26 @@ from typing import Optional import torch -from sglang.srt.utils import is_npu - -if not is_npu(): - from sgl_kernel.kvcacheio import transfer_kv_all_layer_mla - from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.mem_cache.allocator import ( BaseTokenToKVPoolAllocator, PagedTokenToKVPoolAllocator, ) from sglang.srt.mem_cache.memory_pool import NSATokenToKVPool +from sglang.srt.utils import is_cuda, is_hip + +# sgl_kernel.kvcacheio is only available in CUDA/ROCm sgl-kernel builds (not XPU/MPS/NPU/CPU). +_is_cuda = is_cuda() +_is_hip = is_hip() +if _is_cuda or _is_hip: + from sgl_kernel.kvcacheio import transfer_kv_all_layer_mla +else: + + def transfer_kv_all_layer_mla(*args, **kwargs): + raise RuntimeError( + "HiSparse device KV transfer requires sgl_kernel.kvcacheio (CUDA/ROCm). " + "It is not available on this backend." + ) class HiSparseNSATokenToKVPool(NSATokenToKVPool): diff --git a/python/sglang/srt/mem_cache/storage/hf3fs/test_hf3fs_utils.py b/python/sglang/srt/mem_cache/storage/hf3fs/test_hf3fs_utils.py index 365effdef..82d93b07e 100644 --- a/python/sglang/srt/mem_cache/storage/hf3fs/test_hf3fs_utils.py +++ b/python/sglang/srt/mem_cache/storage/hf3fs/test_hf3fs_utils.py @@ -1,4 +1,5 @@ import multiprocessing.shared_memory +import sys from pathlib import Path import pytest @@ -40,4 +41,4 @@ def test_rw_shm(): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/python/sglang/srt/utils/hf_transformers_utils.py b/python/sglang/srt/utils/hf_transformers_utils.py index bafff0464..318b2e186 100644 --- a/python/sglang/srt/utils/hf_transformers_utils.py +++ b/python/sglang/srt/utils/hf_transformers_utils.py @@ -797,11 +797,6 @@ def get_tokenizer( # when trust_remote_code=False and the model requires a custom tokenizer. # Detect this and auto-retry with trust_remote_code=True. if not trust_remote_code and type(tokenizer).__name__ == "TokenizersBackend": - logger.info( - "Detected generic TokenizersBackend for %s, " - "retrying with trust_remote_code=True", - tokenizer_name, - ) tokenizer = AutoTokenizer.from_pretrained( tokenizer_name, *args, diff --git a/python/sglang/test/ci/ci_register.py b/python/sglang/test/ci/ci_register.py index 5c564e416..385373510 100644 --- a/python/sglang/test/ci/ci_register.py +++ b/python/sglang/test/ci/ci_register.py @@ -31,10 +31,14 @@ class HWBackend(Enum): class CIRegistry: backend: HWBackend filename: str + # Estimated time to run the test in seconds. est_time: float + # The suite this test is registered in. suite: str + # Whether the test is a nightly test. nightly: bool = False - disabled: Optional[str] = None # None = enabled, string = disabled with reason + # Reason for disabling the test. None = enabled, string = disabled with reason. + disabled: Optional[str] = None def register_cpu_ci( diff --git a/python/sglang/test/test_custom_ops.py b/python/sglang/test/test_custom_ops.py index c07c95db6..f360c5d82 100644 --- a/python/sglang/test/test_custom_ops.py +++ b/python/sglang/test/test_custom_ops.py @@ -1,5 +1,7 @@ # Adapted from https://github.com/vllm-project/vllm/blob/8ca7a71df787ad711ad3ac70a5bd2eb2bb398938/tests/quantization/test_fp8.py +import sys + import pytest import torch @@ -145,4 +147,4 @@ if _is_cuda or _is_hip: if __name__ == "__main__": # Run the specific test function directly - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/scripts/check_vram_clear.sh b/scripts/ci/amd/check_vram_clear.sh similarity index 100% rename from scripts/check_vram_clear.sh rename to scripts/ci/amd/check_vram_clear.sh diff --git a/scripts/ensure_vram_clear.sh b/scripts/ci/amd/ensure_vram_clear.sh similarity index 100% rename from scripts/ensure_vram_clear.sh rename to scripts/ci/amd/ensure_vram_clear.sh diff --git a/sgl-kernel/tests/spatial/test_greenctx_stream.py b/sgl-kernel/tests/spatial/test_greenctx_stream.py index c57bc3360..e43dd38f6 100644 --- a/sgl-kernel/tests/spatial/test_greenctx_stream.py +++ b/sgl-kernel/tests/spatial/test_greenctx_stream.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch import torch.nn.functional as F @@ -22,4 +24,4 @@ def test_green_ctx(): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/speculative/test_eagle_utils.py b/sgl-kernel/tests/speculative/test_eagle_utils.py index 503355387..3acc1bb91 100644 --- a/sgl-kernel/tests/speculative/test_eagle_utils.py +++ b/sgl-kernel/tests/speculative/test_eagle_utils.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch import torch.nn.functional as F @@ -84,4 +86,4 @@ def test_verify_tree_greedy(): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/speculative/test_ngram_utils.py b/sgl-kernel/tests/speculative/test_ngram_utils.py index 29bf89f93..0aa7393a8 100644 --- a/sgl-kernel/tests/speculative/test_ngram_utils.py +++ b/sgl-kernel/tests/speculative/test_ngram_utils.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch import torch.nn.functional as F @@ -73,4 +75,4 @@ def test_reconstruct_indices_from_tree_mask(): if __name__ == "__main__": test_reconstruct_indices_from_tree_mask() - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/speculative/test_speculative_sampling.py b/sgl-kernel/tests/speculative/test_speculative_sampling.py index a9b59bb2e..5a95f6e15 100644 --- a/sgl-kernel/tests/speculative/test_speculative_sampling.py +++ b/sgl-kernel/tests/speculative/test_speculative_sampling.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch import torch.nn.functional as F @@ -126,4 +128,4 @@ def test_tree_speculative_sampling_target_only( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_activation.py b/sgl-kernel/tests/test_activation.py index 43593441e..a5428c10a 100644 --- a/sgl-kernel/tests/test_activation.py +++ b/sgl-kernel/tests/test_activation.py @@ -1,5 +1,7 @@ # Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_activation.py +import sys + import pytest import sgl_kernel import torch @@ -36,4 +38,4 @@ def test_fused_gelu_mul(dim, batch_size, seq_len): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_apply_token_bitmask_inplace.py b/sgl-kernel/tests/test_apply_token_bitmask_inplace.py index 480479134..bd70643ec 100644 --- a/sgl-kernel/tests/test_apply_token_bitmask_inplace.py +++ b/sgl-kernel/tests/test_apply_token_bitmask_inplace.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel import apply_token_bitmask_inplace_cuda @@ -20,4 +22,4 @@ def test_apply_token_bitmask_inplace_kernel(): if __name__ == "__main__": test_apply_token_bitmask_inplace_kernel() - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_awq_dequant.py b/sgl-kernel/tests/test_awq_dequant.py index da68e88d0..ce95f5a72 100644 --- a/sgl-kernel/tests/test_awq_dequant.py +++ b/sgl-kernel/tests/test_awq_dequant.py @@ -1,4 +1,5 @@ import itertools +import sys from typing import Optional, Tuple import pytest @@ -112,4 +113,4 @@ def test_awq_dequant_compare_implementations( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_bmm_fp8.py b/sgl-kernel/tests/test_bmm_fp8.py index e0be92896..c6c463d9b 100644 --- a/sgl-kernel/tests/test_bmm_fp8.py +++ b/sgl-kernel/tests/test_bmm_fp8.py @@ -1,5 +1,7 @@ # Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_bmm_fp8.py +import sys + import pytest import torch import torch.nn.functional as F @@ -40,4 +42,4 @@ def test_bmm_fp8(input_dtype, mat2_dtype, res_dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_causal_conv1d.py b/sgl-kernel/tests/test_causal_conv1d.py index a10e1f45e..93731cbf1 100644 --- a/sgl-kernel/tests/test_causal_conv1d.py +++ b/sgl-kernel/tests/test_causal_conv1d.py @@ -1,4 +1,5 @@ # Adapted from https://github.com/vllm-project/vllm/blob/main/tests/kernels/mamba/test_causal_conv1d.py +import sys from typing import Optional import torch @@ -486,4 +487,4 @@ def test_causal_conv1d_varlen( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_copy.py b/sgl-kernel/tests/test_copy.py index 70ed864c1..499834ef4 100644 --- a/sgl-kernel/tests/test_copy.py +++ b/sgl-kernel/tests/test_copy.py @@ -1,3 +1,5 @@ +import sys + import pytest import sgl_kernel import torch @@ -13,4 +15,4 @@ def test_copy_to_gpu_no_ce(size): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_cutlass_mla.py b/sgl-kernel/tests/test_cutlass_mla.py index 71de8327a..6f2ec7b81 100644 --- a/sgl-kernel/tests/test_cutlass_mla.py +++ b/sgl-kernel/tests/test_cutlass_mla.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch import torch.nn.functional as F @@ -101,4 +103,4 @@ def test_cutlass_mla_decode( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py b/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py index 73274165f..5a6b109e7 100644 --- a/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py +++ b/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel import cutlass_w4a8_moe_mm @@ -282,4 +284,4 @@ def ref_grouped_gemm( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_dsv3_fused_a_gemm.py b/sgl-kernel/tests/test_dsv3_fused_a_gemm.py index 914af95e2..1ec683f8a 100644 --- a/sgl-kernel/tests/test_dsv3_fused_a_gemm.py +++ b/sgl-kernel/tests/test_dsv3_fused_a_gemm.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch import torch.nn.functional as F @@ -29,4 +31,4 @@ def test_dsv3_fused_a_gemm(num_tokens): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_dsv3_router_gemm.py b/sgl-kernel/tests/test_dsv3_router_gemm.py index 575769d6d..fa9a830aa 100644 --- a/sgl-kernel/tests/test_dsv3_router_gemm.py +++ b/sgl-kernel/tests/test_dsv3_router_gemm.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch import torch.nn.functional as F @@ -32,4 +34,4 @@ def test_dsv3_router_gemm(num_tokens, num_experts): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_es_fp8_blockwise_moe.py b/sgl-kernel/tests/test_es_fp8_blockwise_moe.py index cd5bd6d67..2aa9b48bd 100644 --- a/sgl-kernel/tests/test_es_fp8_blockwise_moe.py +++ b/sgl-kernel/tests/test_es_fp8_blockwise_moe.py @@ -1,4 +1,5 @@ import random +import sys from typing import Tuple import pytest @@ -202,4 +203,4 @@ def test_fp8_blockwise_scaled_grouped_mm(num_experts, out_dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_es_mxfp8_blockscaled_moe.py b/sgl-kernel/tests/test_es_mxfp8_blockscaled_moe.py index ac7445315..8d857819e 100644 --- a/sgl-kernel/tests/test_es_mxfp8_blockscaled_moe.py +++ b/sgl-kernel/tests/test_es_mxfp8_blockscaled_moe.py @@ -1,4 +1,5 @@ import random +import sys import pytest import torch @@ -152,4 +153,4 @@ def test_es_sm100_mxfp8_blockscaled_grouped_mm(num_experts, out_dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_flash_attention.py b/sgl-kernel/tests/test_flash_attention.py index 159390e54..725244997 100644 --- a/sgl-kernel/tests/test_flash_attention.py +++ b/sgl-kernel/tests/test_flash_attention.py @@ -1,6 +1,7 @@ # Adapted from https://github.com/Dao-AILab/flash-attention/blob/main/hopper/test_flash_attn.py import itertools import math +import sys from typing import Optional import pytest @@ -1365,4 +1366,4 @@ def test_flash_attn_varlen_output( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_flash_attn_sparse.py b/sgl-kernel/tests/test_flash_attn_sparse.py index 28c64cb61..f8d344ef3 100644 --- a/sgl-kernel/tests/test_flash_attn_sparse.py +++ b/sgl-kernel/tests/test_flash_attn_sparse.py @@ -1,4 +1,5 @@ import math +import sys from typing import List, Optional, Tuple import pytest @@ -489,4 +490,4 @@ def test_convert_vertical_slash_indexes_mergehead(causal): # f"{torch.max(torch.abs(lse - ref_lse))}" if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_flashmla.py b/sgl-kernel/tests/test_flashmla.py index 40da3ee49..3afdd7866 100644 --- a/sgl-kernel/tests/test_flashmla.py +++ b/sgl-kernel/tests/test_flashmla.py @@ -1,5 +1,6 @@ import math import random +import sys from typing import Optional, Tuple import pytest @@ -659,4 +660,4 @@ def test_flash_mla_fp8( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_fp8_blockwise_gemm.py b/sgl-kernel/tests/test_fp8_blockwise_gemm.py index 4c1dde336..a4438de4a 100644 --- a/sgl-kernel/tests/test_fp8_blockwise_gemm.py +++ b/sgl-kernel/tests/test_fp8_blockwise_gemm.py @@ -1,5 +1,6 @@ import os import random +import sys from typing import Optional, Type import pytest @@ -90,4 +91,4 @@ def test_accuracy(M, N, K, out_dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_fp8_blockwise_moe.py b/sgl-kernel/tests/test_fp8_blockwise_moe.py index 0488c094b..d1cbbd6ef 100755 --- a/sgl-kernel/tests/test_fp8_blockwise_moe.py +++ b/sgl-kernel/tests/test_fp8_blockwise_moe.py @@ -1,4 +1,5 @@ import random +import sys from typing import Tuple import pytest @@ -218,4 +219,4 @@ def test_fp8_blockwise_scaled_grouped_mm(num_experts, out_dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_fp8_gemm.py b/sgl-kernel/tests/test_fp8_gemm.py index e70e62af2..e809a4d66 100644 --- a/sgl-kernel/tests/test_fp8_gemm.py +++ b/sgl-kernel/tests/test_fp8_gemm.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel import fp8_scaled_mm @@ -46,4 +48,4 @@ def test_accuracy(M, N, K, with_bias, out_dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_gguf.py b/sgl-kernel/tests/test_gguf.py index 3be5e6f33..2f1f0935f 100644 --- a/sgl-kernel/tests/test_gguf.py +++ b/sgl-kernel/tests/test_gguf.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 import random +import sys from pathlib import Path import numpy as np @@ -163,4 +164,4 @@ def test_mmq( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_gptq_kernel.py b/sgl-kernel/tests/test_gptq_kernel.py index 0ef4fad2c..e7596e379 100644 --- a/sgl-kernel/tests/test_gptq_kernel.py +++ b/sgl-kernel/tests/test_gptq_kernel.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel import gptq_gemm @@ -128,4 +130,4 @@ def test_gptq_gemm(M, N, K, bit, group_size, use_shuffle, dtype): if __name__ == "__main__": - pytest.main([__file__, "-v"]) + sys.exit(pytest.main([__file__, "-v"])) diff --git a/sgl-kernel/tests/test_hadamard.py b/sgl-kernel/tests/test_hadamard.py index a70e8f1f1..a0eea45b2 100644 --- a/sgl-kernel/tests/test_hadamard.py +++ b/sgl-kernel/tests/test_hadamard.py @@ -1,4 +1,5 @@ import math +import sys import pytest import torch @@ -82,4 +83,4 @@ def test_fast_hadamard_transform(dim, dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_int8_gemm.py b/sgl-kernel/tests/test_int8_gemm.py index 80f32cd02..b64a27292 100644 --- a/sgl-kernel/tests/test_int8_gemm.py +++ b/sgl-kernel/tests/test_int8_gemm.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel import int8_scaled_mm @@ -45,4 +47,4 @@ def test_accuracy(M, N, K, with_bias, out_dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_kimi_k2_moe_fused_gate.py b/sgl-kernel/tests/test_kimi_k2_moe_fused_gate.py index f96312a19..b70dcd65b 100644 --- a/sgl-kernel/tests/test_kimi_k2_moe_fused_gate.py +++ b/sgl-kernel/tests/test_kimi_k2_moe_fused_gate.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel import kimi_k2_moe_fused_gate @@ -121,4 +123,4 @@ def test_kimi_k2_specific_case(seq_length, num_experts, topk): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_kvcacheio.py b/sgl-kernel/tests/test_kvcacheio.py index 16fc7e826..92af5e986 100644 --- a/sgl-kernel/tests/test_kvcacheio.py +++ b/sgl-kernel/tests/test_kvcacheio.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel.kvcacheio import ( @@ -703,4 +705,4 @@ def test_transfer_kv_page_head( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_merge_state.py b/sgl-kernel/tests/test_merge_state.py index 70b9628d9..3aedb0f94 100644 --- a/sgl-kernel/tests/test_merge_state.py +++ b/sgl-kernel/tests/test_merge_state.py @@ -1,5 +1,6 @@ # Adapted from https://github.com/flashinfer-ai/flashinfer/blob/55576c626421b5ee7e7ebe74afd26465c8ae863f/flashinfer/triton/kernels/cascade.py +import sys from typing import List import pytest @@ -139,4 +140,4 @@ def test_merge_state(seq_len, num_heads, head_dim): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_merge_state_v2.py b/sgl-kernel/tests/test_merge_state_v2.py index 62326f75c..4bbf1704d 100644 --- a/sgl-kernel/tests/test_merge_state_v2.py +++ b/sgl-kernel/tests/test_merge_state_v2.py @@ -1,3 +1,4 @@ +import sys from typing import Optional import pytest @@ -397,4 +398,4 @@ def test_merge_attn_states( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_moe_align.py b/sgl-kernel/tests/test_moe_align.py index 6a27f6e72..ea9861e6d 100644 --- a/sgl-kernel/tests/test_moe_align.py +++ b/sgl-kernel/tests/test_moe_align.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -271,4 +272,4 @@ def test_moe_sum(m: int, topk: int, k: int, dtype: torch.dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_moe_fused_gate.py b/sgl-kernel/tests/test_moe_fused_gate.py index 4d98002d4..9d4c52741 100644 --- a/sgl-kernel/tests/test_moe_fused_gate.py +++ b/sgl-kernel/tests/test_moe_fused_gate.py @@ -1,3 +1,4 @@ +import sys from typing import Optional import pytest @@ -215,4 +216,4 @@ def test_moe_fused_gate_combined( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_moe_topk_sigmoid.py b/sgl-kernel/tests/test_moe_topk_sigmoid.py index 45b8222a9..1f9beb07a 100644 --- a/sgl-kernel/tests/test_moe_topk_sigmoid.py +++ b/sgl-kernel/tests/test_moe_topk_sigmoid.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -180,4 +181,4 @@ def test_topk_sigmoid_renormalize_correction_bias(num_tokens, num_experts, topk) if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_moe_topk_softmax.py b/sgl-kernel/tests/test_moe_topk_softmax.py index d6441a030..1a8bfb93c 100644 --- a/sgl-kernel/tests/test_moe_topk_softmax.py +++ b/sgl-kernel/tests/test_moe_topk_softmax.py @@ -1,4 +1,5 @@ import itertools +import sys import pytest import torch @@ -180,4 +181,4 @@ def test_topk_softmax_renormalize(num_tokens, num_experts, topk): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_norm.py b/sgl-kernel/tests/test_norm.py index ed61663ed..6ec0b1cf8 100644 --- a/sgl-kernel/tests/test_norm.py +++ b/sgl-kernel/tests/test_norm.py @@ -1,5 +1,7 @@ # Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_norm.py +import sys + import pytest import sgl_kernel import torch @@ -139,4 +141,4 @@ def test_gemma_fused_add_rmsnorm(batch_size, hidden_size, dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_per_token_group_quant_8bit.py b/sgl-kernel/tests/test_per_token_group_quant_8bit.py index 1f0190d99..24ebeb877 100644 --- a/sgl-kernel/tests/test_per_token_group_quant_8bit.py +++ b/sgl-kernel/tests/test_per_token_group_quant_8bit.py @@ -1,5 +1,6 @@ import itertools import os +import sys import time from pathlib import Path @@ -182,4 +183,4 @@ def test_per_token_group_quant_with_column_major( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_per_token_quant_fp8.py b/sgl-kernel/tests/test_per_token_quant_fp8.py index 4e1f8a116..e7b18555c 100644 --- a/sgl-kernel/tests/test_per_token_quant_fp8.py +++ b/sgl-kernel/tests/test_per_token_quant_fp8.py @@ -1,4 +1,5 @@ import itertools +import sys from typing import Optional, Tuple import pytest @@ -54,4 +55,4 @@ def test_per_token_quant_compare_implementations( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py b/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py index 9410710d7..bcb9f3a9c 100644 --- a/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py +++ b/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel import qserve_w4a8_per_chn_gemm @@ -115,4 +117,4 @@ def test_accuracy(M, N, K, out_dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py b/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py index fc26a2e60..ee4a2250d 100644 --- a/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py +++ b/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch from sgl_kernel import qserve_w4a8_per_group_gemm @@ -180,4 +182,4 @@ def test_accuracy(M, N, K, group_size, out_dtype): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_sampling.py b/sgl-kernel/tests/test_sampling.py index 51b05c7ef..933e395ee 100644 --- a/sgl-kernel/tests/test_sampling.py +++ b/sgl-kernel/tests/test_sampling.py @@ -1,5 +1,7 @@ # Adapted from https://github.com/flashinfer-ai/flashinfer/blob/93e1a2634e22355b0856246b032b285ad1d1da6b/tests/test_sampling.py +import sys + import flashinfer.sampling import pytest import sgl_kernel @@ -183,4 +185,4 @@ def test_min_p_sampling(batch_size, vocab_size, p): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_topk.py b/sgl-kernel/tests/test_topk.py index dba02321c..742cc4413 100644 --- a/sgl-kernel/tests/test_topk.py +++ b/sgl-kernel/tests/test_topk.py @@ -1,3 +1,4 @@ +import sys from typing import Any, Optional import pytest @@ -249,4 +250,4 @@ def test_topk_transform_ragged_kernel( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/sgl-kernel/tests/test_torch_defaults_reset.py b/sgl-kernel/tests/test_torch_defaults_reset.py index f6fae5d9e..3b7e125c1 100644 --- a/sgl-kernel/tests/test_torch_defaults_reset.py +++ b/sgl-kernel/tests/test_torch_defaults_reset.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch @@ -13,4 +15,4 @@ def test_check_torch_defaults(): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/test/README.md b/test/README.md index 090aaa048..e67025cfe 100644 --- a/test/README.md +++ b/test/README.md @@ -1,40 +1,143 @@ -# Run Unit Tests +# Test and Continuous Integration (CI) System in SGLang -SGLang uses the built-in library [unittest](https://docs.python.org/3/library/unittest.html) as the testing framework. +This page introduces the test system, including the CI pipeline, file organization, and how to add and run tests. -## Test Backend Runtime +## Three Stage CI Pipeline + +The CI pipeline runs in three sequential stages after building the kernel: + +- **Stage A** (pre-flight check, ~3 min): Quick smoke tests on small GPUs and CPU to catch obvious breakages early. +- **Stage B** (basic tests, ~30 min): Core functional tests on both small GPUs (e.g., 5090) and large GPUs (e.g., H100), including 1-GPU and 2-GPU configurations. Kernel tests and multimodal generation tests also run in parallel at this stage. +- **Stage C** (advanced tests, ~30 min): Multi-GPU and specialized hardware tests (H100, H200, B200), plus advanced features such as DeepEP, PD disaggregation, and GB300. + +Here is an illustration +``` + ┌──────────────┐ + │ build kernel │ + └──────┬───────┘ + │ + ├─────────────────────────────────────────────────────┐ + │ │ + ▼ │ + ┌─────────────────────────────────────┐ │ + │ Stage A (~3 min) │ │ + │ pre-flight check │ │ + │ │ │ + │ ┌─────────────────────────────┐ │ │ + │ │ stage-a-test-1-gpu-small │ │ │ + │ │ (small GPUs) │ │ │ + │ └─────────────────────────────┘ │ │ + │ ┌─────────────────────────────┐ │ │ + │ │ stage-a-test-cpu │ │ │ + │ │ (CPU) │ │ │ + │ └─────────────────────────────┘ │ │ + └──────┬──────────────────────────────┘ │ + │ │ + ▼ ▼ + ┌─────────────────────────────────────┐ ┌──────────────────────────┐ + │ Stage B (~30 min) │ │ kernel test │ + │ basic tests │ └──────────────────────────┘ + │ │ ┌──────────────────────────┐ + │ ┌─────────────────────────────┐ │ │ multimodal gen test │ + │ │ stage-b-test-1-gpu-small │ │ └──────────────────────────┘ + │ │ (small GPUs, e.g. 5090) │ │ + │ └─────────────────────────────┘ │ + │ ┌─────────────────────────────┐ │ + │ │ stage-b-test-1-gpu-large │ │ + │ │ (large GPUs, e.g. H100) │ │ + │ └─────────────────────────────┘ │ + │ ┌─────────────────────────────┐ │ + │ │ stage-b-test-2-gpu-large │ │ + │ │ (large GPUs, e.g. H100) │ │ + │ └─────────────────────────────┘ │ + └──────┬──────────────────────────────┘ + │ + ▼ + ┌─────────────────────────────────────┐ + │ Stage C (~30 min) │ + │ advanced tests │ + │ │ + │ ┌─────────────────────────────┐ │ + │ │ stage-c-test-1-gpu-h100 │ │ + │ │ (H100 GPUs) │ │ + │ └─────────────────────────────┘ │ + │ ┌─────────────────────────────┐ │ + │ │ stage-c-test-8-gpu-h200 │ │ + │ │ (8 x H200 GPUs) │ │ + │ └─────────────────────────────┘ │ + │ ┌─────────────────────────────┐ │ + │ │ stage-c-test-4-gpu-b200 │ │ + │ │ (4 x B200 GPUs) │ │ + │ └─────────────────────────────┘ │ + │ ┌─────────────────────────────┐ │ + │ │ Other advanced tests │ │ + │ │ (DeepEP, PD Disagg, GB300) │ │ + │ └─────────────────────────────┘ │ + └─────────────────────────────────────┘ +``` + +- Stage naming convention: `stage-{a,b,c}-test-{gpu_count}-gpu-{hardware}` +- CI runner naming convention: `{gpu_count}-gpu-{hardware}` (e.g., `1-gpu-5090`, `4-gpu-h100`, `8-gpu-h200`) + + +## Folder organization +- `registered`: The registered test files. They are run in CI. Most tests should live in this folder. We use a custom registry system with a file as the basic unit. +- `manual`: Test files that CI does not run; you run them manually. Typically, these are temporary tests, deprecated tests, or tests that are not suitable for CI—such as those that take too long or require special setup. We would still like to keep some files here for anyone who wants to run them locally. +- `run_suite.py`: The launch script to run a test suite. +- Other: utility scripts and metadata folders. The `srt` folder holds our legacy CI setup and should be deprecated as soon as possible. + +Because the system uses a custom registry and the `run_suite.py` launcher, it supports both Python's built-in [unittest](https://docs.python.org/3/library/unittest.html) and the popular [pytest](https://docs.pytest.org/en/stable/) framework. +The basic unit is a file, and you can use either framework in your file. +The launcher runs `python filename.py` to execute tests, so make sure your file includes the following lines. Otherwise, CI will not run it. + +```python +# for unittest +if __name__ == "__main__": + unittest.main() +``` + +```python +# for pytest +if __name__ == "__main__": + import sys + sys.exit(pytest.main([__file__])) +``` + +## Run tests locally + +### Run a single file or a single test ```bash -cd sglang/test/srt - # Run a single file -python3 test_srt_endpoint.py +python3 test/registered/core/test_srt_endpoint.py # Run a single test -python3 test_srt_endpoint.py TestSRTEndpoint.test_simple_decode - -# Run a suite with multiple files -python3 run_suite.py --suite per-commit +python3 test/registered/core/test_srt_endpoint.py TestSRTEndpoint.test_simple_decode ``` -## Test Frontend Language +### Run a suite with multiple files ```bash -cd sglang/test/lang +# Run the CPU-only tests +python3 test/run_suite.py --hw cpu --suite stage-a-test-cpu -# Run a single file -python3 test_choices.py +# Run the small GPU test +python3 test/run_suite.py --hw cuda --suite stage-a-test-1-gpu-small ``` -## Adding or Updating Tests in CI +### More examples +```bash +# Run nightly tests +python test/run_suite.py --hw cuda --suite nightly-1-gpu --nightly + +# With auto-partitioning (for parallel CI jobs) +python test/run_suite.py --hw cuda --suite stage-b-test-1-gpu-small \ + --auto-partition-id 0 --auto-partition-size 4 +``` -- Create new test files under `test/srt` or `test/lang` depending on the type of test. -- For nightly tests, place them in `test/srt/nightly/`. Use the `NightlyBenchmarkRunner` helper class in `nightly_utils.py` for performance benchmarking tests. -- Ensure they are referenced in the respective `run_suite.py` (e.g., `test/srt/run_suite.py`) so they are picked up in CI. For most small test cases, they can be added to the `per-commit-1-gpu` suite. Sort the test cases alphabetically by name. -- Ensure you added `unittest.main()` for unittest and `sys.exit(pytest.main([__file__]))` for pytest in the scripts. The CI run them via `python3 test_file.py`. -- The CI will run some suites such as `per-commit-1-gpu`, `per-commit-2-gpu`, and `nightly-1-gpu` automatically. If you need special setup or custom test groups, you may modify the workflows in [`.github/workflows/`](https://github.com/sgl-project/sglang/tree/main/.github/workflows). ## CI Registry System Tests in `test/registered/` use a registry-based CI system for flexible backend/schedule configuration. +For every test file you add, you need to register it in a suite and provide an estimate execution time in seconds. ### Registration Functions @@ -59,85 +162,90 @@ register_cuda_ci(est_time=200, suite="stage-b-test-2-gpu-large") register_cuda_ci(est_time=200, suite="nightly-1-gpu", nightly=True) # Multi-backend test -register_cuda_ci(est_time=80, suite="stage-b-test-1-gpu-small") +register_cuda_ci(est_time=80, suite="stage-a-test-1-gpu-small") register_amd_ci(est_time=120, suite="stage-a-test-1-gpu-small-amd") +register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True) # Temporarily disabled test register_cuda_ci(est_time=80, suite="stage-b-test-1-gpu-small", disabled="flaky - see #12345") ``` -### Choosing Between 1-GPU Suites (5090 vs H100) +## Available Suites -When adding 1-GPU tests, choose the appropriate suite based on hardware compatibility: +You can find the available suites for each hardware backend at [`test/run_suite.py`](run_suite.py) (`PER_COMMIT_SUITES`, `NIGHTLY_SUITES`). Here we briefly describe some suites. -| Suite | Runner | GPU | When to Use | -|-------|--------|-----|-------------| -| `stage-a-test-1-gpu-small` | `1-gpu-5090` | RTX 5090 (32GB, SM120) | Stage A per-commit smoke on 5090 (CUDA) | -| `stage-a-test-1-gpu-small-amd` | AMD CI runners | ROCm | Stage A per-commit smoke (AMD) | -| `stage-b-test-1-gpu-small` | `1-gpu-5090` | RTX 5090 (32GB, SM120) | 5090-compatible tests (preferred) | -| `stage-b-test-1-gpu-large` | `1-gpu-h100` | H100 (80GB, SM90) | Large models or 5090-incompatible tests | +### Per-commit (CUDA) -**Use `stage-b-test-1-gpu-small` (5090) whenever possible** - this is the preferred suite for most 1-GPU tests. +| Suite | Runner (label) | Description | +| --- | --- | --- | +| `stage-a-test-1-gpu-small` | `1-gpu-5090` | Quick checks on a small NVIDIA GPU before heavier stages | +| `stage-b-test-1-gpu-small` | `1-gpu-5090` | Core engine tests that fit a 5090-class card | +| `stage-b-test-1-gpu-large` | `1-gpu-h100` | Tests that need H100-class memory or kernels (e.g. FA3) | +| `stage-b-test-2-gpu-large` | `2-gpu-h100` | Two-GPU correctness and parallelism (TP/PP-style workloads) on H100 | +| `stage-b-test-4-gpu-b200` | `4-gpu-b200` | Early Blackwell coverage (e.g. SM100+ paths) on four GPUs | +| `stage-c-test-4-gpu-h100` | `4-gpu-h100` | Large 4-GPU H100 integration and scaling tests | +| `stage-c-test-8-gpu-h200` | `8-gpu-h200` | Large 8-GPU H200 runs for big models and parallelism | +| `stage-c-test-8-gpu-h20` | `8-gpu-h20` | Large 8-GPU H20 runs for big models | +| `stage-c-test-deepep-4-gpu-h100` | `4-gpu-h100` | DeepEP expert-parallel and related networking on four H100s. | +| `stage-c-test-deepep-8-gpu-h200` | `8-gpu-h200` | DeepEP at 8-GPU H200 scale. | +| `stage-c-test-4-gpu-b200` | `4-gpu-b200` | 4-GPU B200 suite for large models on blackwell | +| `stage-c-test-4-gpu-gb200` | `4-gpu-gb200`| 4-GPU GB200 suite for large models on grace blackwell | -**Use `stage-b-test-1-gpu-large` (H100) if ANY of these apply:** +Multimodal diffusion uses `python/sglang/multimodal_gen/test/run_suite.py`, not `test/run_suite.py`. -1. **Architecture incompatibility (SM120/Blackwell)**: - - FA3 attention backend (requires SM≤90) - - MLA with FA3 backend - - FP8/MXFP4 quantization (not supported on SM120) - - Certain Triton kernels (shared memory limits) +### Per-commit (CPU) -2. **Memory requirements**: - - Models >30B params or large MoE - - Tests requiring >32GB VRAM +| Suite | Runner (label) | Description | +| --- | --- | --- | +| `stage-a-test-cpu` | `ubuntu-latest` | CPU-only unit tests | -3. **Known 5090 failures**: - - Weight update/sync tests - - Certain spec decoding tests +### Per-commit (AMD) -If a test cannot run on 5090 due to any of the above, use `stage-b-test-1-gpu-large` which runs on H100. +| Suite | Runner (label) | Description | +| --- | --- | --- | +| `stage-a-test-1-gpu-small-amd` | `linux-mi325-1gpu-sglang` | Quick checks on one MI325-class GPU in the AMD CI container. | +| `stage-b-test-2-gpu-large-amd` | `linux-mi325-2gpu-sglang` | 2-GPU ROCm correctness and parallel setups. | +| `stage-b-test-large-8-gpu-35x-disaggregation-amd` | `linux-mi35x-gpu-8.fabric` | Prefill–decode disaggregation and RDMA-oriented tests on an 8×MI35x fabric runner. | +| `stage-c-test-large-8-gpu-amd` | `linux-mi325-8gpu-sglang` | 8-GPU MI325 scaling and integration. | -### Available Suites +### Nightly -**Per-Commit (CUDA)**: -- Stage A: `stage-a-test-1-gpu-small` (5090), `stage-a-test-2`, `stage-a-test-cpu` -- Stage B: `stage-b-test-1-gpu-small` (5090), `stage-b-test-1-gpu-large` (H100), `stage-b-test-2-gpu-large` -- Stage C (4-GPU): `stage-c-test-4-gpu-h100`, `stage-c-test-4-gpu-b200`, `stage-c-test-4-gpu-gb200`, `stage-c-test-deepep-4-gpu-h100` -- Stage C (8-GPU): `stage-c-test-8-gpu-h20`, `stage-c-test-8-gpu-h200`, `stage-c-test-8-gpu-b200`, `stage-c-test-deepep-8-gpu-h200` +Nightly registry suites are listed in `NIGHTLY_SUITES` in [`test/run_suite.py`](run_suite.py). They are not driven by `pr-test.yml` / `pr-test-amd*.yml`; see workflows such as `nightly-test-nvidia.yml` and `nightly-test-amd.yml`. Examples: -**Per-Commit (AMD)**: -- `stage-a-test-1-gpu-small-amd`, `stage-b-test-1-gpu-small-amd`, `stage-b-test-2-gpu-large-amd` +- `nightly-1-gpu` (CUDA) +- `nightly-8-gpu-h200` (CUDA) +- `nightly-eval-vlm-2-gpu` (CUDA) +- `nightly-amd` (AMD) +- `nightly-amd-8-gpu-mi35x` (AMD) -**Nightly**: -- `nightly-1-gpu`, `nightly-2-gpu`, `nightly-4-gpu`, `nightly-8-gpu`, etc. +### Choosing a suite for your test -### Running Tests with run_suite.py +Use the lightest suite that still meets your test's needs. -```bash -# Run per-commit tests -python test/run_suite.py --hw cuda --suite stage-b-test-1-gpu-small +- Prefer the CPU suite (`stage-a-test-cpu`) when no GPU is required. +- For most small GPU workloads that fit a 5090-class card in CI, use `stage-b-test-1-gpu-small`. Most tests should go here. +- If you really need more GPU memory capacity or Hopper-specific features, use `stage-b-test-1-gpu-large`. +- Use multi-GPU suites only when the test actually needs multiple GPUs or other advanced multi-GPU behavior. -# Run nightly tests -python test/run_suite.py --hw cuda --suite nightly-1-gpu --nightly +In rare cases, if you need a new runner or custom setup, you might need to add a new suite. -# With auto-partitioning (for parallel CI jobs) -python test/run_suite.py --hw cuda --suite stage-b-test-1-gpu-small \ - --auto-partition-id 0 --auto-partition-size 4 -``` +## Steps for Adding a Test +Please refer to [.claude/skills/write-sglang-test/SKILL.md](../.claude/skills/write-sglang-test/SKILL.md) -## Writing Elegant Test Cases +## Multi-hardware backends +This README mostly describes the CI pipeline for NVIDIA GPU backends. +Other hardware backends should follow the same practices, use the multi-backend registry system, and build their own pipelines. +A scheduled job summarizes test coverage across all backends; [here is an example run](https://github.com/sgl-project/sglang/actions/runs/23424304300). -- Learn from existing examples in [sglang/test/srt](https://github.com/sgl-project/sglang/tree/main/test/srt). -- Reduce the test time by using smaller models and reusing the server for multiple test cases. Launching a server takes a lot of time. -- Use as few GPUs as possible. Do not run long tests with 8-gpu runners. -- If the test cases take too long, considering adding them to nightly tests instead of per-commit tests. -- Keep each test function focused on a single scenario or piece of functionality. -- Give tests descriptive names reflecting their purpose. -- Use robust assertions (e.g., assert, unittest methods) to validate outcomes. -- Clean up resources to avoid side effects and preserve test independence. -- Reduce the test time by using smaller models and reusing the server for multiple test cases. +## Tips for Writing Elegant Test Cases +- Learn from existing examples in [test/registered](https://github.com/sgl-project/sglang/tree/main/test/registered). +- Reduce the test time by using smaller models and reusing the server for multiple test cases. Launching a server takes a lot of time, so please reuse a single server for many tests instead of launching many servers. +- Use as few GPUs as possible. Use 1-GPU runners whenever possible. Do not run long tests with 8-gpu runners. +- If the test cases take too long, consider adding them to nightly tests instead of per-commit tests. +- Each test file `test_xxx.py` should take less than 500 seconds. If a single file takes longer than that, split it into multiple files. +- Each GitHub Actions job should take less than 30 minutes. If a single job takes longer than that, split it into multiple jobs. - -## Adding New Models to Nightly CI -- **For text models**: extend [global model lists variables](https://github.com/sgl-project/sglang/blob/85c1f7937781199203b38bb46325a2840f353a04/python/sglang/test/test_utils.py#L104) in `test_utils.py`, or add more model lists -- **For vlms**: extend the `MODEL_THRESHOLDS` global dictionary in `test/srt/nightly/test_vlms_mmmu_eval.py` +## Other Notes +### Adding New Models to Nightly CI +- **For text models**: Extend the [global model list variables](https://github.com/sgl-project/sglang/blob/85c1f7937781199203b38bb46325a2840f353a04/python/sglang/test/test_utils.py#L104) in `test_utils.py`, or add more model lists. +- **For VLMs**: Extend the `MODEL_THRESHOLDS` global dictionary in `test/srt/nightly/test_vlms_mmmu_eval.py`. diff --git a/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json b/test/manual/double-sparsity-config-Llama-3.1-8B-Instruct.json similarity index 100% rename from test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json rename to test/manual/double-sparsity-config-Llama-3.1-8B-Instruct.json diff --git a/test/srt/kv_cache_scales_llama3_1_8b.json b/test/manual/quant/kv_cache_scales_llama3_1_8b.json similarity index 100% rename from test/srt/kv_cache_scales_llama3_1_8b.json rename to test/manual/quant/kv_cache_scales_llama3_1_8b.json diff --git a/test/srt/kv_cache_scales_llama3_8b.json b/test/manual/quant/kv_cache_scales_llama3_8b.json similarity index 100% rename from test/srt/kv_cache_scales_llama3_8b.json rename to test/manual/quant/kv_cache_scales_llama3_8b.json diff --git a/test/srt/kv_cache_scales_qwen2_1_5b.json b/test/manual/quant/kv_cache_scales_qwen2_1_5b.json similarity index 100% rename from test/srt/kv_cache_scales_qwen2_1_5b.json rename to test/manual/quant/kv_cache_scales_qwen2_1_5b.json diff --git a/test/manual/test_async_dynamic_batch_tokenizer.py b/test/manual/test_async_dynamic_batch_tokenizer.py index f5d50ab56..bfb9044ec 100644 --- a/test/manual/test_async_dynamic_batch_tokenizer.py +++ b/test/manual/test_async_dynamic_batch_tokenizer.py @@ -7,6 +7,7 @@ including batch efficiency, timeout handling, and error cases. import asyncio import logging +import sys import time from unittest.mock import Mock @@ -292,4 +293,4 @@ class TestAsyncDynamicbatchTokenizer: if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/test/manual/test_async_mm_data_processor.py b/test/manual/test_async_mm_data_processor.py index 65a8dff52..0edc2f5cc 100644 --- a/test/manual/test_async_mm_data_processor.py +++ b/test/manual/test_async_mm_data_processor.py @@ -11,6 +11,7 @@ Covers: import asyncio import logging +import sys import threading import time from unittest.mock import Mock @@ -361,4 +362,4 @@ class TestAsyncMMDataProcessor: if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/test/manual/test_config_integration.py b/test/manual/test_config_integration.py index a44bd979d..924f0eb5e 100644 --- a/test/manual/test_config_integration.py +++ b/test/manual/test_config_integration.py @@ -4,6 +4,7 @@ Test script to verify SGLang config file integration. import argparse import os +import sys import tempfile import pytest @@ -162,4 +163,4 @@ def test_error_handling(): if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py index 917e0caea..dfc6783ae 100644 --- a/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py +++ b/test/registered/debug_utils/test_engine_dumper_comparator_e2e.py @@ -15,6 +15,7 @@ so the YAML only needs ``dumper.dump(...)`` calls. import os import subprocess +import sys import tempfile from pathlib import Path from typing import Optional @@ -341,4 +342,4 @@ def _save_comparator_output(*, stdout: str, stderr: str) -> Path: if __name__ == "__main__": - pytest.main([__file__, "-v"]) + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/test_fused_topk_deepseek.py b/test/registered/kernels/test_fused_topk_deepseek.py index 8c228433d..beed115cc 100644 --- a/test/registered/kernels/test_fused_topk_deepseek.py +++ b/test/registered/kernels/test_fused_topk_deepseek.py @@ -1,3 +1,5 @@ +import sys + import pytest import torch @@ -94,4 +96,4 @@ def test_fused_topk_deepseek(seq_length, params, apply_routed_scaling_factor_on_ if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/test/registered/layers/test_fla_layernorm_guard.py b/test/registered/layers/test_fla_layernorm_guard.py index c6b910a7d..9a99efda6 100644 --- a/test/registered/layers/test_fla_layernorm_guard.py +++ b/test/registered/layers/test_fla_layernorm_guard.py @@ -1,6 +1,7 @@ from __future__ import annotations import socket +import sys from dataclasses import dataclass import pytest @@ -391,4 +392,4 @@ def _layernorm_guard_misc_worker( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/test/registered/lora/test_fused_moe_lora_kernel.py b/test/registered/lora/test_fused_moe_lora_kernel.py index cf3bc1f2c..8c0bde3f3 100644 --- a/test/registered/lora/test_fused_moe_lora_kernel.py +++ b/test/registered/lora/test_fused_moe_lora_kernel.py @@ -1,5 +1,6 @@ # Temporarily adapted from https://github.com/vllm-project/vllm/blob/main/tests/lora/test_fused_moe_lora_kernel.py, will optimize in future refactor import random +import sys import pytest import torch @@ -377,4 +378,4 @@ def test_fused_moe_lora_kernel( if __name__ == "__main__": - pytest.main([__file__]) + sys.exit(pytest.main([__file__])) diff --git a/test/registered/unit/entrypoints/openai/test_serving_embedding.py b/test/registered/unit/entrypoints/openai/test_serving_embedding.py index 0e9dd8c4a..abbec63ec 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_embedding.py +++ b/test/registered/unit/entrypoints/openai/test_serving_embedding.py @@ -2,16 +2,47 @@ Unit tests for the OpenAIServingEmbedding class from serving_embedding.py. """ +import importlib +import importlib.abc +import importlib.machinery import sys +import types import unittest import uuid from unittest.mock import MagicMock, Mock + # Stub out sgl_kernel (and all submodules) before any sglang import so # the test runs on CPU-only runners without the real CUDA library. -for _mod in ("sgl_kernel", "sgl_kernel.kvcacheio"): - if _mod not in sys.modules: - sys.modules[_mod] = MagicMock() +class _SglKernelMockLoader(importlib.abc.Loader): + def create_module(self, spec): + mod = types.ModuleType(spec.name) + mod.__path__ = [] + mod.__package__ = spec.name + mod.__loader__ = self + mod.__getattr__ = lambda name: MagicMock() + return mod + + def exec_module(self, module): + pass + + +class _SglKernelMockFinder(importlib.abc.MetaPathFinder): + """Import hook that intercepts all sgl_kernel.* imports and returns mocks.""" + + _PREFIX = "sgl_kernel" + _loader = _SglKernelMockLoader() + + def find_spec(self, fullname, path, target=None): + if fullname == self._PREFIX or fullname.startswith(self._PREFIX + "."): + return importlib.machinery.ModuleSpec( + fullname, self._loader, is_package=True + ) + return None + + +if "sgl_kernel" not in sys.modules: + sys.meta_path.insert(0, _SglKernelMockFinder()) from fastapi import Request diff --git a/test/run_suite.py b/test/run_suite.py index 876fe9021..93fcaaede 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -1,5 +1,6 @@ import argparse import glob +import os import sys from typing import List @@ -39,13 +40,14 @@ PER_COMMIT_SUITES = { "stage-b-test-1-gpu-small", "stage-b-test-1-gpu-large", "stage-b-test-2-gpu-large", + "stage-b-test-4-gpu-b200", "stage-c-test-4-gpu-h100", "stage-c-test-4-gpu-b200", "stage-c-test-4-gpu-gb200", - "stage-c-test-deepep-4-gpu-h100", "stage-c-test-8-gpu-h20", "stage-c-test-8-gpu-h200", "stage-c-test-8-gpu-b200", + "stage-c-test-deepep-4-gpu-h100", "stage-c-test-deepep-8-gpu-h200", ], HWBackend.NPU: [ @@ -169,9 +171,13 @@ def run_a_suite(args): auto_partition_size = args.auto_partition_size # All tests (per-commit and nightly) are now in registered/ + # Use absolute paths so the script works from any working directory + script_dir = os.path.dirname(os.path.abspath(__file__)) files = [ f - for f in glob.glob("registered/**/*.py", recursive=True) + for f in glob.glob( + os.path.join(script_dir, "registered", "**", "*.py"), recursive=True + ) if not f.endswith("/conftest.py") and not f.endswith("/__init__.py") ] # Strict: all registered files must have proper registration diff --git a/test/run_suite_nightly.py b/test/run_suite_nightly.py deleted file mode 100644 index 6e6c701b0..000000000 --- a/test/run_suite_nightly.py +++ /dev/null @@ -1,97 +0,0 @@ -import argparse -import os -import sys -from pathlib import Path - -from sglang.test.ci.ci_utils import TestFile, run_unittest_files - -# Nightly test suites -suites = { - "nightly-1-gpu": [ - TestFile("test_nsa_indexer.py", 2), - TestFile("test_lora_qwen3.py", 97), - TestFile("test_lora_radix_cache.py", 200), - TestFile("test_lora_eviction_policy.py", 200), - TestFile("test_lora_openai_api.py", 30), - TestFile("test_lora_openai_compatible.py", 150), - TestFile("test_lora_hf_sgl_logprob_diff.py", 300), - TestFile("test_batch_invariant_ops.py", 10), - TestFile("test_cpp_radix_cache.py", 60), - TestFile("test_deepseek_v3_deterministic.py", 240), - ], - "nightly-4-gpu-b200": [ - TestFile("test_flashinfer_trtllm_gen_moe_backend.py", 300), - TestFile("test_gpt_oss_4gpu_perf.py", 600), - TestFile("test_flashinfer_trtllm_gen_attn_backend.py", 300), - TestFile("test_fp4_moe.py", 300), - TestFile("test_qwen3_fp4_trtllm_gen_moe.py", 300), - TestFile("test_eagle_infer_beta_dp_attention_large.py", 600), - ], - "nightly-8-gpu-b200": [ - TestFile("test_deepseek_r1_fp8_trtllm_backend.py", 3600), - TestFile("test_deepseek_v32_gpqa.py", 3600), - TestFile("test_mistral_large3_basic.py", 600), - ], - "nightly-4-gpu": [ - TestFile("test_encoder_dp.py", 500), - TestFile("test_qwen3_next_deterministic.py", 200), - ], - "nightly-8-gpu": [], - "nightly-8-gpu-h200": [ - TestFile("test_deepseek_v32_nsabackend.py", 600), - ], - "nightly-8-gpu-h20": [], -} - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--suite", - type=str, - required=True, - help="Test suite to run (e.g., nightly-1-gpu, nightly-4-gpu, etc.).", - ) - parser.add_argument( - "--timeout-per-file", - type=int, - default=1200, - help="The time limit for running one file in seconds (default: 1200).", - ) - parser.add_argument( - "--continue-on-error", - action="store_true", - default=False, - help="Continue running remaining tests even if one fails (default: False, useful for nightly tests).", - ) - args = parser.parse_args() - - if args.suite not in suites: - print(f"Error: Suite '{args.suite}' not found in available suites") - print(f"Available suites: {list(suites.keys())}") - exit(1) - - files = suites[args.suite] - - # Change directory to test/nightly where the test files are located - nightly_dir = Path(__file__).parent / "nightly" - os.chdir(nightly_dir) - - # Add test/ to PYTHONPATH so tests can import shared utils - test_dir = str(Path(__file__).parent) - pythonpath = os.environ.get("PYTHONPATH", "") - os.environ["PYTHONPATH"] = f"{test_dir}:{pythonpath}" if pythonpath else test_dir - - print(f"Running {len(files)} tests from suite: {args.suite}") - print(f"Test files: {[f.name for f in files]}") - - exit_code = run_unittest_files( - files, - timeout_per_file=args.timeout_per_file, - continue_on_error=args.continue_on_error, - ) - sys.exit(exit_code) - - -if __name__ == "__main__": - main() diff --git a/test/srt/test_embed_interpolate_unittest.py b/test/srt/ascend/test_embed_interpolate_unittest.py similarity index 100% rename from test/srt/test_embed_interpolate_unittest.py rename to test/srt/ascend/test_embed_interpolate_unittest.py diff --git a/test/srt/experiment_runner.py b/test/srt/experiment_runner.py deleted file mode 100644 index 04368e252..000000000 --- a/test/srt/experiment_runner.py +++ /dev/null @@ -1,369 +0,0 @@ -import argparse -import logging -import os -import queue -import re -import subprocess -import threading -import time -from dataclasses import dataclass -from datetime import datetime -from typing import List, Optional, Tuple - -import psutil -import yaml - -from sglang.utils import wait_for_http_ready - - -@dataclass -class ServerConfig: - command: str - process_names: List[str] - default_port: int - - -@dataclass -class TaskConfig: - server_cmd: str - client_cmd: str - name: Optional[str] = None - server_type: Optional[str] = None - - -@dataclass -class TaskResult: - name: str - success: bool - output: str - runtime: float - timestamp: str - - -SERVER_DEFAULTS = { - "sglang": ServerConfig( - command="sglang.launch_server", - process_names=["sglang.launch_server"], - default_port=30000, - ), - "vllm": ServerConfig( - command="vllm.entrypoints.openai.api_server", - process_names=["vllm.entrypoints.openai.api_server"], - default_port=8000, - ), -} - - -def parse_key_info(output: str) -> str: - """Extract and format key information from the output""" - key_info = [] - - # Extract Args namespace - args_match = re.search(r"Namespace\(.*?\)", output, re.DOTALL) - if args_match: - key_info.append(args_match.group(0)) - - # Extract input/output token counts - token_matches = re.findall(r"#(Input|Output) tokens: \d+", output) - key_info.extend(token_matches) - - # Extract benchmark result section - result_match = re.search( - r"============ Serving Benchmark Result ============.*?={50,}", - output, - re.DOTALL, - ) - if result_match: - key_info.append(result_match.group(0)) - - return "\n\n".join(key_info) - - -def extract_port_from_command(cmd: str, server_type: str) -> int: - port_match = re.search(r"--port[= ](\d+)", cmd) - if port_match: - return int(port_match.group(1)) - return SERVER_DEFAULTS.get(server_type, ServerConfig("", [], 8000)).default_port - - -def detect_server_type(cmd: str) -> str: - for server_type, config in SERVER_DEFAULTS.items(): - if config.command in cmd: - return server_type - return "unknown" - - -def stream_output( - process: subprocess.Popen, prefix: str, logger: logging.Logger -) -> queue.Queue: - output_queue = queue.Queue() - - def stream_pipe(pipe, prefix): - for line in iter(pipe.readline, ""): - if prefix == "CLIENT": - output_queue.put(line.rstrip()) - logger.debug(f"{prefix} | {line.rstrip()}") - - stdout_thread = threading.Thread( - target=stream_pipe, args=(process.stdout, prefix), daemon=True - ) - stderr_thread = threading.Thread( - target=stream_pipe, args=(process.stderr, prefix), daemon=True - ) - - stdout_thread.start() - stderr_thread.start() - return output_queue, (stdout_thread, stderr_thread) - - -class ProcessManager: - def __init__(self): - self.server_process: Optional[subprocess.Popen] = None - self.client_process: Optional[subprocess.Popen] = None - self.logger = logging.getLogger(__name__) - - def start_process( - self, command: str, prefix: str - ) -> Tuple[subprocess.Popen, queue.Queue]: - process = subprocess.Popen( - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - bufsize=1, - ) - - output_queue, threads = stream_output(process, prefix, self.logger) - return process, output_queue, threads - - def kill_process_tree(self, process: subprocess.Popen): - try: - parent = psutil.Process(process.pid) - children = parent.children(recursive=True) - - for child in children: - try: - child.kill() - except psutil.NoSuchProcess: - pass - - parent.kill() - gone, alive = psutil.wait_procs(children + [parent], timeout=3) - - for p in alive: - try: - p.kill() - except psutil.NoSuchProcess: - pass - - except psutil.NoSuchProcess: - pass - - def cleanup(self, process_names: List[str]): - if self.client_process: - self.kill_process_tree(self.client_process) - self.client_process = None - - if self.server_process: - self.kill_process_tree(self.server_process) - self.server_process = None - - for proc in psutil.process_iter(["pid", "name", "cmdline"]): - try: - cmdline = " ".join(proc.cmdline()) - if any(name in cmdline for name in process_names): - proc.kill() - except (psutil.NoSuchProcess, psutil.AccessDenied): - continue - - -class ExperimentRunner: - def __init__(self): - self.process_manager = ProcessManager() - self.logger = logging.getLogger(__name__) - - def wait_for_server( - self, port: int, timeout: int = 300, process: Optional[subprocess.Popen] = None - ) -> bool: - try: - wait_for_http_ready( - url=f"http://localhost:{port}/health", - timeout=timeout, - process=process, - ) - self.logger.debug(f"Server ready on port {port}") - return True - except (RuntimeError, TimeoutError) as e: - self.logger.error("Server failed to become ready: %s", e) - return False - - def run_task(self, config: TaskConfig) -> TaskResult: - start_time = time.perf_counter() - client_output = [] - - try: - if not config.server_type: - config.server_type = detect_server_type(config.server_cmd) - - server_config = SERVER_DEFAULTS.get(config.server_type) - if not server_config: - raise ValueError(f"Unknown server type: {config.server_type}") - - port = extract_port_from_command(config.server_cmd, config.server_type) - - self.process_manager.cleanup(server_config.process_names) - - self.logger.debug(f"Starting server: {config.name}") - self.process_manager.server_process, _, server_threads = ( - self.process_manager.start_process(config.server_cmd, "SERVER") - ) - - if not self.wait_for_server( - port, process=self.process_manager.server_process - ): - raise TimeoutError("Server startup timeout") - - time.sleep(10) - - self.logger.debug("Starting client") - self.process_manager.client_process, output_queue, client_threads = ( - self.process_manager.start_process(config.client_cmd, "CLIENT") - ) - - returncode = self.process_manager.client_process.wait() - - while True: - try: - line = output_queue.get_nowait() - client_output.append(line) - except queue.Empty: - break - - if returncode != 0: - raise RuntimeError(f"Client failed with code {returncode}") - - # Parse and format the output - full_output = "\n".join(client_output) - formatted_output = parse_key_info(full_output) - - return TaskResult( - name=config.name, - success=True, - output=formatted_output, - runtime=time.perf_counter() - start_time, - timestamp=datetime.now().isoformat(), - ) - - except Exception as e: - return TaskResult( - name=config.name, - success=False, - output=str(e), - runtime=time.perf_counter() - start_time, - timestamp=datetime.now().isoformat(), - ) - - finally: - if config.server_type in SERVER_DEFAULTS: - self.process_manager.cleanup( - SERVER_DEFAULTS[config.server_type].process_names - ) - time.sleep(10) - - -def load_config(config_path: str) -> List[TaskConfig]: - with open(config_path, "r") as f: - config_data = yaml.safe_load(f) - - configs = [] - for idx, entry in enumerate(config_data.get("tasks", [])): - if not isinstance(entry, dict): - raise ValueError(f"Invalid entry at index {idx}") - - config = TaskConfig( - server_cmd=entry.get("server_cmd"), - client_cmd=entry.get("client_cmd"), - name=entry.get("name", f"task-{idx+1}"), - server_type=entry.get("server_type"), - ) - - if not config.server_cmd or not config.client_cmd: - raise ValueError(f"Missing commands in {config.name}") - - configs.append(config) - - return configs - - -def setup_logging(debug: bool = False): - level = logging.DEBUG if debug else logging.INFO - logging.basicConfig( - level=level, - format="%(asctime)s - %(levelname)s - %(message)s", - handlers=[logging.StreamHandler(), logging.FileHandler("experiment.log")], - ) - - -def format_results(results: List[TaskResult]) -> str: - """Format experiment results in Markdown for GitHub step summary.""" - output = ["# Experiment Results\n"] - - for result in results: - output.append(f"## {result.name}") - output.append(f"**Status**: {'✅ Success' if result.success else '❌ Failed'}") - output.append(f"**Runtime**: {result.runtime:.2f} seconds") - output.append(f"**Timestamp**: {result.timestamp}") - output.append("\n**Output**:\n```") - output.append(result.output) - output.append("```\n") - - return "\n".join(output) - - -def get_bool_env_var(name: str, default: str = "false") -> bool: - value = os.getenv(name, default) - return value.lower() in ("true", "1") - - -def write_in_github_step_summary(results: List[TaskResult]): - """Write formatted results to GitHub step summary.""" - if not os.environ.get("GITHUB_STEP_SUMMARY"): - logging.warning("GITHUB_STEP_SUMMARY environment variable not set") - return - - formatted_content = format_results(results) - with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: - f.write(formatted_content) - - -def main(): - parser = argparse.ArgumentParser(description="Experiment Runner") - parser.add_argument( - "--config", type=str, required=True, help="Path to YAML config file" - ) - parser.add_argument("--debug", action="store_true", help="Enable debug output") - args = parser.parse_args() - - setup_logging(args.debug) - logger = logging.getLogger(__name__) - results = [] - - try: - configs = load_config(args.config) - runner = ExperimentRunner() - - for config in configs: - logger.info(f"Running {config.name}") - result = runner.run_task(config) - results.append(result) - - if get_bool_env_var("SGLANG_IS_IN_CI"): - write_in_github_step_summary(results) - except Exception as e: - logger.error(f"Error: {e}") - raise - - -if __name__ == "__main__": - main() diff --git a/test/srt/models/compare.py b/test/srt/models/compare.py deleted file mode 100644 index 2fe35357c..000000000 --- a/test/srt/models/compare.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -used for debug using tensor comparison -dump {name: tensor} into "log_hf.jsonl" and "log_srt.jsonl" -use the same name for two tensors that supposed to be close -recommend name like: "layer 2 after mlp" -""" - -import json -import sys - -import torch - -if len(sys.argv) > 1: - assert sys.argv[1] == "base" - hf_log = "base_log_hf.jsonl" - srt_log = "base_log_srt.jsonl" -else: - hf_log = "log_hf.jsonl" - srt_log = "log_srt.jsonl" - - -def load_data(filepath): - tensors = {} - with open(filepath, "r") as f: - lines = f.readlines() - for line in lines: - data = json.loads(line) - for k, v in data.items(): - tensors[k] = torch.tensor(v) - return tensors - - -hf_tensors = load_data(hf_log) -srt_tensors = load_data(srt_log) - - -def get_diff(t1, t2): - t1 = t1.reshape(t2.shape) - max_diff = torch.max(abs(t1.reshape(t2.shape) - t2)) - l2_dis = torch.dist(t1, t2, p=2) - return l2_dis, max_diff - - -for k, _ in srt_tensors.items(): - l2_dis, max_diff = get_diff(hf_tensors[k], srt_tensors[k]) - print(f"{k} {l2_dis=} {max_diff=}") - if k == "layer 1 attn": - print(hf_tensors[k]) - print(srt_tensors[k]) - if k == "layer 0 prefill k": - print(srt_tensors[k].shape) - print(hf_tensors[k].shape) diff --git a/test/srt/parse_results.py b/test/srt/parse_results.py deleted file mode 100644 index f552739f5..000000000 --- a/test/srt/parse_results.py +++ /dev/null @@ -1,57 +0,0 @@ -import argparse -import json -import os - -import pandas as pd -from tabulate import tabulate - -# Parse command-line arguments -parser = argparse.ArgumentParser(description="Parse JSONL benchmark and summarize.") -parser.add_argument("input_file", type=str, help="Path to input JSONL file") -parser.add_argument( - "--md", - action="store_true", - help="If set, print the summary table in Markdown format (GitHub style)", -) -args = parser.parse_args() - -input_file = args.input_file -base_name = os.path.splitext(os.path.basename(input_file))[0] -output_file = f"{base_name}_summary.csv" - -fields = [ - "max_concurrency", - "input_throughput", - "output_throughput", - "mean_ttft_ms", - "median_ttft_ms", - "p99_ttft_ms", - "mean_tpot_ms", - "median_tpot_ms", - "p99_tpot_ms", -] - -# Read JSONL and parse -results = [] -with open(input_file, "r") as f: - for line in f: - data = json.loads(line) - row = {field: data.get(field, None) for field in fields} - max_conc = data.get("max_concurrency") - out_tp = data.get("output_throughput") - row["per_user_throughput"] = out_tp / max_conc if max_conc else None - results.append(row) - -# Convert to DataFrame -df = pd.DataFrame(results) - -# Save to CSV -df.to_csv(output_file, index=False) -print(f"\nSaved summary to: {output_file}\n") - -if args.md: - # Print Markdown table - print(tabulate(df, headers="keys", tablefmt="github", floatfmt=".3f")) -else: - # Print ASCII table - print(tabulate(df, headers="keys", tablefmt="grid", floatfmt=".3f")) diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 5ed743aa0..a7d5b6744 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -96,7 +96,7 @@ suite_ascend = { TestFile("ascend/test_ascend_tp1_bf16.py", 400), TestFile("ascend/test_ascend_compile_graph_tp1_bf16.py", 400), TestFile("ascend/test_ascend_w8a8_quantization.py", 400), - TestFile("test_embed_interpolate_unittest.py", 400), + TestFile("ascend/test_embed_interpolate_unittest.py", 400), ], "per-commit-2-npu-a2": [ TestFile("ascend/test_ascend_graph_tp2_bf16.py", 400), @@ -326,4 +326,9 @@ def main(): if __name__ == "__main__": + print( + "DEPRECATION NOTICE: The folder `test/srt` should be deprecated as soon as possible. " + "Migrate tests to the new CI registry system described in `test/README.md`.", + flush=True, + ) main()