minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[MUSA][23/N] CI: Fix torchada preflight lock cleanup and add LLM server smoke test (#27242)

Browse Source 
Co-authored-by: xinyue.fan <xinyue.fan@mthreads.com>
This commit is contained in:
Stella-17
2026-06-08 08:45:16 -07:00
committed by GitHub
co-authored by xinyue.fan
parent eb646c7b78
commit 12de907bc2
7 changed files with 225 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-musa.yml
+53 -4
View File
@@ -14,6 +14,7 @@ on:
- 'all'
- 'nightly-test-musa-general-kernel'
- 'nightly-test-musa-general-multimodal-layer'
- 'nightly-test-llm-server-1-gpu-musa'
- 'nightly-test-multimodal-server-1-gpu-musa'
- 'nightly-test-multimodal-server-2-gpu-musa'
workflow_call:
@@ -65,7 +66,7 @@ jobs:
- name: Install dependencies
if: steps.gate.outputs.run_job == 'true'
timeout-minutes: 10
timeout-minutes: 15
run: |
bash scripts/ci/musa/musa_install_dependency.sh
@@ -111,7 +112,7 @@ jobs:
ref: ${{ inputs.ref || github.ref }}
- name: Install dependencies
timeout-minutes: 10
timeout-minutes: 15
if: steps.gate.outputs.run_job == 'true'
run: |
bash scripts/ci/musa/musa_install_dependency.sh
@@ -123,6 +124,53 @@ jobs:
pytest python/sglang/multimodal_gen/test/layers/test_musa_rmsnorm.py
pytest python/sglang/multimodal_gen/test/layers/test_musa_silu_and_mul.py
# ==================== LLM server: 1-GPU ====================
nightly-test-llm-server-1-gpu-musa:
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
runs-on: s5000-1-gpu-runner
timeout-minutes: 240
env:
SGLANG_USE_MODELSCOPE: false
SGLANG_IS_IN_CI: true
TORCHADA_ENABLE_CPP_OPS: 1
HF_HOME: /hf-cache
HF_HUB_CACHE: /hf-cache/hub
HF_HUB_OFFLINE: 1
steps:
- name: Gate by job_filter
id: gate
run: |
filter="${{ inputs.job_filter || 'all' }}"
if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-llm-server-1-gpu-musa" ]]; then
echo "run_job=true" >> "$GITHUB_OUTPUT"
else
echo "run_job=false" >> "$GITHUB_OUTPUT"
fi
- name: Checkout code
timeout-minutes: 10
if: steps.gate.outputs.run_job == 'true'
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Install dependencies
timeout-minutes: 15
if: steps.gate.outputs.run_job == 'true'
run: |
bash scripts/ci/musa/musa_install_dependency.sh
- name: Run DeepSeek-V2-Lite-Chat LLM server smoke test
if: steps.gate.outputs.run_job == 'true'
timeout-minutes: 120
run: |
cd test
python3 run_suite.py \
--hw musa \
--suite nightly-musa-1-gpu \
--nightly \
--timeout-per-file 7200
# ==================== Multimodal: 1-GPU (split) ====================
nightly-test-multimodal-server-1-gpu-musa:
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
@@ -159,7 +207,7 @@ jobs:
ref: ${{ inputs.ref || github.ref }}
- name: Install dependencies
timeout-minutes: 10
timeout-minutes: 15
if: steps.gate.outputs.run_job == 'true'
run: |
bash scripts/ci/musa/musa_install_dependency.sh
@@ -208,7 +256,7 @@ jobs:
ref: ${{ inputs.ref || github.ref }}
- name: Install dependencies
timeout-minutes: 10
timeout-minutes: 15
if: steps.gate.outputs.run_job == 'true'
run: |
bash scripts/ci/musa/musa_install_dependency.sh
@@ -230,6 +278,7 @@ jobs:
[
nightly-test-musa-general-kernel,
nightly-test-musa-general-multimodal-layer,
nightly-test-llm-server-1-gpu-musa,
nightly-test-multimodal-server-1-gpu-musa,
nightly-test-multimodal-server-2-gpu-musa,
]
.github/workflows/pr-test-musa.yml
+4 -4
View File
@@ -117,7 +117,7 @@ jobs:
ref: ${{ inputs.ref || github.ref }}
- name: Install dependencies
timeout-minutes: 10
timeout-minutes: 15
run: |
bash scripts/ci/musa/musa_install_dependency.sh
@@ -152,7 +152,7 @@ jobs:
ref: ${{ inputs.ref || github.ref }}
- name: Install dependencies
timeout-minutes: 10
timeout-minutes: 15
run: |
bash scripts/ci/musa/musa_install_dependency.sh
@@ -183,7 +183,7 @@ jobs:
uses: actions/checkout@v4
- name: Install dependencies
timeout-minutes: 10
timeout-minutes: 15
run: |
bash scripts/ci/musa/musa_install_dependency.sh
@@ -210,7 +210,7 @@ jobs:
ref: ${{ inputs.ref || github.ref }}
- name: Install dependencies
timeout-minutes: 10
timeout-minutes: 15
run: |
bash scripts/ci/musa/musa_install_dependency.sh
.github/workflows/release-whl-kernel.yml
+12
View File
@@ -437,6 +437,18 @@ jobs:
with:
submodules: "recursive"
- name: Clean stale torchada extension locks
run: |
if ps -eo comm=,args= | grep -E '(^|[[:space:]])(mcc|ninja)([[:space:]]|$)|torchada_cpp_ops' | grep -v grep; then
echo "::error::Active torchada extension build detected; refusing to remove lock files"
exit 1
fi
find "${HOME}/.cache/torch_extensions" \
-path '*/torchada_cpp_ops/lock' \
-type f \
-print \
-delete 2>/dev/null || true
- name: Build wheels
run: |
cd sgl-kernel
python/sglang/test/ci/ci_register.py
+15
View File
@@ -12,6 +12,7 @@ __all__ = [
"register_cpu_ci",
"register_cuda_ci",
"register_amd_ci",
"register_musa_ci",
"register_npu_ci",
"register_xpu_ci",
"register_musa_ci",
@@ -95,6 +96,19 @@ def register_amd_ci(
return None
def register_musa_ci(
est_time: float,
suite: Optional[str] = None,
nightly: bool = False,
disabled: Optional[str] = None,
*,
stage: Optional[str] = None,
runner_config: Optional[str] = None,
):
"""Marker for MUSA CI registration (parsed via AST; runtime no-op)."""
return None
def register_npu_ci(
est_time: float,
suite: Optional[str] = None,
@@ -138,6 +152,7 @@ REGISTER_MAPPING = {
"register_cpu_ci": HWBackend.CPU,
"register_cuda_ci": HWBackend.CUDA,
"register_amd_ci": HWBackend.AMD,
"register_musa_ci": HWBackend.MUSA,
"register_npu_ci": HWBackend.NPU,
"register_xpu_ci": HWBackend.XPU,
"register_musa_ci": HWBackend.MUSA,
scripts/ci/musa/musa_install_dependency.sh
+19
View File
@@ -24,6 +24,25 @@ done
PIP_INSTALL="python3 -m pip install --no-cache-dir"
${PIP_INSTALL} --upgrade pip setuptools torchada --user
echo "Checking stale torchada extension locks..."
active_torchada_builds="$(
pgrep -af '(^|[[:space:]/])(mcc|ninja)([[:space:]]|$)|torchada_cpp_ops' 2>/dev/null \
| awk -v self="$$" '$1 != self'
)" || true
if [ -n "$active_torchada_builds" ]; then
echo "$active_torchada_builds"
echo "::error::Active torchada extension build detected; refusing to remove lock files"
exit 1
fi
torch_extensions_dir="${HOME}/.cache/torch_extensions"
if [ -d "$torch_extensions_dir" ]; then
find "$torch_extensions_dir" \
-path '*/torchada_cpp_ops/lock' \
-type f \
-print \
-delete
fi
WHL_DIR="/sglang-checkout/whl"
if [ -d "$WHL_DIR" ] && compgen -G "${WHL_DIR}"/*.whl > /dev/null; then
echo "Uninstall old packages based on wheel METADATA..."
test/registered/musa/test_llm_server_smoke_musa.py
+111
View File
@@ -0,0 +1,111 @@
import os
import unittest
import requests
import torch
from sglang.test.ci.ci_register import register_musa_ci
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
register_musa_ci(est_time=1200, suite="nightly-musa-1-gpu", nightly=True)
_REQUEST_TIMEOUT = 60
@unittest.skipIf(
not (hasattr(torch, "musa") and torch.musa.is_available()),
"MUSA device not available",
)
class TestMusaDeepSeekV2LiteChatServerSmoke(DefaultServerBase):
"""MUSA LLM server smoke test: launch, health check, and non-empty generation."""
model = os.getenv("SGLANG_MUSA_LLM_MODEL", "deepseek-ai/DeepSeek-V2-Lite-Chat")
served_model_name = "deepseek-v2-lite-chat"
other_args = [
"--trust-remote-code",
"--served-model-name",
served_model_name,
"--attention-backend",
"fa3",
"--cuda-graph-max-bs",
"32",
"--tp-size",
"1",
"--chunked-prefill-size",
"-1",
"--disable-piecewise-cuda-graph",
"--context-length",
"4096",
"--max-total-tokens",
"8192",
"--max-running-requests",
"4",
]
def test_health(self):
resp = requests.get(self.base_url + "/health", timeout=10)
self.assertEqual(resp.status_code, 200, resp.text)
def test_health_generate(self):
resp = requests.get(
self.base_url + "/health_generate", timeout=_REQUEST_TIMEOUT
)
self.assertEqual(resp.status_code, 200, resp.text)
def test_send_receive_chat_message_contains_beijing(self):
resp = requests.post(
self.base_url + "/v1/chat/completions",
json={
"model": self.served_model_name,
"messages": [
{
"role": "user",
"content": (
"What is the capital of China? Answer in one word."
),
},
],
"temperature": 0.0,
"max_tokens": 16,
},
timeout=_REQUEST_TIMEOUT,
)
self.assertEqual(resp.status_code, 200, resp.text)
body = resp.json()
self.assertIn("choices", body)
self.assertGreater(len(body["choices"]), 0)
content = body["choices"][0]["message"]["content"]
print(
f"[MUSA Chat Completion] prompt='What is the capital of China? Answer in one word.' response={content!r}",
flush=True,
)
self.assertIsInstance(content, str)
self.assertGreater(len(content.strip()), 0)
self.assertIn("Beijing", content.lower())
def test_generate(self):
resp = requests.post(
self.base_url + "/generate",
json={
"text": "The capital of France is",
"sampling_params": {
"temperature": 0.0,
"max_new_tokens": 16,
},
"stream": False,
},
timeout=_REQUEST_TIMEOUT,
)
self.assertEqual(resp.status_code, 200, resp.text)
body = resp.json()
if isinstance(body, list):
self.assertGreater(len(body), 0)
body = body[0]
self.assertIn("text", body)
self.assertIsInstance(body["text"], str)
self.assertGreater(len(body["text"].strip()), 0)
if __name__ == "__main__":
unittest.main()
test/run_suite.py
+11 -1
View File
@@ -19,6 +19,7 @@ HW_MAPPING = {
"cpu": HWBackend.CPU,
"cuda": HWBackend.CUDA,
"amd": HWBackend.AMD,
"musa": HWBackend.MUSA,
"npu": HWBackend.NPU,
"xpu": HWBackend.XPU,
}
@@ -42,6 +43,7 @@ PER_COMMIT_SUITES = {
"stage-c-test-large-8-gpu-amd",
"stage-c-test-large-8-gpu-amd-mi35x",
],
HWBackend.MUSA: [],
HWBackend.CUDA: [
"base-a-test-1-gpu-small",
"base-b-test-1-gpu-small",
@@ -122,6 +124,9 @@ NIGHTLY_SUITES = {
# MI35x 8-GPU suite (different model configs)
"nightly-amd-8-gpu-mi35x",
],
HWBackend.MUSA: [
"nightly-musa-1-gpu",
],
HWBackend.CPU: [],
HWBackend.NPU: [
"nightly-1-npu-a3",
@@ -150,7 +155,12 @@ OTHER_SUITES = {
}
_SUITE_CHECKED_BACKENDS = {HWBackend.CUDA, HWBackend.CPU, HWBackend.XPU}
_SUITE_CHECKED_BACKENDS = {
HWBackend.CUDA,
HWBackend.CPU,
HWBackend.MUSA,
HWBackend.XPU,
}
def _valid_suites_by_backend() -> dict: