[MUSA][23/N] CI: Fix torchada preflight lock cleanup and add LLM server smoke test (#27242)

Co-authored-by: xinyue.fan <xinyue.fan@mthreads.com>

.github/workflows/nightly-test-musa.yml

@@ -14,6 +14,7 @@ on:
           - 'all'
           - 'nightly-test-musa-general-kernel'
           - 'nightly-test-musa-general-multimodal-layer'
+          - 'nightly-test-llm-server-1-gpu-musa'
           - 'nightly-test-multimodal-server-1-gpu-musa'
           - 'nightly-test-multimodal-server-2-gpu-musa'
   workflow_call:
@@ -65,7 +66,7 @@ jobs:
 
       - name: Install dependencies
         if: steps.gate.outputs.run_job == 'true'
-        timeout-minutes: 10
+        timeout-minutes: 15
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
 
@@ -111,7 +112,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
-        timeout-minutes: 10
+        timeout-minutes: 15
         if: steps.gate.outputs.run_job == 'true'
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
@@ -123,6 +124,53 @@ jobs:
           pytest python/sglang/multimodal_gen/test/layers/test_musa_rmsnorm.py
           pytest python/sglang/multimodal_gen/test/layers/test_musa_silu_and_mul.py
 
+  # ==================== LLM server: 1-GPU ====================
+  nightly-test-llm-server-1-gpu-musa:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-1-gpu-runner
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-llm-server-1-gpu-musa" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 15
+        if: steps.gate.outputs.run_job == 'true'
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run DeepSeek-V2-Lite-Chat LLM server smoke test
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 120
+        run: |
+          cd test
+          python3 run_suite.py \
+            --hw musa \
+            --suite nightly-musa-1-gpu \
+            --nightly \
+            --timeout-per-file 7200
+
   # ==================== Multimodal: 1-GPU (split) ====================
   nightly-test-multimodal-server-1-gpu-musa:
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
@@ -159,7 +207,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
-        timeout-minutes: 10
+        timeout-minutes: 15
         if: steps.gate.outputs.run_job == 'true'
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
@@ -208,7 +256,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
-        timeout-minutes: 10
+        timeout-minutes: 15
         if: steps.gate.outputs.run_job == 'true'
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
@@ -230,6 +278,7 @@ jobs:
       [
         nightly-test-musa-general-kernel,
         nightly-test-musa-general-multimodal-layer,
+        nightly-test-llm-server-1-gpu-musa,
         nightly-test-multimodal-server-1-gpu-musa,
         nightly-test-multimodal-server-2-gpu-musa,
       ]

.github/workflows/pr-test-musa.yml

@@ -117,7 +117,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
-        timeout-minutes: 10
+        timeout-minutes: 15
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
 
@@ -152,7 +152,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
-        timeout-minutes: 10
+        timeout-minutes: 15
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
 
@@ -183,7 +183,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install dependencies
-        timeout-minutes: 10
+        timeout-minutes: 15
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
 
@@ -210,7 +210,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
-        timeout-minutes: 10
+        timeout-minutes: 15
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
 

.github/workflows/release-whl-kernel.yml

@@ -437,6 +437,18 @@ jobs:
         with:
           submodules: "recursive"
 
+      - name: Clean stale torchada extension locks
+        run: |
+          if ps -eo comm=,args= | grep -E '(^|[[:space:]])(mcc|ninja)([[:space:]]|$)|torchada_cpp_ops' | grep -v grep; then
+            echo "::error::Active torchada extension build detected; refusing to remove lock files"
+            exit 1
+          fi
+          find "${HOME}/.cache/torch_extensions" \
+            -path '*/torchada_cpp_ops/lock' \
+            -type f \
+            -print \
+            -delete 2>/dev/null || true
+
       - name: Build wheels
         run: |
           cd sgl-kernel