[MUSA][22/N] ci(musa): repack wheels with +musa metadata, refine path filters, sync multimodal tests, and add nightly workflow (#25074)

Co-authored-by: xinyue.fan <xinyue.fan@mthreads.com>

.github/workflows/nightly-test-musa.yml

@@ -0,0 +1,252 @@
+name: Nightly Test (MUSA)
+
+on:
+  schedule:
+    - cron: '0 16 * * *'
+  workflow_dispatch:
+    inputs:
+      job_filter:
+        description: "Select which job to run (empty/all to run all jobs)"
+        required: false
+        type: choice
+        default: 'all'
+        options:
+          - 'all'
+          - 'nightly-test-musa-general-kernel'
+          - 'nightly-test-musa-general-multimodal-layer'
+          - 'nightly-test-multimodal-server-1-gpu-musa'
+          - 'nightly-test-multimodal-server-2-gpu-musa'
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
+        required: false
+        type: string
+        default: ''
+      job_filter:
+        description: 'Select which job to run (empty or "all" to run all jobs)'
+        required: false
+        type: string
+        default: 'all'
+
+concurrency:
+  group: nightly-test-musa-${{ inputs.ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
+
+env:
+  SGLANG_IS_IN_CI: true
+
+jobs:
+  # ==================== General: kernel ====================
+  nightly-test-musa-general-kernel:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-1-gpu-runner
+    timeout-minutes: 240
+    env:
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HUB_CACHE: /hf-cache/hub
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-kernel" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        timeout-minutes: 10
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 10
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run sgl-kernel unit tests (MUSA)
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 30
+        run: |
+          pytest sgl-kernel/tests/test_dsv3_router_gemm.py
+          pytest sgl-kernel/tests/test_per_token_quant_fp8.py
+          pytest sgl-kernel/tests/speculative/test_eagle_utils.py
+          pytest sgl-kernel/tests/speculative/test_ngram_utils.py
+          pytest sgl-kernel/tests/speculative/test_speculative_sampling.py
+          pytest sgl-kernel/tests/test_torch_defaults_reset.py
+
+  # ==================== General: multimodal layer ====================
+  nightly-test-musa-general-multimodal-layer:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-1-gpu-runner
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-multimodal-layer" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run multimodal MUSA layer unit tests
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 30
+        run: |
+          pytest python/sglang/multimodal_gen/test/layers/test_musa_rmsnorm.py
+          pytest python/sglang/multimodal_gen/test/layers/test_musa_silu_and_mul.py
+
+  # ==================== Multimodal: 1-GPU (split) ====================
+  nightly-test-multimodal-server-1-gpu-musa:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-1-gpu-runner
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+      matrix:
+        part: [0, 1]
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-1-gpu-musa" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run diffusion server tests (1-GPU)
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 60
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite_musa.py \
+            --suite 1-gpu-musa-nightly \
+            --partition-id ${{ matrix.part }} \
+            --total-partitions 2 \
+            --continue-on-error
+
+  # ==================== Multimodal: 2-GPU ====================
+  nightly-test-multimodal-server-2-gpu-musa:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+    runs-on: s5000-2-gpu-runner
+    timeout-minutes: 240
+    env:
+      SGLANG_USE_MODELSCOPE: false
+      SGLANG_IS_IN_CI: true
+      TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
+      HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
+    steps:
+      - name: Gate by job_filter
+        id: gate
+        run: |
+          filter="${{ inputs.job_filter || 'all' }}"
+          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-2-gpu-musa" ]]; then
+            echo "run_job=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_job=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout code
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        timeout-minutes: 10
+        if: steps.gate.outputs.run_job == 'true'
+        run: |
+          bash scripts/ci/musa/musa_install_dependency.sh
+
+      - name: Run diffusion server tests (2-GPU)
+        if: steps.gate.outputs.run_job == 'true'
+        timeout-minutes: 60
+        env:
+          RUNAI_STREAMER_MEMORY_LIMIT: 0
+        run: |
+          cd python
+          python3 sglang/multimodal_gen/test/run_suite_musa.py \
+            --suite 2-gpu-musa \
+            --continue-on-error
+
+  # ==================== finish ====================
+  nightly-test-musa-finish:
+    needs:
+      [
+        nightly-test-musa-general-kernel,
+        nightly-test-musa-general-multimodal-layer,
+        nightly-test-multimodal-server-1-gpu-musa,
+        nightly-test-multimodal-server-2-gpu-musa,
+      ]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check all dependent job statuses
+        run: |
+          json_needs='${{ toJson(needs) }}'
+          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')
+
+          for job in $job_names; do
+            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
+            echo "$job: $result"
+            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
+              echo "Nightly failed."
+              exit 1
+            fi
+          done
+          echo "All jobs completed successfully."

.github/workflows/pr-test-musa.yml

@@ -68,19 +68,22 @@ jobs:
         with:
           filters: |
             main_package:
-              - "python/sglang/!(multimodal_gen)/**"
+              - ".github/workflows/pr-test-musa.yml"
               - "python/pyproject_other.toml"
+              - "python/sglang/!(multimodal_gen)/**"
+              - "python/sglang/srt/hardware_backend/musa/**"
               - "scripts/ci/musa/*"
               - "scripts/ci/utils/*"
-              - "test/**"
-              - ".github/workflows/pr-test-musa.yml"
             multimodal_gen:
-              - "python/sglang/multimodal_gen/**"
-              - "python/sglang/cli/**"
               - "python/pyproject_other.toml"
+              - "python/sglang/multimodal_gen/runtime/platforms/musa.py"
+              - "python/sglang/multimodal_gen/test/layers/test_musa_rmsnorm.py"
+              - "python/sglang/multimodal_gen/test/layers/test_musa_silu_and_mul.py"
+              - "python/sglang/multimodal_gen/test/run_suite_musa.py"
+              - "python/sglang/multimodal_gen/test/server/musa/**"
             sgl_kernel:
-              - "sgl-kernel/**"
               - ".github/workflows/pr-test-musa.yml"
+              - "sgl-kernel/csrc/musa/**"
 
   # ==================== PR Gate ==================== #
   pr-gate:
@@ -100,10 +103,12 @@ jobs:
     runs-on: s5000-1-gpu-runner
     timeout-minutes: 240
     env:
-      USE_MODELSCOPE: true
+      SGLANG_USE_MODELSCOPE: false
       SGLANG_IS_IN_CI: true
       TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
       HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
     steps:
       - name: Checkout code
         timeout-minutes: 10
@@ -112,6 +117,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
+        timeout-minutes: 10
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
 
@@ -132,10 +138,12 @@ jobs:
     runs-on: s5000-2-gpu-runner
     timeout-minutes: 240
     env:
-      USE_MODELSCOPE: true
+      SGLANG_USE_MODELSCOPE: false
       SGLANG_IS_IN_CI: true
       TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
       HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
     steps:
       - name: Checkout code
         timeout-minutes: 10
@@ -144,6 +152,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
+        timeout-minutes: 10
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
 
@@ -162,16 +171,19 @@ jobs:
     runs-on: s5000-1-gpu-runner
     timeout-minutes: 240
     env:
-      USE_MODELSCOPE: true
+      SGLANG_USE_MODELSCOPE: false
       SGLANG_IS_IN_CI: true
       TORCHADA_ENABLE_CPP_OPS: 1
+      HF_HOME: /hf-cache
       HF_HUB_CACHE: /hf-cache/hub
+      HF_HUB_OFFLINE: 1
     steps:
       - name: Checkout code
         timeout-minutes: 10
         uses: actions/checkout@v4
 
       - name: Install dependencies
+        timeout-minutes: 10
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
 
@@ -198,6 +210,7 @@ jobs:
           ref: ${{ inputs.ref || github.ref }}
 
       - name: Install dependencies
+        timeout-minutes: 10
         run: |
           bash scripts/ci/musa/musa_install_dependency.sh
 

python/sglang/multimodal_gen/test/run_suite_musa.py

@@ -22,11 +22,13 @@ logger = init_logger(__name__)
 
 SUITES = {
     "1-gpu-musa": [
-        "musa/test_server_a_musa.py",
-        "musa/test_server_b_musa.py",
+        "musa/test_server_1_gpu_musa.py",
+    ],
+    "1-gpu-musa-nightly": [
+        "musa/test_server_1_gpu_musa_nightly.py",
     ],
     "2-gpu-musa": [
-        "musa/test_server_2_gpu_a_musa.py",
+        "musa/test_server_2_gpu_musa.py",
     ],
 }
 

python/sglang/multimodal_gen/test/server/musa/perf_baselines_musa.json

@@ -1,140 +1,316 @@
 {
-    "metadata":{
-        "model":"Diffusion Server",
-        "hardware":"CI S5000 pool",
-        "description":"Reference numbers captured from the CI diffusion server baseline run"
+    "metadata": {
+        "model": "Diffusion Server",
+        "hardware": "CI S5000 pool",
+        "description": "Reference numbers captured from the CI diffusion server baseline run"
     },
-    "scenarios":{
-        "qwen_image_t2i_musa":{
-            "stages_ms":{
-                "InputValidationStage":0.09,
-                "TextEncodingStage":658.4,
-                "LatentPreparationStage":0.33,
-                "TimestepPreparationStage":24.39,
-                "DenoisingStage":36196.6,
-                "DecodingStage":40.44
+    "scenarios": {
+        "zimage_image_t2i_musa": {
+            "stages_ms": {
+                "InputValidationStage": 0.08,
+                "TextEncodingStage": 3761.4,
+                "LatentPreparationStage": 4.06,
+                "TimestepPreparationStage": 21.94,
+                "DenoisingStage": 4873.8,
+                "DecodingStage": 252.49
             },
-            "denoise_step_ms":{
-                "0":666.68,
-                "1":732.33,
-                "2":721.29,
-                "3":729.27,
-                "4":725.05,
-                "5":721.71,
-                "6":722.22,
-                "7":725.93,
-                "8":724.94,
-                "9":724.14,
-                "10":730.43,
-                "11":719.92,
-                "12":726.24,
-                "13":722.04,
-                "14":727.68,
-                "15":720.31,
-                "16":721.75,
-                "17":725.65,
-                "18":720.23,
-                "19":724.12,
-                "20":726.35,
-                "21":723.27,
-                "22":731.58,
-                "23":724.97,
-                "24":721.48,
-                "25":722.0,
-                "26":722.37,
-                "27":719.81,
-                "28":721.64,
-                "29":724.81,
-                "30":723.9,
-                "31":725.42,
-                "32":719.86,
-                "33":728.04,
-                "34":728.55,
-                "35":723.13,
-                "36":722.0,
-                "37":730.11,
-                "38":724.06,
-                "39":728.35,
-                "40":728.04,
-                "41":726.62,
-                "42":728.47,
-                "43":728.11,
-                "44":728.59,
-                "45":721.5,
-                "46":724.59,
-                "47":729.26,
-                "48":726.05,
-                "49":721.13
+            "denoise_step_ms": {
+                "0": 3281.35,
+                "2": 209.69,
+                "3": 199.26,
+                "5": 200.42,
+                "6": 207.8,
+                "8": 205.19
             },
-            "expected_e2e_ms":37190.98,
-            "expected_avg_denoise_ms":723.72,
-            "expected_median_denoise_ms":724.7
+            "expected_e2e_ms": 12161.85,
+            "expected_avg_denoise_ms": 540.67,
+            "expected_median_denoise_ms": 205.19,
+            "estimated_full_test_time_s": 64.6
         },
-        "wan2_1_t2v_1.3b_musa":{
-            "stages_ms":{
-                "InputValidationStage":0.12,
-                "TextEncodingStage":1097.75,
-                "LatentPreparationStage":0.24,
-                "TimestepPreparationStage":5.66,
-                "DenoisingStage":47399.84,
-                "DecodingStage":946.08,
-                "per_frame_generation":null
+        "qwen_image_layered_i2i_musa": {
+            "stages_ms": {
+                "QwenImageLayeredBeforeDenoisingStage": 287.12,
+                "TimestepPreparationStage": 0.01,
+                "DenoisingStage": 79962.68,
+                "DecodingStage": 232.86
             },
-            "denoise_step_ms":{
-                "0":783.06,
-                "1":970.52,
-                "2":939.72,
-                "3":947.58,
-                "4":941.44,
-                "5":955.26,
-                "6":960.39,
-                "7":951.84,
-                "8":959.68,
-                "9":953.33,
-                "10":940.87,
-                "11":958.5,
-                "12":952.7,
-                "13":933.4,
-                "14":952.0,
-                "15":951.6,
-                "16":947.04,
-                "17":939.28,
-                "18":956.88,
-                "19":960.1,
-                "20":949.73,
-                "21":954.77,
-                "22":959.98,
-                "23":947.37,
-                "24":957.51,
-                "25":953.39,
-                "26":953.73,
-                "27":959.57,
-                "28":942.59,
-                "29":958.05,
-                "30":952.76,
-                "31":952.76,
-                "32":950.6,
-                "33":948.76,
-                "34":957.53,
-                "35":940.86,
-                "36":958.11,
-                "37":940.9,
-                "38":949.1,
-                "39":951.81,
-                "40":948.61,
-                "41":957.28,
-                "42":951.41,
-                "43":953.09,
-                "44":955.69,
-                "45":941.93,
-                "46":952.96,
-                "47":953.5,
-                "48":939.25,
-                "49":942.69
+            "denoise_step_ms": {
+                "0": 1377.49,
+                "1": 1608.67,
+                "2": 1606.57,
+                "3": 1601.06,
+                "4": 1606.65,
+                "5": 1605.0,
+                "6": 1596.31,
+                "7": 1609.13,
+                "8": 1599.48,
+                "9": 1601.81,
+                "10": 1606.7,
+                "11": 1601.7,
+                "12": 1607.68,
+                "13": 1606.42,
+                "14": 1599.13,
+                "15": 1611.78,
+                "16": 1598.31,
+                "17": 1600.97,
+                "18": 1611.38,
+                "19": 1598.6,
+                "20": 1599.72,
+                "21": 1608.24,
+                "22": 1599.4,
+                "23": 1613.52,
+                "24": 1600.56,
+                "25": 1605.91,
+                "26": 1605.47,
+                "27": 1598.85,
+                "28": 1607.93,
+                "29": 1603.65,
+                "30": 1598.64,
+                "31": 1607.1,
+                "32": 1595.55,
+                "33": 1608.09,
+                "34": 1606.47,
+                "35": 1596.18,
+                "36": 1599.88,
+                "37": 1607.12,
+                "38": 1595.63,
+                "39": 1612.07,
+                "40": 1596.19,
+                "41": 1602.35,
+                "42": 1604.17,
+                "43": 1598.26,
+                "44": 1602.67,
+                "45": 1611.54,
+                "46": 1599.01,
+                "47": 1619.72,
+                "48": 1593.45,
+                "49": 1602.33
             },
-            "expected_e2e_ms":50007.17,
-            "expected_avg_denoise_ms":947.83,
-            "expected_median_denoise_ms":952.35
+            "expected_e2e_ms": 80490.53,
+            "expected_avg_denoise_ms": 1599.09,
+            "expected_median_denoise_ms": 1602.51,
+            "estimated_full_test_time_s": 159.4
+        },
+        "fast_hunyuan_video_musa": {
+            "stages_ms": {
+                "InputValidationStage": 0.08,
+                "TextEncodingStage": 4146.14,
+                "TimestepPreparationStage": 18.33,
+                "LatentPreparationStage": 2.42,
+                "DenoisingStage": 17236.59,
+                "DecodingStage": 7019.29
+            },
+            "denoise_step_ms": {
+                "0": 5462.17,
+                "1": 2338.96,
+                "2": 2355.67,
+                "3": 2353.45,
+                "4": 2359.64,
+                "5": 2363.04
+            },
+            "expected_e2e_ms": 32799.37,
+            "expected_avg_denoise_ms": 2872.16,
+            "expected_median_denoise_ms": 2357.66,
+            "estimated_full_test_time_s": 111.3
+        },
+        "qwen_image_t2i_musa": {
+            "stages_ms": {
+                "InputValidationStage": 0.09,
+                "TextEncodingStage": 658.4,
+                "LatentPreparationStage": 0.33,
+                "TimestepPreparationStage": 24.39,
+                "DenoisingStage": 36196.6,
+                "DecodingStage": 40.44
+            },
+            "denoise_step_ms": {
+                "0": 666.68,
+                "1": 732.33,
+                "2": 721.29,
+                "3": 729.27,
+                "4": 725.05,
+                "5": 721.71,
+                "6": 722.22,
+                "7": 725.93,
+                "8": 724.94,
+                "9": 724.14,
+                "10": 730.43,
+                "11": 719.92,
+                "12": 726.24,
+                "13": 722.04,
+                "14": 727.68,
+                "15": 720.31,
+                "16": 721.75,
+                "17": 725.65,
+                "18": 720.23,
+                "19": 724.12,
+                "20": 726.35,
+                "21": 723.27,
+                "22": 731.58,
+                "23": 724.97,
+                "24": 721.48,
+                "25": 722.0,
+                "26": 722.37,
+                "27": 719.81,
+                "28": 721.64,
+                "29": 724.81,
+                "30": 723.9,
+                "31": 725.42,
+                "32": 719.86,
+                "33": 728.04,
+                "34": 728.55,
+                "35": 723.13,
+                "36": 722.0,
+                "37": 730.11,
+                "38": 724.06,
+                "39": 728.35,
+                "40": 728.04,
+                "41": 726.62,
+                "42": 728.47,
+                "43": 728.11,
+                "44": 728.59,
+                "45": 721.5,
+                "46": 724.59,
+                "47": 729.26,
+                "48": 726.05,
+                "49": 721.13
+            },
+            "expected_e2e_ms": 37190.98,
+            "expected_avg_denoise_ms": 723.72,
+            "expected_median_denoise_ms": 724.7,
+            "estimated_full_test_time_s": 137.0
+        },
+        "qwen_image_2512_t2i_musa": {
+            "stages_ms": {
+                "InputValidationStage": 0.07,
+                "TextEncodingStage": 849.1,
+                "LatentPreparationStage": 0.26,
+                "TimestepPreparationStage": 21.95,
+                "DenoisingStage": 31935.97,
+                "DecodingStage": 98.62
+            },
+            "denoise_step_ms": {
+                "0": 553.87,
+                "1": 639.78,
+                "2": 640.39,
+                "3": 638.78,
+                "4": 640.13,
+                "5": 639.69,
+                "6": 640.19,
+                "7": 639.67,
+                "8": 638.95,
+                "9": 640.06,
+                "10": 639.8,
+                "11": 640.81,
+                "12": 638.7,
+                "13": 640.16,
+                "14": 639.76,
+                "15": 640.17,
+                "16": 639.26,
+                "17": 640.33,
+                "18": 639.91,
+                "19": 640.47,
+                "20": 640.1,
+                "21": 639.81,
+                "22": 639.05,
+                "23": 639.7,
+                "24": 640.02,
+                "25": 640.67,
+                "26": 639.04,
+                "27": 640.21,
+                "28": 639.78,
+                "29": 643.32,
+                "30": 640.2,
+                "31": 639.54,
+                "32": 640.14,
+                "33": 640.14,
+                "34": 639.54,
+                "35": 639.22,
+                "36": 639.94,
+                "37": 639.87,
+                "38": 639.41,
+                "39": 639.6,
+                "40": 638.5,
+                "41": 639.5,
+                "42": 639.35,
+                "43": 641.43,
+                "44": 638.73,
+                "45": 639.74,
+                "46": 639.72,
+                "47": 639.72,
+                "48": 638.64,
+                "49": 638.72
+            },
+            "expected_e2e_ms": 32915.75,
+            "expected_avg_denoise_ms": 638.55,
+            "expected_median_denoise_ms": 640.05,
+            "estimated_full_test_time_s": 137.8
+        },
+        "wan2_1_t2v_1.3b_musa": {
+            "stages_ms": {
+                "InputValidationStage": 0.06,
+                "TextEncodingStage": 1381.91,
+                "LatentPreparationStage": 0.18,
+                "TimestepPreparationStage": 3.99,
+                "DenoisingStage": 23679.86,
+                "DecodingStage": 1059.53,
+                "per_frame_generation": null
+            },
+            "denoise_step_ms": {
+                "0": 331.71,
+                "1": 478.93,
+                "2": 481.37,
+                "3": 483.42,
+                "4": 476.27,
+                "5": 488.73,
+                "6": 486.79,
+                "7": 467.89,
+                "8": 465.47,
+                "9": 472.81,
+                "10": 478.41,
+                "11": 488.84,
+                "12": 474.68,
+                "13": 468.48,
+                "14": 483.79,
+                "15": 476.05,
+                "16": 483.77,
+                "17": 476.04,
+                "18": 484.27,
+                "19": 486.43,
+                "20": 483.04,
+                "21": 473.76,
+                "22": 464.09,
+                "23": 474.14,
+                "24": 470.76,
+                "25": 487.99,
+                "26": 477.77,
+                "27": 465.76,
+                "28": 483.96,
+                "29": 484.14,
+                "30": 471.99,
+                "31": 483.41,
+                "32": 486.66,
+                "33": 467.07,
+                "34": 478.04,
+                "35": 476.9,
+                "36": 462.4,
+                "37": 476.53,
+                "38": 485.65,
+                "39": 478.89,
+                "40": 465.22,
+                "41": 472.03,
+                "42": 479.65,
+                "43": 479.74,
+                "44": 479.48,
+                "45": 474.19,
+                "46": 464.47,
+                "47": 463.09,
+                "48": 463.77,
+                "49": 463.49
+            },
+            "expected_e2e_ms": 26134.51,
+            "expected_avg_denoise_ms": 473.44,
+            "expected_median_denoise_ms": 476.72,
+            "estimated_full_test_time_s": 101.5
         },
         "wan2_2_t2v_a14b_2gpu_musa": {
             "stages_ms": {
@@ -259,6 +435,130 @@
             "expected_e2e_ms": 138624.98,
             "expected_avg_denoise_ms": 2686.91,
             "expected_median_denoise_ms": 2691.24
+        },
+        "qwen_image_edit_t2i_musa": {
+            "stages_ms": {
+                "InputValidationStage": 35.9,
+                "ImageEncodingStage": 7483.38,
+                "ImageVAEEncodingStage": 160.82,
+                "LatentPreparationStage": 3.0,
+                "TimestepPreparationStage": 19.02,
+                "DenoisingStage": 84367.86,
+                "DecodingStage": 36.82
+            },
+            "denoise_step_ms": {
+                "0": 7612.62,
+                "1": 1567.7,
+                "2": 1559.81,
+                "3": 1570.75,
+                "4": 1568.7,
+                "5": 1572.44,
+                "6": 1565.71,
+                "7": 1559.33,
+                "8": 1567.63,
+                "9": 1566.83,
+                "10": 1560.77,
+                "11": 1561.03,
+                "12": 1565.61,
+                "13": 1560.43,
+                "14": 1562.47,
+                "15": 1562.22,
+                "16": 1564.1,
+                "17": 1571.28,
+                "18": 1563.26,
+                "19": 1561.37,
+                "20": 1559.04,
+                "21": 1556.07,
+                "22": 1577.83,
+                "23": 1564.54,
+                "24": 1564.28,
+                "25": 1573.79,
+                "26": 1572.94,
+                "27": 1568.81,
+                "28": 1568.73,
+                "29": 1571.71,
+                "30": 1557.83,
+                "31": 1568.7,
+                "32": 1570.3,
+                "33": 1567.36,
+                "34": 1566.47,
+                "35": 1567.2,
+                "36": 1560.98,
+                "37": 1563.43,
+                "38": 1570.74,
+                "39": 1568.01,
+                "40": 1560.57,
+                "41": 1572.64,
+                "42": 1564.01,
+                "43": 1566.34,
+                "44": 1569.09,
+                "45": 1573.18,
+                "46": 1566.5,
+                "47": 1567.04,
+                "48": 1570.02,
+                "49": 1559.91
+            },
+            "expected_e2e_ms": 92355.9,
+            "expected_avg_denoise_ms": 1687.04,
+            "expected_median_denoise_ms": 1566.67,
+            "estimated_full_test_time_s": 217.0
+        },
+        "qwen_image_edit_2509_ti2i_musa": {
+            "stages_ms": {
+                "InputValidationStage": 125.7,
+                "ImageEncodingStage": 1018.93,
+                "ImageVAEEncodingStage": 311.41,
+                "LatentPreparationStage": 0.24,
+                "TimestepPreparationStage": 33.39,
+                "DenoisingStage": 88792.03,
+                "DecodingStage": 320.64
+            },
+            "denoise_step_ms": {
+                "0": 1914.75,
+                "1": 2230.29,
+                "2": 2216.93,
+                "3": 2231.22,
+                "4": 2230.63,
+                "5": 2222.13,
+                "6": 2224.43,
+                "7": 2235.47,
+                "8": 2220.55,
+                "9": 2239.83,
+                "10": 2239.29,
+                "11": 2216.95,
+                "12": 2221.39,
+                "13": 2229.65,
+                "14": 2231.94,
+                "15": 2222.23,
+                "16": 2230.03,
+                "17": 2236.55,
+                "18": 2217.18,
+                "19": 2231.48,
+                "20": 2236.88,
+                "21": 2226.74,
+                "22": 2224.26,
+                "23": 2231.1,
+                "24": 2214.29,
+                "25": 2224.57,
+                "26": 2233.64,
+                "27": 2217.0,
+                "28": 2226.08,
+                "29": 2229.47,
+                "30": 2230.21,
+                "31": 2224.45,
+                "32": 2230.89,
+                "33": 2232.82,
+                "34": 2219.97,
+                "35": 2228.74,
+                "36": 2231.6,
+                "37": 2225.25,
+                "38": 2223.72,
+                "39": 2228.41
+            },
+            "expected_e2e_ms": 90612.84,
+            "expected_avg_denoise_ms": 2219.58,
+            "expected_median_denoise_ms": 2228.58,
+            "estimated_full_test_time_s": 220.8
         }
     }
 }

python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa.py

@@ -1,5 +1,5 @@
 """
-MUSA-specific diffusion performance test (1-GPU).
+MUSA-specific 1-GPU diffusion performance tests.
 """
 
 from __future__ import annotations
@@ -8,7 +8,7 @@ import pytest
 
 from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
 from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import (
-    ONE_GPU_MUSA_CASES_A,
+    ONE_GPU_MUSA_CASES,
 )
 from sglang.multimodal_gen.test.server.test_server_common import (  # noqa: F401
     DiffusionServerBase,
@@ -19,10 +19,10 @@ from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase
 logger = init_logger(__name__)
 
 
-class TestDiffusionServerOneGpuMusaImage(DiffusionServerBase):
-    """Performance tests for 1-GPU diffusion cases on MUSA"""
+class TestDiffusionServerOneGpuMusa(DiffusionServerBase):
+    """Performance tests for 1-GPU diffusion cases on MUSA."""
 
-    @pytest.fixture(params=ONE_GPU_MUSA_CASES_A, ids=lambda c: c.id)
+    @pytest.fixture(params=ONE_GPU_MUSA_CASES, ids=lambda c: c.id)
     def case(self, request) -> DiffusionTestCase:
         """Provide a DiffusionTestCase for each 1-GPU MUSA test."""
         return request.param

python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa_nightly.py

@@ -1,5 +1,5 @@
 """
-MUSA-specific diffusion performance test (1-GPU).
+MUSA-specific 1-GPU diffusion performance tests for nightly suite.
 """
 
 from __future__ import annotations
@@ -8,7 +8,7 @@ import pytest
 
 from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
 from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import (
-    ONE_GPU_MUSA_CASES_B,
+    ONE_GPU_NIGHTLY_MUSA_CASES,
 )
 from sglang.multimodal_gen.test.server.test_server_common import (  # noqa: F401
     DiffusionServerBase,
@@ -19,10 +19,10 @@ from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase
 logger = init_logger(__name__)
 
 
-class TestDiffusionServerOneGpuMusaVideo(DiffusionServerBase):
-    """Performance tests for 1-GPU diffusion cases on MUSA"""
+class TestDiffusionServerOneGpuMusaNightly(DiffusionServerBase):
+    """Performance tests for 1-GPU diffusion cases on MUSA (nightly-only)."""
 
-    @pytest.fixture(params=ONE_GPU_MUSA_CASES_B, ids=lambda c: c.id)
+    @pytest.fixture(params=ONE_GPU_NIGHTLY_MUSA_CASES, ids=lambda c: c.id)
     def case(self, request) -> DiffusionTestCase:
-        """Provide a DiffusionTestCase for each 1-GPU MUSA test."""
+        """Provide a DiffusionTestCase for each 1-GPU MUSA nightly test."""
         return request.param

python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_musa.py

@@ -1,5 +1,5 @@
 """
-MUSA-specific 2-GPU diffusion performance test.
+MUSA-specific 2-GPU diffusion performance tests.
 """
 
 from __future__ import annotations
@@ -8,7 +8,7 @@ import pytest
 
 from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
 from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import (
-    TWO_GPU_MUSA_CASES_A,
+    TWO_GPU_MUSA_CASES,
 )
 from sglang.multimodal_gen.test.server.test_server_common import (  # noqa: F401
     DiffusionServerBase,
@@ -19,10 +19,10 @@ from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase
 logger = init_logger(__name__)
 
 
-class TestDiffusionServerTwoGpuMusaA(DiffusionServerBase):
+class TestDiffusionServerTwoGpuMusa(DiffusionServerBase):
     """Performance tests for 2-GPU diffusion cases on MUSA."""
 
-    @pytest.fixture(params=TWO_GPU_MUSA_CASES_A, ids=lambda c: c.id)
+    @pytest.fixture(params=TWO_GPU_MUSA_CASES, ids=lambda c: c.id)
     def case(self, request) -> DiffusionTestCase:
         """Provide a DiffusionTestCase for each 2-GPU MUSA test."""
         return request.param

python/sglang/multimodal_gen/test/server/musa/testcase_configs_musa.py

@@ -1,32 +1,49 @@
 from __future__ import annotations
 
+from dataclasses import replace
+from functools import lru_cache
+
 from sglang.multimodal_gen.test.server.testcase_configs import (
     T2V_PROMPT,
     DiffusionSamplingParams,
     DiffusionServerArgs,
     DiffusionTestCase,
+    MULTI_FRAME_I2I_sampling_params,
+    MULTI_IMAGE_TI2I_sampling_params,
     T2I_sampling_params,
+    T2V_sampling_params,
+    TI2I_sampling_params,
     TI2V_sampling_params,
 )
 
-ONE_GPU_MUSA_CASES_A: list[DiffusionTestCase] = [
+
+@lru_cache(maxsize=None)
+def hf_cached_model(repo_id: str) -> str:
+    """Resolve an HF repo id to the local cache snapshot prepared on MUSA runners."""
+    from huggingface_hub import snapshot_download
+
+    return snapshot_download(repo_id, local_files_only=True)
+
+
+MUSA_TI2I_sampling_params = replace(
+    TI2I_sampling_params,
+    image_path="/hf-cache/hub/musa-test-assets/TI2I_Qwen_Image_Edit_Input.jpg",
+)
+
+ONE_GPU_MUSA_CASES: list[DiffusionTestCase] = [
     DiffusionTestCase(
         "qwen_image_t2i_musa",
         DiffusionServerArgs(
-            model_path="Qwen/Qwen-Image",
+            model_path=hf_cached_model("Qwen/Qwen-Image"),
             modality="image",
         ),
         T2I_sampling_params,
         run_consistency_check=False,
     ),
-]
-
-
-ONE_GPU_MUSA_CASES_B: list[DiffusionTestCase] = [
     DiffusionTestCase(
         "wan2_1_t2v_1.3b_musa",
         DiffusionServerArgs(
-            model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
+            model_path=hf_cached_model("Wan-AI/Wan2.1-T2V-1.3B-Diffusers"),
             modality="video",
             custom_validator="video",
         ),
@@ -38,11 +55,75 @@ ONE_GPU_MUSA_CASES_B: list[DiffusionTestCase] = [
 ]
 
 
-TWO_GPU_MUSA_CASES_A: list[DiffusionTestCase] = [
+NIGHTLY_1_GPU_MUSA_CASES: list[DiffusionTestCase] = [
+    DiffusionTestCase(
+        "zimage_image_t2i_musa",
+        DiffusionServerArgs(
+            model_path=hf_cached_model("Tongyi-MAI/Z-Image-Turbo"),
+            modality="image",
+        ),
+        T2I_sampling_params,
+        run_consistency_check=False,
+    ),
+    DiffusionTestCase(
+        "qwen_image_layered_i2i_musa",
+        DiffusionServerArgs(
+            model_path=hf_cached_model("Qwen/Qwen-Image-Layered"),
+            modality="image",
+        ),
+        MULTI_FRAME_I2I_sampling_params,
+        run_consistency_check=False,
+    ),
+    DiffusionTestCase(
+        "fast_hunyuan_video_musa",
+        DiffusionServerArgs(
+            model_path=hf_cached_model("FastVideo/FastHunyuan-diffusers"),
+            modality="video",
+            custom_validator="video",
+        ),
+        T2V_sampling_params,
+        run_consistency_check=False,
+    ),
+    DiffusionTestCase(
+        "qwen_image_2512_t2i_musa",
+        DiffusionServerArgs(
+            model_path=hf_cached_model("Qwen/Qwen-Image-2512"),
+            modality="image",
+        ),
+        T2I_sampling_params,
+        run_consistency_check=False,
+    ),
+    DiffusionTestCase(
+        "qwen_image_edit_t2i_musa",
+        DiffusionServerArgs(
+            model_path=hf_cached_model("Qwen/Qwen-Image-Edit"),
+            modality="image",
+        ),
+        MUSA_TI2I_sampling_params,
+        run_consistency_check=False,
+    ),
+    DiffusionTestCase(
+        "qwen_image_edit_2509_ti2i_musa",
+        DiffusionServerArgs(
+            model_path=hf_cached_model("Qwen/Qwen-Image-Edit-2509"),
+            modality="image",
+        ),
+        MULTI_IMAGE_TI2I_sampling_params,
+        run_consistency_check=False,
+    ),
+]
+
+
+ONE_GPU_NIGHTLY_MUSA_CASES: list[DiffusionTestCase] = (
+    ONE_GPU_MUSA_CASES + NIGHTLY_1_GPU_MUSA_CASES
+)
+
+
+TWO_GPU_MUSA_CASES: list[DiffusionTestCase] = [
     DiffusionTestCase(
         "wan2_1_i2v_14b_480P_2gpu_musa",
         DiffusionServerArgs(
-            model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
+            model_path=hf_cached_model("Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"),
             modality="video",
             custom_validator="video",
             num_gpus=2,

scripts/ci/musa/rename_wheels_musa.sh

@@ -1,46 +1,102 @@
 #!/usr/bin/env bash
-set -euo pipefail
-
-# Rename MUSA wheels to include a +musa<suffix> build tag.
+# Align MUSA wheel filenames (+musa43/...) with internal METADATA Version and
+# WHEEL tags after build. Two drifts need fixing in lockstep:
+#   - METADATA `Version:` must carry the `+musa<suffix>` local version, or
+#     recent pip versions reject the wheel with "inconsistent version".
+#   - WHEEL `Tag:` must be `manylinux2014_*` when the filename says so;
+#     leaving it as `linux_*` can trip installers that re-derive the platform.
+# Unpack → patch WHEEL/METADATA → wheel pack (RECORD regenerated; no hand-editing).
+#
 # Usage:
 #   rename_wheels_musa.sh <musa_suffix> [wheel_dir]
 # Example:
 #   rename_wheels_musa.sh 43 sgl-kernel/dist
+set -euxo pipefail
 
 if [[ $# -lt 1 || $# -gt 2 ]]; then
   echo "Usage: $0 <musa_suffix> [wheel_dir]" >&2
   exit 1
 fi
 
-MUSA_SUFFIX="$1"
+MUSA_SUFFIX="+musa$1"
 WHEEL_DIR="${2:-dist}"
 
-wheel_files=("$WHEEL_DIR"/*.whl)
+patch_wheel_platform_tags() {
+    local wheel_file="$1"
+    # Line-end anchors: "linux_x86_64" is a substring of "manylinux2014_x86_64", so
+    # unanchored global replace corrupts tags on a second run.
+    sed -i \
+        -e 's/-linux_x86_64$/-manylinux2014_x86_64/' \
+        -e 's/-linux_aarch64$/-manylinux2014_aarch64/' \
+        "$wheel_file"
+}
 
+wheel_files=("$WHEEL_DIR"/*.whl)
 if [[ ! -e "${wheel_files[0]}" ]]; then
   echo "No wheel files found in ${WHEEL_DIR}/, nothing to rename."
   exit 0
 fi
 
 for wheel in "${wheel_files[@]}"; do
-  # Normalize platform tag to manylinux2014
-  intermediate_wheel="${wheel/linux/manylinux2014}"
+    [[ -f "$wheel" ]] || continue
 
-  # Extract Python ABI version (e.g. cp310)
-  if [[ $intermediate_wheel =~ -cp([0-9]+)- ]]; then
-    cp_version="${BASH_REMATCH[1]}"
-  else
-    echo "Could not extract Python version from wheel name: $intermediate_wheel" >&2
-    continue
-  fi
+    intermediate_wheel="$wheel"
+    case "$wheel" in
+        *-linux_x86_64.whl)
+            intermediate_wheel="${wheel%-linux_x86_64.whl}-manylinux2014_x86_64.whl"
+            ;;
+        *-linux_aarch64.whl)
+            intermediate_wheel="${wheel%-linux_aarch64.whl}-manylinux2014_aarch64.whl"
+            ;;
+    esac
+    if [[ "$wheel" != "$intermediate_wheel" ]]; then
+        mv -- "$wheel" "$intermediate_wheel"
+        wheel="$intermediate_wheel"
+    fi
 
-  # Insert +musa<suffix> before the Python ABI tag
-  new_wheel="${intermediate_wheel/-cp${cp_version}/+musa${MUSA_SUFFIX}-cp${cp_version}}"
+    TMPDIR=$(mktemp -d)
+    trap 'rm -rf -- "$TMPDIR"' ERR
 
-  if [[ "$wheel" != "$new_wheel" ]]; then
-    echo "Renaming $wheel -> $new_wheel"
-    mv -- "$wheel" "$new_wheel"
-  fi
+    "${PYTHON:-python3}" -m wheel unpack "$wheel" --dest "$TMPDIR"
+    # `find | head -1` succeeds with empty stdout when there are no matches —
+    # `set -e` won't catch that. Assert each path is real so a malformed wheel
+    # surfaces with a useful message instead of a downstream `sed: /WHEEL` error.
+    UNPACKED=$(find "$TMPDIR" -mindepth 1 -maxdepth 1 -type d | head -1)
+    [[ -d "$UNPACKED" ]] || { echo "ERROR: wheel unpack produced no top-level dir for $wheel" >&2; exit 1; }
+    DIST_INFO=$(find "$UNPACKED" -maxdepth 1 -type d -name "*.dist-info" | head -1)
+    [[ -d "$DIST_INFO" ]] || { echo "ERROR: no *.dist-info under $UNPACKED (malformed wheel?): $wheel" >&2; exit 1; }
+    WHEEL_META="${DIST_INFO}/WHEEL"
+    METADATA_FILE="${DIST_INFO}/METADATA"
+    [[ -f "$WHEEL_META" && -f "$METADATA_FILE" ]] || { echo "ERROR: missing WHEEL or METADATA in $DIST_INFO" >&2; exit 1; }
+
+    patch_wheel_platform_tags "$WHEEL_META"
+
+    ORIG_VERSION=$(grep '^Version:' "$METADATA_FILE" | head -1 | sed 's/^Version:[[:space:]]*//')
+    # Empty ORIG_VERSION would fall through the `+musa` check below and silently
+    # produce `Version: +musa43` — a broken release. Fail loud instead.
+    [[ -n "$ORIG_VERSION" ]] || { echo "ERROR: no 'Version:' line in $METADATA_FILE" >&2; exit 1; }
+    if [[ "$ORIG_VERSION" == *"$MUSA_SUFFIX"* ]]; then
+        echo "Skipping $wheel: version in METADATA is already suffixed."
+        rm -rf "$TMPDIR"
+        trap - ERR
+        continue
+    fi
+    NEW_VERSION="${ORIG_VERSION}${MUSA_SUFFIX}"
+    sed -i "s/^Version:.*/Version: ${NEW_VERSION}/" "$METADATA_FILE"
+    # `sed -i` exits 0 even when the pattern matched zero lines. Verify the
+    # rewrite actually landed before we publish.
+    grep -qx "Version: ${NEW_VERSION}" "$METADATA_FILE" || { echo "ERROR: METADATA Version rewrite did not land in $METADATA_FILE" >&2; exit 1; }
+
+    OLD_BASE=$(basename "$DIST_INFO")
+    NEW_BASE="${OLD_BASE/${ORIG_VERSION}/${NEW_VERSION}}"
+    # `${var/pat/repl}` silently leaves var unchanged if pat is empty or absent.
+    [[ "$NEW_BASE" != "$OLD_BASE" ]] || { echo "ERROR: dist-info dir '$OLD_BASE' did not contain ORIG_VERSION='$ORIG_VERSION'" >&2; exit 1; }
+    mv "$DIST_INFO" "${UNPACKED}/${NEW_BASE}"
+
+    rm -f "$wheel"
+    "${PYTHON:-python3}" -m wheel pack "$UNPACKED" --dest-dir "$WHEEL_DIR"
+    rm -rf "$TMPDIR"
+    trap - ERR
 done
 
 echo "MUSA wheel renaming completed."