diff --git a/.github/workflows/nightly-test-musa.yml b/.github/workflows/nightly-test-musa.yml new file mode 100644 index 000000000..9216800dd --- /dev/null +++ b/.github/workflows/nightly-test-musa.yml @@ -0,0 +1,252 @@ +name: Nightly Test (MUSA) + +on: + schedule: + - cron: '0 16 * * *' + workflow_dispatch: + inputs: + job_filter: + description: "Select which job to run (empty/all to run all jobs)" + required: false + type: choice + default: 'all' + options: + - 'all' + - 'nightly-test-musa-general-kernel' + - 'nightly-test-musa-general-multimodal-layer' + - 'nightly-test-multimodal-server-1-gpu-musa' + - 'nightly-test-multimodal-server-2-gpu-musa' + workflow_call: + inputs: + ref: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + required: false + type: string + default: '' + job_filter: + description: 'Select which job to run (empty or "all" to run all jobs)' + required: false + type: string + default: 'all' + +concurrency: + group: nightly-test-musa-${{ inputs.ref || github.ref }} + cancel-in-progress: ${{ github.event_name != 'workflow_call' }} + +env: + SGLANG_IS_IN_CI: true + +jobs: + # ==================== General: kernel ==================== + nightly-test-musa-general-kernel: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + runs-on: s5000-1-gpu-runner + timeout-minutes: 240 + env: + TORCHADA_ENABLE_CPP_OPS: 1 + HF_HUB_CACHE: /hf-cache/hub + steps: + - name: Gate by job_filter + id: gate + run: | + filter="${{ inputs.job_filter || 'all' }}" + if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-kernel" ]]; then + echo "run_job=true" >> "$GITHUB_OUTPUT" + else + echo "run_job=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout code + if: steps.gate.outputs.run_job == 'true' + uses: actions/checkout@v4 + timeout-minutes: 10 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 10 + run: | + bash scripts/ci/musa/musa_install_dependency.sh + + - name: Run sgl-kernel unit tests (MUSA) + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 30 + run: | + pytest sgl-kernel/tests/test_dsv3_router_gemm.py + pytest sgl-kernel/tests/test_per_token_quant_fp8.py + pytest sgl-kernel/tests/speculative/test_eagle_utils.py + pytest sgl-kernel/tests/speculative/test_ngram_utils.py + pytest sgl-kernel/tests/speculative/test_speculative_sampling.py + pytest sgl-kernel/tests/test_torch_defaults_reset.py + + # ==================== General: multimodal layer ==================== + nightly-test-musa-general-multimodal-layer: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + runs-on: s5000-1-gpu-runner + timeout-minutes: 240 + env: + SGLANG_USE_MODELSCOPE: false + SGLANG_IS_IN_CI: true + TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 + steps: + - name: Gate by job_filter + id: gate + run: | + filter="${{ inputs.job_filter || 'all' }}" + if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-multimodal-layer" ]]; then + echo "run_job=true" >> "$GITHUB_OUTPUT" + else + echo "run_job=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout code + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + run: | + bash scripts/ci/musa/musa_install_dependency.sh + + - name: Run multimodal MUSA layer unit tests + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 30 + run: | + pytest python/sglang/multimodal_gen/test/layers/test_musa_rmsnorm.py + pytest python/sglang/multimodal_gen/test/layers/test_musa_silu_and_mul.py + + # ==================== Multimodal: 1-GPU (split) ==================== + nightly-test-multimodal-server-1-gpu-musa: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + runs-on: s5000-1-gpu-runner + strategy: + fail-fast: false + max-parallel: 2 + matrix: + part: [0, 1] + timeout-minutes: 240 + env: + SGLANG_USE_MODELSCOPE: false + SGLANG_IS_IN_CI: true + TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 + steps: + - name: Gate by job_filter + id: gate + run: | + filter="${{ inputs.job_filter || 'all' }}" + if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-1-gpu-musa" ]]; then + echo "run_job=true" >> "$GITHUB_OUTPUT" + else + echo "run_job=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout code + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + run: | + bash scripts/ci/musa/musa_install_dependency.sh + + - name: Run diffusion server tests (1-GPU) + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 60 + env: + RUNAI_STREAMER_MEMORY_LIMIT: 0 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite_musa.py \ + --suite 1-gpu-musa-nightly \ + --partition-id ${{ matrix.part }} \ + --total-partitions 2 \ + --continue-on-error + + # ==================== Multimodal: 2-GPU ==================== + nightly-test-multimodal-server-2-gpu-musa: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' + runs-on: s5000-2-gpu-runner + timeout-minutes: 240 + env: + SGLANG_USE_MODELSCOPE: false + SGLANG_IS_IN_CI: true + TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache + HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 + steps: + - name: Gate by job_filter + id: gate + run: | + filter="${{ inputs.job_filter || 'all' }}" + if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-2-gpu-musa" ]]; then + echo "run_job=true" >> "$GITHUB_OUTPUT" + else + echo "run_job=false" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout code + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Install dependencies + timeout-minutes: 10 + if: steps.gate.outputs.run_job == 'true' + run: | + bash scripts/ci/musa/musa_install_dependency.sh + + - name: Run diffusion server tests (2-GPU) + if: steps.gate.outputs.run_job == 'true' + timeout-minutes: 60 + env: + RUNAI_STREAMER_MEMORY_LIMIT: 0 + run: | + cd python + python3 sglang/multimodal_gen/test/run_suite_musa.py \ + --suite 2-gpu-musa \ + --continue-on-error + + # ==================== finish ==================== + nightly-test-musa-finish: + needs: + [ + nightly-test-musa-general-kernel, + nightly-test-musa-general-multimodal-layer, + nightly-test-multimodal-server-1-gpu-musa, + nightly-test-multimodal-server-2-gpu-musa, + ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + json_needs='${{ toJson(needs) }}' + job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]') + + for job in $job_names; do + result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result') + echo "$job: $result" + if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then + echo "Nightly failed." + exit 1 + fi + done + echo "All jobs completed successfully." diff --git a/.github/workflows/pr-test-musa.yml b/.github/workflows/pr-test-musa.yml index 21381c652..5d6ae58da 100644 --- a/.github/workflows/pr-test-musa.yml +++ b/.github/workflows/pr-test-musa.yml @@ -68,19 +68,22 @@ jobs: with: filters: | main_package: - - "python/sglang/!(multimodal_gen)/**" + - ".github/workflows/pr-test-musa.yml" - "python/pyproject_other.toml" + - "python/sglang/!(multimodal_gen)/**" + - "python/sglang/srt/hardware_backend/musa/**" - "scripts/ci/musa/*" - "scripts/ci/utils/*" - - "test/**" - - ".github/workflows/pr-test-musa.yml" multimodal_gen: - - "python/sglang/multimodal_gen/**" - - "python/sglang/cli/**" - "python/pyproject_other.toml" + - "python/sglang/multimodal_gen/runtime/platforms/musa.py" + - "python/sglang/multimodal_gen/test/layers/test_musa_rmsnorm.py" + - "python/sglang/multimodal_gen/test/layers/test_musa_silu_and_mul.py" + - "python/sglang/multimodal_gen/test/run_suite_musa.py" + - "python/sglang/multimodal_gen/test/server/musa/**" sgl_kernel: - - "sgl-kernel/**" - ".github/workflows/pr-test-musa.yml" + - "sgl-kernel/csrc/musa/**" # ==================== PR Gate ==================== # pr-gate: @@ -100,10 +103,12 @@ jobs: runs-on: s5000-1-gpu-runner timeout-minutes: 240 env: - USE_MODELSCOPE: true + SGLANG_USE_MODELSCOPE: false SGLANG_IS_IN_CI: true TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 steps: - name: Checkout code timeout-minutes: 10 @@ -112,6 +117,7 @@ jobs: ref: ${{ inputs.ref || github.ref }} - name: Install dependencies + timeout-minutes: 10 run: | bash scripts/ci/musa/musa_install_dependency.sh @@ -132,10 +138,12 @@ jobs: runs-on: s5000-2-gpu-runner timeout-minutes: 240 env: - USE_MODELSCOPE: true + SGLANG_USE_MODELSCOPE: false SGLANG_IS_IN_CI: true TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 steps: - name: Checkout code timeout-minutes: 10 @@ -144,6 +152,7 @@ jobs: ref: ${{ inputs.ref || github.ref }} - name: Install dependencies + timeout-minutes: 10 run: | bash scripts/ci/musa/musa_install_dependency.sh @@ -162,16 +171,19 @@ jobs: runs-on: s5000-1-gpu-runner timeout-minutes: 240 env: - USE_MODELSCOPE: true + SGLANG_USE_MODELSCOPE: false SGLANG_IS_IN_CI: true TORCHADA_ENABLE_CPP_OPS: 1 + HF_HOME: /hf-cache HF_HUB_CACHE: /hf-cache/hub + HF_HUB_OFFLINE: 1 steps: - name: Checkout code timeout-minutes: 10 uses: actions/checkout@v4 - name: Install dependencies + timeout-minutes: 10 run: | bash scripts/ci/musa/musa_install_dependency.sh @@ -198,6 +210,7 @@ jobs: ref: ${{ inputs.ref || github.ref }} - name: Install dependencies + timeout-minutes: 10 run: | bash scripts/ci/musa/musa_install_dependency.sh diff --git a/python/sglang/multimodal_gen/test/run_suite_musa.py b/python/sglang/multimodal_gen/test/run_suite_musa.py index b8cd91e22..5ec408ef0 100644 --- a/python/sglang/multimodal_gen/test/run_suite_musa.py +++ b/python/sglang/multimodal_gen/test/run_suite_musa.py @@ -22,11 +22,13 @@ logger = init_logger(__name__) SUITES = { "1-gpu-musa": [ - "musa/test_server_a_musa.py", - "musa/test_server_b_musa.py", + "musa/test_server_1_gpu_musa.py", + ], + "1-gpu-musa-nightly": [ + "musa/test_server_1_gpu_musa_nightly.py", ], "2-gpu-musa": [ - "musa/test_server_2_gpu_a_musa.py", + "musa/test_server_2_gpu_musa.py", ], } diff --git a/python/sglang/multimodal_gen/test/server/musa/perf_baselines_musa.json b/python/sglang/multimodal_gen/test/server/musa/perf_baselines_musa.json index 464544501..e71766e37 100644 --- a/python/sglang/multimodal_gen/test/server/musa/perf_baselines_musa.json +++ b/python/sglang/multimodal_gen/test/server/musa/perf_baselines_musa.json @@ -1,140 +1,316 @@ { - "metadata":{ - "model":"Diffusion Server", - "hardware":"CI S5000 pool", - "description":"Reference numbers captured from the CI diffusion server baseline run" + "metadata": { + "model": "Diffusion Server", + "hardware": "CI S5000 pool", + "description": "Reference numbers captured from the CI diffusion server baseline run" }, - "scenarios":{ - "qwen_image_t2i_musa":{ - "stages_ms":{ - "InputValidationStage":0.09, - "TextEncodingStage":658.4, - "LatentPreparationStage":0.33, - "TimestepPreparationStage":24.39, - "DenoisingStage":36196.6, - "DecodingStage":40.44 + "scenarios": { + "zimage_image_t2i_musa": { + "stages_ms": { + "InputValidationStage": 0.08, + "TextEncodingStage": 3761.4, + "LatentPreparationStage": 4.06, + "TimestepPreparationStage": 21.94, + "DenoisingStage": 4873.8, + "DecodingStage": 252.49 }, - "denoise_step_ms":{ - "0":666.68, - "1":732.33, - "2":721.29, - "3":729.27, - "4":725.05, - "5":721.71, - "6":722.22, - "7":725.93, - "8":724.94, - "9":724.14, - "10":730.43, - "11":719.92, - "12":726.24, - "13":722.04, - "14":727.68, - "15":720.31, - "16":721.75, - "17":725.65, - "18":720.23, - "19":724.12, - "20":726.35, - "21":723.27, - "22":731.58, - "23":724.97, - "24":721.48, - "25":722.0, - "26":722.37, - "27":719.81, - "28":721.64, - "29":724.81, - "30":723.9, - "31":725.42, - "32":719.86, - "33":728.04, - "34":728.55, - "35":723.13, - "36":722.0, - "37":730.11, - "38":724.06, - "39":728.35, - "40":728.04, - "41":726.62, - "42":728.47, - "43":728.11, - "44":728.59, - "45":721.5, - "46":724.59, - "47":729.26, - "48":726.05, - "49":721.13 + "denoise_step_ms": { + "0": 3281.35, + "2": 209.69, + "3": 199.26, + "5": 200.42, + "6": 207.8, + "8": 205.19 }, - "expected_e2e_ms":37190.98, - "expected_avg_denoise_ms":723.72, - "expected_median_denoise_ms":724.7 + "expected_e2e_ms": 12161.85, + "expected_avg_denoise_ms": 540.67, + "expected_median_denoise_ms": 205.19, + "estimated_full_test_time_s": 64.6 }, - "wan2_1_t2v_1.3b_musa":{ - "stages_ms":{ - "InputValidationStage":0.12, - "TextEncodingStage":1097.75, - "LatentPreparationStage":0.24, - "TimestepPreparationStage":5.66, - "DenoisingStage":47399.84, - "DecodingStage":946.08, - "per_frame_generation":null + "qwen_image_layered_i2i_musa": { + "stages_ms": { + "QwenImageLayeredBeforeDenoisingStage": 287.12, + "TimestepPreparationStage": 0.01, + "DenoisingStage": 79962.68, + "DecodingStage": 232.86 }, - "denoise_step_ms":{ - "0":783.06, - "1":970.52, - "2":939.72, - "3":947.58, - "4":941.44, - "5":955.26, - "6":960.39, - "7":951.84, - "8":959.68, - "9":953.33, - "10":940.87, - "11":958.5, - "12":952.7, - "13":933.4, - "14":952.0, - "15":951.6, - "16":947.04, - "17":939.28, - "18":956.88, - "19":960.1, - "20":949.73, - "21":954.77, - "22":959.98, - "23":947.37, - "24":957.51, - "25":953.39, - "26":953.73, - "27":959.57, - "28":942.59, - "29":958.05, - "30":952.76, - "31":952.76, - "32":950.6, - "33":948.76, - "34":957.53, - "35":940.86, - "36":958.11, - "37":940.9, - "38":949.1, - "39":951.81, - "40":948.61, - "41":957.28, - "42":951.41, - "43":953.09, - "44":955.69, - "45":941.93, - "46":952.96, - "47":953.5, - "48":939.25, - "49":942.69 + "denoise_step_ms": { + "0": 1377.49, + "1": 1608.67, + "2": 1606.57, + "3": 1601.06, + "4": 1606.65, + "5": 1605.0, + "6": 1596.31, + "7": 1609.13, + "8": 1599.48, + "9": 1601.81, + "10": 1606.7, + "11": 1601.7, + "12": 1607.68, + "13": 1606.42, + "14": 1599.13, + "15": 1611.78, + "16": 1598.31, + "17": 1600.97, + "18": 1611.38, + "19": 1598.6, + "20": 1599.72, + "21": 1608.24, + "22": 1599.4, + "23": 1613.52, + "24": 1600.56, + "25": 1605.91, + "26": 1605.47, + "27": 1598.85, + "28": 1607.93, + "29": 1603.65, + "30": 1598.64, + "31": 1607.1, + "32": 1595.55, + "33": 1608.09, + "34": 1606.47, + "35": 1596.18, + "36": 1599.88, + "37": 1607.12, + "38": 1595.63, + "39": 1612.07, + "40": 1596.19, + "41": 1602.35, + "42": 1604.17, + "43": 1598.26, + "44": 1602.67, + "45": 1611.54, + "46": 1599.01, + "47": 1619.72, + "48": 1593.45, + "49": 1602.33 }, - "expected_e2e_ms":50007.17, - "expected_avg_denoise_ms":947.83, - "expected_median_denoise_ms":952.35 + "expected_e2e_ms": 80490.53, + "expected_avg_denoise_ms": 1599.09, + "expected_median_denoise_ms": 1602.51, + "estimated_full_test_time_s": 159.4 + }, + "fast_hunyuan_video_musa": { + "stages_ms": { + "InputValidationStage": 0.08, + "TextEncodingStage": 4146.14, + "TimestepPreparationStage": 18.33, + "LatentPreparationStage": 2.42, + "DenoisingStage": 17236.59, + "DecodingStage": 7019.29 + }, + "denoise_step_ms": { + "0": 5462.17, + "1": 2338.96, + "2": 2355.67, + "3": 2353.45, + "4": 2359.64, + "5": 2363.04 + }, + "expected_e2e_ms": 32799.37, + "expected_avg_denoise_ms": 2872.16, + "expected_median_denoise_ms": 2357.66, + "estimated_full_test_time_s": 111.3 + }, + "qwen_image_t2i_musa": { + "stages_ms": { + "InputValidationStage": 0.09, + "TextEncodingStage": 658.4, + "LatentPreparationStage": 0.33, + "TimestepPreparationStage": 24.39, + "DenoisingStage": 36196.6, + "DecodingStage": 40.44 + }, + "denoise_step_ms": { + "0": 666.68, + "1": 732.33, + "2": 721.29, + "3": 729.27, + "4": 725.05, + "5": 721.71, + "6": 722.22, + "7": 725.93, + "8": 724.94, + "9": 724.14, + "10": 730.43, + "11": 719.92, + "12": 726.24, + "13": 722.04, + "14": 727.68, + "15": 720.31, + "16": 721.75, + "17": 725.65, + "18": 720.23, + "19": 724.12, + "20": 726.35, + "21": 723.27, + "22": 731.58, + "23": 724.97, + "24": 721.48, + "25": 722.0, + "26": 722.37, + "27": 719.81, + "28": 721.64, + "29": 724.81, + "30": 723.9, + "31": 725.42, + "32": 719.86, + "33": 728.04, + "34": 728.55, + "35": 723.13, + "36": 722.0, + "37": 730.11, + "38": 724.06, + "39": 728.35, + "40": 728.04, + "41": 726.62, + "42": 728.47, + "43": 728.11, + "44": 728.59, + "45": 721.5, + "46": 724.59, + "47": 729.26, + "48": 726.05, + "49": 721.13 + }, + "expected_e2e_ms": 37190.98, + "expected_avg_denoise_ms": 723.72, + "expected_median_denoise_ms": 724.7, + "estimated_full_test_time_s": 137.0 + }, + "qwen_image_2512_t2i_musa": { + "stages_ms": { + "InputValidationStage": 0.07, + "TextEncodingStage": 849.1, + "LatentPreparationStage": 0.26, + "TimestepPreparationStage": 21.95, + "DenoisingStage": 31935.97, + "DecodingStage": 98.62 + }, + "denoise_step_ms": { + "0": 553.87, + "1": 639.78, + "2": 640.39, + "3": 638.78, + "4": 640.13, + "5": 639.69, + "6": 640.19, + "7": 639.67, + "8": 638.95, + "9": 640.06, + "10": 639.8, + "11": 640.81, + "12": 638.7, + "13": 640.16, + "14": 639.76, + "15": 640.17, + "16": 639.26, + "17": 640.33, + "18": 639.91, + "19": 640.47, + "20": 640.1, + "21": 639.81, + "22": 639.05, + "23": 639.7, + "24": 640.02, + "25": 640.67, + "26": 639.04, + "27": 640.21, + "28": 639.78, + "29": 643.32, + "30": 640.2, + "31": 639.54, + "32": 640.14, + "33": 640.14, + "34": 639.54, + "35": 639.22, + "36": 639.94, + "37": 639.87, + "38": 639.41, + "39": 639.6, + "40": 638.5, + "41": 639.5, + "42": 639.35, + "43": 641.43, + "44": 638.73, + "45": 639.74, + "46": 639.72, + "47": 639.72, + "48": 638.64, + "49": 638.72 + }, + "expected_e2e_ms": 32915.75, + "expected_avg_denoise_ms": 638.55, + "expected_median_denoise_ms": 640.05, + "estimated_full_test_time_s": 137.8 + }, + "wan2_1_t2v_1.3b_musa": { + "stages_ms": { + "InputValidationStage": 0.06, + "TextEncodingStage": 1381.91, + "LatentPreparationStage": 0.18, + "TimestepPreparationStage": 3.99, + "DenoisingStage": 23679.86, + "DecodingStage": 1059.53, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 331.71, + "1": 478.93, + "2": 481.37, + "3": 483.42, + "4": 476.27, + "5": 488.73, + "6": 486.79, + "7": 467.89, + "8": 465.47, + "9": 472.81, + "10": 478.41, + "11": 488.84, + "12": 474.68, + "13": 468.48, + "14": 483.79, + "15": 476.05, + "16": 483.77, + "17": 476.04, + "18": 484.27, + "19": 486.43, + "20": 483.04, + "21": 473.76, + "22": 464.09, + "23": 474.14, + "24": 470.76, + "25": 487.99, + "26": 477.77, + "27": 465.76, + "28": 483.96, + "29": 484.14, + "30": 471.99, + "31": 483.41, + "32": 486.66, + "33": 467.07, + "34": 478.04, + "35": 476.9, + "36": 462.4, + "37": 476.53, + "38": 485.65, + "39": 478.89, + "40": 465.22, + "41": 472.03, + "42": 479.65, + "43": 479.74, + "44": 479.48, + "45": 474.19, + "46": 464.47, + "47": 463.09, + "48": 463.77, + "49": 463.49 + }, + "expected_e2e_ms": 26134.51, + "expected_avg_denoise_ms": 473.44, + "expected_median_denoise_ms": 476.72, + "estimated_full_test_time_s": 101.5 }, "wan2_2_t2v_a14b_2gpu_musa": { "stages_ms": { @@ -259,6 +435,130 @@ "expected_e2e_ms": 138624.98, "expected_avg_denoise_ms": 2686.91, "expected_median_denoise_ms": 2691.24 + }, + "qwen_image_edit_t2i_musa": { + "stages_ms": { + "InputValidationStage": 35.9, + "ImageEncodingStage": 7483.38, + "ImageVAEEncodingStage": 160.82, + "LatentPreparationStage": 3.0, + "TimestepPreparationStage": 19.02, + "DenoisingStage": 84367.86, + "DecodingStage": 36.82 + }, + "denoise_step_ms": { + "0": 7612.62, + "1": 1567.7, + "2": 1559.81, + "3": 1570.75, + "4": 1568.7, + "5": 1572.44, + "6": 1565.71, + "7": 1559.33, + "8": 1567.63, + "9": 1566.83, + "10": 1560.77, + "11": 1561.03, + "12": 1565.61, + "13": 1560.43, + "14": 1562.47, + "15": 1562.22, + "16": 1564.1, + "17": 1571.28, + "18": 1563.26, + "19": 1561.37, + "20": 1559.04, + "21": 1556.07, + "22": 1577.83, + "23": 1564.54, + "24": 1564.28, + "25": 1573.79, + "26": 1572.94, + "27": 1568.81, + "28": 1568.73, + "29": 1571.71, + "30": 1557.83, + "31": 1568.7, + "32": 1570.3, + "33": 1567.36, + "34": 1566.47, + "35": 1567.2, + "36": 1560.98, + "37": 1563.43, + "38": 1570.74, + "39": 1568.01, + "40": 1560.57, + "41": 1572.64, + "42": 1564.01, + "43": 1566.34, + "44": 1569.09, + "45": 1573.18, + "46": 1566.5, + "47": 1567.04, + "48": 1570.02, + "49": 1559.91 + }, + "expected_e2e_ms": 92355.9, + "expected_avg_denoise_ms": 1687.04, + "expected_median_denoise_ms": 1566.67, + "estimated_full_test_time_s": 217.0 + }, + "qwen_image_edit_2509_ti2i_musa": { + "stages_ms": { + "InputValidationStage": 125.7, + "ImageEncodingStage": 1018.93, + "ImageVAEEncodingStage": 311.41, + "LatentPreparationStage": 0.24, + "TimestepPreparationStage": 33.39, + "DenoisingStage": 88792.03, + "DecodingStage": 320.64 + }, + "denoise_step_ms": { + "0": 1914.75, + "1": 2230.29, + "2": 2216.93, + "3": 2231.22, + "4": 2230.63, + "5": 2222.13, + "6": 2224.43, + "7": 2235.47, + "8": 2220.55, + "9": 2239.83, + "10": 2239.29, + "11": 2216.95, + "12": 2221.39, + "13": 2229.65, + "14": 2231.94, + "15": 2222.23, + "16": 2230.03, + "17": 2236.55, + "18": 2217.18, + "19": 2231.48, + "20": 2236.88, + "21": 2226.74, + "22": 2224.26, + "23": 2231.1, + "24": 2214.29, + "25": 2224.57, + "26": 2233.64, + "27": 2217.0, + "28": 2226.08, + "29": 2229.47, + "30": 2230.21, + "31": 2224.45, + "32": 2230.89, + "33": 2232.82, + "34": 2219.97, + "35": 2228.74, + "36": 2231.6, + "37": 2225.25, + "38": 2223.72, + "39": 2228.41 + }, + "expected_e2e_ms": 90612.84, + "expected_avg_denoise_ms": 2219.58, + "expected_median_denoise_ms": 2228.58, + "estimated_full_test_time_s": 220.8 } } } diff --git a/python/sglang/multimodal_gen/test/server/musa/test_server_a_musa.py b/python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa.py similarity index 69% rename from python/sglang/multimodal_gen/test/server/musa/test_server_a_musa.py rename to python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa.py index 7b4c4bc29..753fb8899 100644 --- a/python/sglang/multimodal_gen/test/server/musa/test_server_a_musa.py +++ b/python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa.py @@ -1,5 +1,5 @@ """ -MUSA-specific diffusion performance test (1-GPU). +MUSA-specific 1-GPU diffusion performance tests. """ from __future__ import annotations @@ -8,7 +8,7 @@ import pytest from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import ( - ONE_GPU_MUSA_CASES_A, + ONE_GPU_MUSA_CASES, ) from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 DiffusionServerBase, @@ -19,10 +19,10 @@ from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase logger = init_logger(__name__) -class TestDiffusionServerOneGpuMusaImage(DiffusionServerBase): - """Performance tests for 1-GPU diffusion cases on MUSA""" +class TestDiffusionServerOneGpuMusa(DiffusionServerBase): + """Performance tests for 1-GPU diffusion cases on MUSA.""" - @pytest.fixture(params=ONE_GPU_MUSA_CASES_A, ids=lambda c: c.id) + @pytest.fixture(params=ONE_GPU_MUSA_CASES, ids=lambda c: c.id) def case(self, request) -> DiffusionTestCase: """Provide a DiffusionTestCase for each 1-GPU MUSA test.""" return request.param diff --git a/python/sglang/multimodal_gen/test/server/musa/test_server_b_musa.py b/python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa_nightly.py similarity index 57% rename from python/sglang/multimodal_gen/test/server/musa/test_server_b_musa.py rename to python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa_nightly.py index f961648fa..7b0384fee 100644 --- a/python/sglang/multimodal_gen/test/server/musa/test_server_b_musa.py +++ b/python/sglang/multimodal_gen/test/server/musa/test_server_1_gpu_musa_nightly.py @@ -1,5 +1,5 @@ """ -MUSA-specific diffusion performance test (1-GPU). +MUSA-specific 1-GPU diffusion performance tests for nightly suite. """ from __future__ import annotations @@ -8,7 +8,7 @@ import pytest from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import ( - ONE_GPU_MUSA_CASES_B, + ONE_GPU_NIGHTLY_MUSA_CASES, ) from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 DiffusionServerBase, @@ -19,10 +19,10 @@ from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase logger = init_logger(__name__) -class TestDiffusionServerOneGpuMusaVideo(DiffusionServerBase): - """Performance tests for 1-GPU diffusion cases on MUSA""" +class TestDiffusionServerOneGpuMusaNightly(DiffusionServerBase): + """Performance tests for 1-GPU diffusion cases on MUSA (nightly-only).""" - @pytest.fixture(params=ONE_GPU_MUSA_CASES_B, ids=lambda c: c.id) + @pytest.fixture(params=ONE_GPU_NIGHTLY_MUSA_CASES, ids=lambda c: c.id) def case(self, request) -> DiffusionTestCase: - """Provide a DiffusionTestCase for each 1-GPU MUSA test.""" + """Provide a DiffusionTestCase for each 1-GPU MUSA nightly test.""" return request.param diff --git a/python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_a_musa.py b/python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_musa.py similarity index 76% rename from python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_a_musa.py rename to python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_musa.py index 48909e1a2..bacf613dc 100644 --- a/python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_a_musa.py +++ b/python/sglang/multimodal_gen/test/server/musa/test_server_2_gpu_musa.py @@ -1,5 +1,5 @@ """ -MUSA-specific 2-GPU diffusion performance test. +MUSA-specific 2-GPU diffusion performance tests. """ from __future__ import annotations @@ -8,7 +8,7 @@ import pytest from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import ( - TWO_GPU_MUSA_CASES_A, + TWO_GPU_MUSA_CASES, ) from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401 DiffusionServerBase, @@ -19,10 +19,10 @@ from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase logger = init_logger(__name__) -class TestDiffusionServerTwoGpuMusaA(DiffusionServerBase): +class TestDiffusionServerTwoGpuMusa(DiffusionServerBase): """Performance tests for 2-GPU diffusion cases on MUSA.""" - @pytest.fixture(params=TWO_GPU_MUSA_CASES_A, ids=lambda c: c.id) + @pytest.fixture(params=TWO_GPU_MUSA_CASES, ids=lambda c: c.id) def case(self, request) -> DiffusionTestCase: """Provide a DiffusionTestCase for each 2-GPU MUSA test.""" return request.param diff --git a/python/sglang/multimodal_gen/test/server/musa/testcase_configs_musa.py b/python/sglang/multimodal_gen/test/server/musa/testcase_configs_musa.py index ca8363349..c3019dedb 100644 --- a/python/sglang/multimodal_gen/test/server/musa/testcase_configs_musa.py +++ b/python/sglang/multimodal_gen/test/server/musa/testcase_configs_musa.py @@ -1,32 +1,49 @@ from __future__ import annotations +from dataclasses import replace +from functools import lru_cache + from sglang.multimodal_gen.test.server.testcase_configs import ( T2V_PROMPT, DiffusionSamplingParams, DiffusionServerArgs, DiffusionTestCase, + MULTI_FRAME_I2I_sampling_params, + MULTI_IMAGE_TI2I_sampling_params, T2I_sampling_params, + T2V_sampling_params, + TI2I_sampling_params, TI2V_sampling_params, ) -ONE_GPU_MUSA_CASES_A: list[DiffusionTestCase] = [ + +@lru_cache(maxsize=None) +def hf_cached_model(repo_id: str) -> str: + """Resolve an HF repo id to the local cache snapshot prepared on MUSA runners.""" + from huggingface_hub import snapshot_download + + return snapshot_download(repo_id, local_files_only=True) + + +MUSA_TI2I_sampling_params = replace( + TI2I_sampling_params, + image_path="/hf-cache/hub/musa-test-assets/TI2I_Qwen_Image_Edit_Input.jpg", +) + +ONE_GPU_MUSA_CASES: list[DiffusionTestCase] = [ DiffusionTestCase( "qwen_image_t2i_musa", DiffusionServerArgs( - model_path="Qwen/Qwen-Image", + model_path=hf_cached_model("Qwen/Qwen-Image"), modality="image", ), T2I_sampling_params, run_consistency_check=False, ), -] - - -ONE_GPU_MUSA_CASES_B: list[DiffusionTestCase] = [ DiffusionTestCase( "wan2_1_t2v_1.3b_musa", DiffusionServerArgs( - model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + model_path=hf_cached_model("Wan-AI/Wan2.1-T2V-1.3B-Diffusers"), modality="video", custom_validator="video", ), @@ -38,11 +55,75 @@ ONE_GPU_MUSA_CASES_B: list[DiffusionTestCase] = [ ] -TWO_GPU_MUSA_CASES_A: list[DiffusionTestCase] = [ +NIGHTLY_1_GPU_MUSA_CASES: list[DiffusionTestCase] = [ + DiffusionTestCase( + "zimage_image_t2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Tongyi-MAI/Z-Image-Turbo"), + modality="image", + ), + T2I_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "qwen_image_layered_i2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Qwen/Qwen-Image-Layered"), + modality="image", + ), + MULTI_FRAME_I2I_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "fast_hunyuan_video_musa", + DiffusionServerArgs( + model_path=hf_cached_model("FastVideo/FastHunyuan-diffusers"), + modality="video", + custom_validator="video", + ), + T2V_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "qwen_image_2512_t2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Qwen/Qwen-Image-2512"), + modality="image", + ), + T2I_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "qwen_image_edit_t2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Qwen/Qwen-Image-Edit"), + modality="image", + ), + MUSA_TI2I_sampling_params, + run_consistency_check=False, + ), + DiffusionTestCase( + "qwen_image_edit_2509_ti2i_musa", + DiffusionServerArgs( + model_path=hf_cached_model("Qwen/Qwen-Image-Edit-2509"), + modality="image", + ), + MULTI_IMAGE_TI2I_sampling_params, + run_consistency_check=False, + ), +] + + +ONE_GPU_NIGHTLY_MUSA_CASES: list[DiffusionTestCase] = ( + ONE_GPU_MUSA_CASES + NIGHTLY_1_GPU_MUSA_CASES +) + + +TWO_GPU_MUSA_CASES: list[DiffusionTestCase] = [ DiffusionTestCase( "wan2_1_i2v_14b_480P_2gpu_musa", DiffusionServerArgs( - model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + model_path=hf_cached_model("Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"), modality="video", custom_validator="video", num_gpus=2, diff --git a/scripts/ci/musa/rename_wheels_musa.sh b/scripts/ci/musa/rename_wheels_musa.sh index 23ea57f2b..f3816548a 100755 --- a/scripts/ci/musa/rename_wheels_musa.sh +++ b/scripts/ci/musa/rename_wheels_musa.sh @@ -1,46 +1,102 @@ #!/usr/bin/env bash -set -euo pipefail - -# Rename MUSA wheels to include a +musa build tag. +# Align MUSA wheel filenames (+musa43/...) with internal METADATA Version and +# WHEEL tags after build. Two drifts need fixing in lockstep: +# - METADATA `Version:` must carry the `+musa` local version, or +# recent pip versions reject the wheel with "inconsistent version". +# - WHEEL `Tag:` must be `manylinux2014_*` when the filename says so; +# leaving it as `linux_*` can trip installers that re-derive the platform. +# Unpack → patch WHEEL/METADATA → wheel pack (RECORD regenerated; no hand-editing). +# # Usage: # rename_wheels_musa.sh [wheel_dir] # Example: # rename_wheels_musa.sh 43 sgl-kernel/dist +set -euxo pipefail if [[ $# -lt 1 || $# -gt 2 ]]; then echo "Usage: $0 [wheel_dir]" >&2 exit 1 fi -MUSA_SUFFIX="$1" +MUSA_SUFFIX="+musa$1" WHEEL_DIR="${2:-dist}" -wheel_files=("$WHEEL_DIR"/*.whl) +patch_wheel_platform_tags() { + local wheel_file="$1" + # Line-end anchors: "linux_x86_64" is a substring of "manylinux2014_x86_64", so + # unanchored global replace corrupts tags on a second run. + sed -i \ + -e 's/-linux_x86_64$/-manylinux2014_x86_64/' \ + -e 's/-linux_aarch64$/-manylinux2014_aarch64/' \ + "$wheel_file" +} +wheel_files=("$WHEEL_DIR"/*.whl) if [[ ! -e "${wheel_files[0]}" ]]; then echo "No wheel files found in ${WHEEL_DIR}/, nothing to rename." exit 0 fi for wheel in "${wheel_files[@]}"; do - # Normalize platform tag to manylinux2014 - intermediate_wheel="${wheel/linux/manylinux2014}" + [[ -f "$wheel" ]] || continue - # Extract Python ABI version (e.g. cp310) - if [[ $intermediate_wheel =~ -cp([0-9]+)- ]]; then - cp_version="${BASH_REMATCH[1]}" - else - echo "Could not extract Python version from wheel name: $intermediate_wheel" >&2 - continue - fi + intermediate_wheel="$wheel" + case "$wheel" in + *-linux_x86_64.whl) + intermediate_wheel="${wheel%-linux_x86_64.whl}-manylinux2014_x86_64.whl" + ;; + *-linux_aarch64.whl) + intermediate_wheel="${wheel%-linux_aarch64.whl}-manylinux2014_aarch64.whl" + ;; + esac + if [[ "$wheel" != "$intermediate_wheel" ]]; then + mv -- "$wheel" "$intermediate_wheel" + wheel="$intermediate_wheel" + fi - # Insert +musa before the Python ABI tag - new_wheel="${intermediate_wheel/-cp${cp_version}/+musa${MUSA_SUFFIX}-cp${cp_version}}" + TMPDIR=$(mktemp -d) + trap 'rm -rf -- "$TMPDIR"' ERR - if [[ "$wheel" != "$new_wheel" ]]; then - echo "Renaming $wheel -> $new_wheel" - mv -- "$wheel" "$new_wheel" - fi + "${PYTHON:-python3}" -m wheel unpack "$wheel" --dest "$TMPDIR" + # `find | head -1` succeeds with empty stdout when there are no matches — + # `set -e` won't catch that. Assert each path is real so a malformed wheel + # surfaces with a useful message instead of a downstream `sed: /WHEEL` error. + UNPACKED=$(find "$TMPDIR" -mindepth 1 -maxdepth 1 -type d | head -1) + [[ -d "$UNPACKED" ]] || { echo "ERROR: wheel unpack produced no top-level dir for $wheel" >&2; exit 1; } + DIST_INFO=$(find "$UNPACKED" -maxdepth 1 -type d -name "*.dist-info" | head -1) + [[ -d "$DIST_INFO" ]] || { echo "ERROR: no *.dist-info under $UNPACKED (malformed wheel?): $wheel" >&2; exit 1; } + WHEEL_META="${DIST_INFO}/WHEEL" + METADATA_FILE="${DIST_INFO}/METADATA" + [[ -f "$WHEEL_META" && -f "$METADATA_FILE" ]] || { echo "ERROR: missing WHEEL or METADATA in $DIST_INFO" >&2; exit 1; } + + patch_wheel_platform_tags "$WHEEL_META" + + ORIG_VERSION=$(grep '^Version:' "$METADATA_FILE" | head -1 | sed 's/^Version:[[:space:]]*//') + # Empty ORIG_VERSION would fall through the `+musa` check below and silently + # produce `Version: +musa43` — a broken release. Fail loud instead. + [[ -n "$ORIG_VERSION" ]] || { echo "ERROR: no 'Version:' line in $METADATA_FILE" >&2; exit 1; } + if [[ "$ORIG_VERSION" == *"$MUSA_SUFFIX"* ]]; then + echo "Skipping $wheel: version in METADATA is already suffixed." + rm -rf "$TMPDIR" + trap - ERR + continue + fi + NEW_VERSION="${ORIG_VERSION}${MUSA_SUFFIX}" + sed -i "s/^Version:.*/Version: ${NEW_VERSION}/" "$METADATA_FILE" + # `sed -i` exits 0 even when the pattern matched zero lines. Verify the + # rewrite actually landed before we publish. + grep -qx "Version: ${NEW_VERSION}" "$METADATA_FILE" || { echo "ERROR: METADATA Version rewrite did not land in $METADATA_FILE" >&2; exit 1; } + + OLD_BASE=$(basename "$DIST_INFO") + NEW_BASE="${OLD_BASE/${ORIG_VERSION}/${NEW_VERSION}}" + # `${var/pat/repl}` silently leaves var unchanged if pat is empty or absent. + [[ "$NEW_BASE" != "$OLD_BASE" ]] || { echo "ERROR: dist-info dir '$OLD_BASE' did not contain ORIG_VERSION='$ORIG_VERSION'" >&2; exit 1; } + mv "$DIST_INFO" "${UNPACKED}/${NEW_BASE}" + + rm -f "$wheel" + "${PYTHON:-python3}" -m wheel pack "$UNPACKED" --dest-dir "$WHEEL_DIR" + rm -rf "$TMPDIR" + trap - ERR done echo "MUSA wheel renaming completed."