[MUSA][22/N] ci(musa): repack wheels with +musa metadata, refine path filters, sync multimodal tests, and add nightly workflow (#25074)

Co-authored-by: xinyue.fan <xinyue.fan@mthreads.com>
This commit is contained in:
johnnycxm
2026-05-22 16:24:25 +08:00
committed by GitHub
co-authored by xinyue.fan
parent 16d049f898
commit cf5f496183
9 changed files with 890 additions and 186 deletions
@@ -22,11 +22,13 @@ logger = init_logger(__name__)
SUITES = {
"1-gpu-musa": [
"musa/test_server_a_musa.py",
"musa/test_server_b_musa.py",
"musa/test_server_1_gpu_musa.py",
],
"1-gpu-musa-nightly": [
"musa/test_server_1_gpu_musa_nightly.py",
],
"2-gpu-musa": [
"musa/test_server_2_gpu_a_musa.py",
"musa/test_server_2_gpu_musa.py",
],
}
@@ -1,140 +1,316 @@
{
"metadata":{
"model":"Diffusion Server",
"hardware":"CI S5000 pool",
"description":"Reference numbers captured from the CI diffusion server baseline run"
"metadata": {
"model": "Diffusion Server",
"hardware": "CI S5000 pool",
"description": "Reference numbers captured from the CI diffusion server baseline run"
},
"scenarios":{
"qwen_image_t2i_musa":{
"stages_ms":{
"InputValidationStage":0.09,
"TextEncodingStage":658.4,
"LatentPreparationStage":0.33,
"TimestepPreparationStage":24.39,
"DenoisingStage":36196.6,
"DecodingStage":40.44
"scenarios": {
"zimage_image_t2i_musa": {
"stages_ms": {
"InputValidationStage": 0.08,
"TextEncodingStage": 3761.4,
"LatentPreparationStage": 4.06,
"TimestepPreparationStage": 21.94,
"DenoisingStage": 4873.8,
"DecodingStage": 252.49
},
"denoise_step_ms":{
"0":666.68,
"1":732.33,
"2":721.29,
"3":729.27,
"4":725.05,
"5":721.71,
"6":722.22,
"7":725.93,
"8":724.94,
"9":724.14,
"10":730.43,
"11":719.92,
"12":726.24,
"13":722.04,
"14":727.68,
"15":720.31,
"16":721.75,
"17":725.65,
"18":720.23,
"19":724.12,
"20":726.35,
"21":723.27,
"22":731.58,
"23":724.97,
"24":721.48,
"25":722.0,
"26":722.37,
"27":719.81,
"28":721.64,
"29":724.81,
"30":723.9,
"31":725.42,
"32":719.86,
"33":728.04,
"34":728.55,
"35":723.13,
"36":722.0,
"37":730.11,
"38":724.06,
"39":728.35,
"40":728.04,
"41":726.62,
"42":728.47,
"43":728.11,
"44":728.59,
"45":721.5,
"46":724.59,
"47":729.26,
"48":726.05,
"49":721.13
"denoise_step_ms": {
"0": 3281.35,
"2": 209.69,
"3": 199.26,
"5": 200.42,
"6": 207.8,
"8": 205.19
},
"expected_e2e_ms":37190.98,
"expected_avg_denoise_ms":723.72,
"expected_median_denoise_ms":724.7
"expected_e2e_ms": 12161.85,
"expected_avg_denoise_ms": 540.67,
"expected_median_denoise_ms": 205.19,
"estimated_full_test_time_s": 64.6
},
"wan2_1_t2v_1.3b_musa":{
"stages_ms":{
"InputValidationStage":0.12,
"TextEncodingStage":1097.75,
"LatentPreparationStage":0.24,
"TimestepPreparationStage":5.66,
"DenoisingStage":47399.84,
"DecodingStage":946.08,
"per_frame_generation":null
"qwen_image_layered_i2i_musa": {
"stages_ms": {
"QwenImageLayeredBeforeDenoisingStage": 287.12,
"TimestepPreparationStage": 0.01,
"DenoisingStage": 79962.68,
"DecodingStage": 232.86
},
"denoise_step_ms":{
"0":783.06,
"1":970.52,
"2":939.72,
"3":947.58,
"4":941.44,
"5":955.26,
"6":960.39,
"7":951.84,
"8":959.68,
"9":953.33,
"10":940.87,
"11":958.5,
"12":952.7,
"13":933.4,
"14":952.0,
"15":951.6,
"16":947.04,
"17":939.28,
"18":956.88,
"19":960.1,
"20":949.73,
"21":954.77,
"22":959.98,
"23":947.37,
"24":957.51,
"25":953.39,
"26":953.73,
"27":959.57,
"28":942.59,
"29":958.05,
"30":952.76,
"31":952.76,
"32":950.6,
"33":948.76,
"34":957.53,
"35":940.86,
"36":958.11,
"37":940.9,
"38":949.1,
"39":951.81,
"40":948.61,
"41":957.28,
"42":951.41,
"43":953.09,
"44":955.69,
"45":941.93,
"46":952.96,
"47":953.5,
"48":939.25,
"49":942.69
"denoise_step_ms": {
"0": 1377.49,
"1": 1608.67,
"2": 1606.57,
"3": 1601.06,
"4": 1606.65,
"5": 1605.0,
"6": 1596.31,
"7": 1609.13,
"8": 1599.48,
"9": 1601.81,
"10": 1606.7,
"11": 1601.7,
"12": 1607.68,
"13": 1606.42,
"14": 1599.13,
"15": 1611.78,
"16": 1598.31,
"17": 1600.97,
"18": 1611.38,
"19": 1598.6,
"20": 1599.72,
"21": 1608.24,
"22": 1599.4,
"23": 1613.52,
"24": 1600.56,
"25": 1605.91,
"26": 1605.47,
"27": 1598.85,
"28": 1607.93,
"29": 1603.65,
"30": 1598.64,
"31": 1607.1,
"32": 1595.55,
"33": 1608.09,
"34": 1606.47,
"35": 1596.18,
"36": 1599.88,
"37": 1607.12,
"38": 1595.63,
"39": 1612.07,
"40": 1596.19,
"41": 1602.35,
"42": 1604.17,
"43": 1598.26,
"44": 1602.67,
"45": 1611.54,
"46": 1599.01,
"47": 1619.72,
"48": 1593.45,
"49": 1602.33
},
"expected_e2e_ms":50007.17,
"expected_avg_denoise_ms":947.83,
"expected_median_denoise_ms":952.35
"expected_e2e_ms": 80490.53,
"expected_avg_denoise_ms": 1599.09,
"expected_median_denoise_ms": 1602.51,
"estimated_full_test_time_s": 159.4
},
"fast_hunyuan_video_musa": {
"stages_ms": {
"InputValidationStage": 0.08,
"TextEncodingStage": 4146.14,
"TimestepPreparationStage": 18.33,
"LatentPreparationStage": 2.42,
"DenoisingStage": 17236.59,
"DecodingStage": 7019.29
},
"denoise_step_ms": {
"0": 5462.17,
"1": 2338.96,
"2": 2355.67,
"3": 2353.45,
"4": 2359.64,
"5": 2363.04
},
"expected_e2e_ms": 32799.37,
"expected_avg_denoise_ms": 2872.16,
"expected_median_denoise_ms": 2357.66,
"estimated_full_test_time_s": 111.3
},
"qwen_image_t2i_musa": {
"stages_ms": {
"InputValidationStage": 0.09,
"TextEncodingStage": 658.4,
"LatentPreparationStage": 0.33,
"TimestepPreparationStage": 24.39,
"DenoisingStage": 36196.6,
"DecodingStage": 40.44
},
"denoise_step_ms": {
"0": 666.68,
"1": 732.33,
"2": 721.29,
"3": 729.27,
"4": 725.05,
"5": 721.71,
"6": 722.22,
"7": 725.93,
"8": 724.94,
"9": 724.14,
"10": 730.43,
"11": 719.92,
"12": 726.24,
"13": 722.04,
"14": 727.68,
"15": 720.31,
"16": 721.75,
"17": 725.65,
"18": 720.23,
"19": 724.12,
"20": 726.35,
"21": 723.27,
"22": 731.58,
"23": 724.97,
"24": 721.48,
"25": 722.0,
"26": 722.37,
"27": 719.81,
"28": 721.64,
"29": 724.81,
"30": 723.9,
"31": 725.42,
"32": 719.86,
"33": 728.04,
"34": 728.55,
"35": 723.13,
"36": 722.0,
"37": 730.11,
"38": 724.06,
"39": 728.35,
"40": 728.04,
"41": 726.62,
"42": 728.47,
"43": 728.11,
"44": 728.59,
"45": 721.5,
"46": 724.59,
"47": 729.26,
"48": 726.05,
"49": 721.13
},
"expected_e2e_ms": 37190.98,
"expected_avg_denoise_ms": 723.72,
"expected_median_denoise_ms": 724.7,
"estimated_full_test_time_s": 137.0
},
"qwen_image_2512_t2i_musa": {
"stages_ms": {
"InputValidationStage": 0.07,
"TextEncodingStage": 849.1,
"LatentPreparationStage": 0.26,
"TimestepPreparationStage": 21.95,
"DenoisingStage": 31935.97,
"DecodingStage": 98.62
},
"denoise_step_ms": {
"0": 553.87,
"1": 639.78,
"2": 640.39,
"3": 638.78,
"4": 640.13,
"5": 639.69,
"6": 640.19,
"7": 639.67,
"8": 638.95,
"9": 640.06,
"10": 639.8,
"11": 640.81,
"12": 638.7,
"13": 640.16,
"14": 639.76,
"15": 640.17,
"16": 639.26,
"17": 640.33,
"18": 639.91,
"19": 640.47,
"20": 640.1,
"21": 639.81,
"22": 639.05,
"23": 639.7,
"24": 640.02,
"25": 640.67,
"26": 639.04,
"27": 640.21,
"28": 639.78,
"29": 643.32,
"30": 640.2,
"31": 639.54,
"32": 640.14,
"33": 640.14,
"34": 639.54,
"35": 639.22,
"36": 639.94,
"37": 639.87,
"38": 639.41,
"39": 639.6,
"40": 638.5,
"41": 639.5,
"42": 639.35,
"43": 641.43,
"44": 638.73,
"45": 639.74,
"46": 639.72,
"47": 639.72,
"48": 638.64,
"49": 638.72
},
"expected_e2e_ms": 32915.75,
"expected_avg_denoise_ms": 638.55,
"expected_median_denoise_ms": 640.05,
"estimated_full_test_time_s": 137.8
},
"wan2_1_t2v_1.3b_musa": {
"stages_ms": {
"InputValidationStage": 0.06,
"TextEncodingStage": 1381.91,
"LatentPreparationStage": 0.18,
"TimestepPreparationStage": 3.99,
"DenoisingStage": 23679.86,
"DecodingStage": 1059.53,
"per_frame_generation": null
},
"denoise_step_ms": {
"0": 331.71,
"1": 478.93,
"2": 481.37,
"3": 483.42,
"4": 476.27,
"5": 488.73,
"6": 486.79,
"7": 467.89,
"8": 465.47,
"9": 472.81,
"10": 478.41,
"11": 488.84,
"12": 474.68,
"13": 468.48,
"14": 483.79,
"15": 476.05,
"16": 483.77,
"17": 476.04,
"18": 484.27,
"19": 486.43,
"20": 483.04,
"21": 473.76,
"22": 464.09,
"23": 474.14,
"24": 470.76,
"25": 487.99,
"26": 477.77,
"27": 465.76,
"28": 483.96,
"29": 484.14,
"30": 471.99,
"31": 483.41,
"32": 486.66,
"33": 467.07,
"34": 478.04,
"35": 476.9,
"36": 462.4,
"37": 476.53,
"38": 485.65,
"39": 478.89,
"40": 465.22,
"41": 472.03,
"42": 479.65,
"43": 479.74,
"44": 479.48,
"45": 474.19,
"46": 464.47,
"47": 463.09,
"48": 463.77,
"49": 463.49
},
"expected_e2e_ms": 26134.51,
"expected_avg_denoise_ms": 473.44,
"expected_median_denoise_ms": 476.72,
"estimated_full_test_time_s": 101.5
},
"wan2_2_t2v_a14b_2gpu_musa": {
"stages_ms": {
@@ -259,6 +435,130 @@
"expected_e2e_ms": 138624.98,
"expected_avg_denoise_ms": 2686.91,
"expected_median_denoise_ms": 2691.24
},
"qwen_image_edit_t2i_musa": {
"stages_ms": {
"InputValidationStage": 35.9,
"ImageEncodingStage": 7483.38,
"ImageVAEEncodingStage": 160.82,
"LatentPreparationStage": 3.0,
"TimestepPreparationStage": 19.02,
"DenoisingStage": 84367.86,
"DecodingStage": 36.82
},
"denoise_step_ms": {
"0": 7612.62,
"1": 1567.7,
"2": 1559.81,
"3": 1570.75,
"4": 1568.7,
"5": 1572.44,
"6": 1565.71,
"7": 1559.33,
"8": 1567.63,
"9": 1566.83,
"10": 1560.77,
"11": 1561.03,
"12": 1565.61,
"13": 1560.43,
"14": 1562.47,
"15": 1562.22,
"16": 1564.1,
"17": 1571.28,
"18": 1563.26,
"19": 1561.37,
"20": 1559.04,
"21": 1556.07,
"22": 1577.83,
"23": 1564.54,
"24": 1564.28,
"25": 1573.79,
"26": 1572.94,
"27": 1568.81,
"28": 1568.73,
"29": 1571.71,
"30": 1557.83,
"31": 1568.7,
"32": 1570.3,
"33": 1567.36,
"34": 1566.47,
"35": 1567.2,
"36": 1560.98,
"37": 1563.43,
"38": 1570.74,
"39": 1568.01,
"40": 1560.57,
"41": 1572.64,
"42": 1564.01,
"43": 1566.34,
"44": 1569.09,
"45": 1573.18,
"46": 1566.5,
"47": 1567.04,
"48": 1570.02,
"49": 1559.91
},
"expected_e2e_ms": 92355.9,
"expected_avg_denoise_ms": 1687.04,
"expected_median_denoise_ms": 1566.67,
"estimated_full_test_time_s": 217.0
},
"qwen_image_edit_2509_ti2i_musa": {
"stages_ms": {
"InputValidationStage": 125.7,
"ImageEncodingStage": 1018.93,
"ImageVAEEncodingStage": 311.41,
"LatentPreparationStage": 0.24,
"TimestepPreparationStage": 33.39,
"DenoisingStage": 88792.03,
"DecodingStage": 320.64
},
"denoise_step_ms": {
"0": 1914.75,
"1": 2230.29,
"2": 2216.93,
"3": 2231.22,
"4": 2230.63,
"5": 2222.13,
"6": 2224.43,
"7": 2235.47,
"8": 2220.55,
"9": 2239.83,
"10": 2239.29,
"11": 2216.95,
"12": 2221.39,
"13": 2229.65,
"14": 2231.94,
"15": 2222.23,
"16": 2230.03,
"17": 2236.55,
"18": 2217.18,
"19": 2231.48,
"20": 2236.88,
"21": 2226.74,
"22": 2224.26,
"23": 2231.1,
"24": 2214.29,
"25": 2224.57,
"26": 2233.64,
"27": 2217.0,
"28": 2226.08,
"29": 2229.47,
"30": 2230.21,
"31": 2224.45,
"32": 2230.89,
"33": 2232.82,
"34": 2219.97,
"35": 2228.74,
"36": 2231.6,
"37": 2225.25,
"38": 2223.72,
"39": 2228.41
},
"expected_e2e_ms": 90612.84,
"expected_avg_denoise_ms": 2219.58,
"expected_median_denoise_ms": 2228.58,
"estimated_full_test_time_s": 220.8
}
}
}
@@ -1,5 +1,5 @@
"""
MUSA-specific diffusion performance test (1-GPU).
MUSA-specific 1-GPU diffusion performance tests.
"""
from __future__ import annotations
@@ -8,7 +8,7 @@ import pytest
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import (
ONE_GPU_MUSA_CASES_A,
ONE_GPU_MUSA_CASES,
)
from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401
DiffusionServerBase,
@@ -19,10 +19,10 @@ from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase
logger = init_logger(__name__)
class TestDiffusionServerOneGpuMusaImage(DiffusionServerBase):
"""Performance tests for 1-GPU diffusion cases on MUSA"""
class TestDiffusionServerOneGpuMusa(DiffusionServerBase):
"""Performance tests for 1-GPU diffusion cases on MUSA."""
@pytest.fixture(params=ONE_GPU_MUSA_CASES_A, ids=lambda c: c.id)
@pytest.fixture(params=ONE_GPU_MUSA_CASES, ids=lambda c: c.id)
def case(self, request) -> DiffusionTestCase:
"""Provide a DiffusionTestCase for each 1-GPU MUSA test."""
return request.param
@@ -1,5 +1,5 @@
"""
MUSA-specific diffusion performance test (1-GPU).
MUSA-specific 1-GPU diffusion performance tests for nightly suite.
"""
from __future__ import annotations
@@ -8,7 +8,7 @@ import pytest
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import (
ONE_GPU_MUSA_CASES_B,
ONE_GPU_NIGHTLY_MUSA_CASES,
)
from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401
DiffusionServerBase,
@@ -19,10 +19,10 @@ from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase
logger = init_logger(__name__)
class TestDiffusionServerOneGpuMusaVideo(DiffusionServerBase):
"""Performance tests for 1-GPU diffusion cases on MUSA"""
class TestDiffusionServerOneGpuMusaNightly(DiffusionServerBase):
"""Performance tests for 1-GPU diffusion cases on MUSA (nightly-only)."""
@pytest.fixture(params=ONE_GPU_MUSA_CASES_B, ids=lambda c: c.id)
@pytest.fixture(params=ONE_GPU_NIGHTLY_MUSA_CASES, ids=lambda c: c.id)
def case(self, request) -> DiffusionTestCase:
"""Provide a DiffusionTestCase for each 1-GPU MUSA test."""
"""Provide a DiffusionTestCase for each 1-GPU MUSA nightly test."""
return request.param
@@ -1,5 +1,5 @@
"""
MUSA-specific 2-GPU diffusion performance test.
MUSA-specific 2-GPU diffusion performance tests.
"""
from __future__ import annotations
@@ -8,7 +8,7 @@ import pytest
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.server.musa.testcase_configs_musa import (
TWO_GPU_MUSA_CASES_A,
TWO_GPU_MUSA_CASES,
)
from sglang.multimodal_gen.test.server.test_server_common import ( # noqa: F401
DiffusionServerBase,
@@ -19,10 +19,10 @@ from sglang.multimodal_gen.test.server.testcase_configs import DiffusionTestCase
logger = init_logger(__name__)
class TestDiffusionServerTwoGpuMusaA(DiffusionServerBase):
class TestDiffusionServerTwoGpuMusa(DiffusionServerBase):
"""Performance tests for 2-GPU diffusion cases on MUSA."""
@pytest.fixture(params=TWO_GPU_MUSA_CASES_A, ids=lambda c: c.id)
@pytest.fixture(params=TWO_GPU_MUSA_CASES, ids=lambda c: c.id)
def case(self, request) -> DiffusionTestCase:
"""Provide a DiffusionTestCase for each 2-GPU MUSA test."""
return request.param
@@ -1,32 +1,49 @@
from __future__ import annotations
from dataclasses import replace
from functools import lru_cache
from sglang.multimodal_gen.test.server.testcase_configs import (
T2V_PROMPT,
DiffusionSamplingParams,
DiffusionServerArgs,
DiffusionTestCase,
MULTI_FRAME_I2I_sampling_params,
MULTI_IMAGE_TI2I_sampling_params,
T2I_sampling_params,
T2V_sampling_params,
TI2I_sampling_params,
TI2V_sampling_params,
)
ONE_GPU_MUSA_CASES_A: list[DiffusionTestCase] = [
@lru_cache(maxsize=None)
def hf_cached_model(repo_id: str) -> str:
"""Resolve an HF repo id to the local cache snapshot prepared on MUSA runners."""
from huggingface_hub import snapshot_download
return snapshot_download(repo_id, local_files_only=True)
MUSA_TI2I_sampling_params = replace(
TI2I_sampling_params,
image_path="/hf-cache/hub/musa-test-assets/TI2I_Qwen_Image_Edit_Input.jpg",
)
ONE_GPU_MUSA_CASES: list[DiffusionTestCase] = [
DiffusionTestCase(
"qwen_image_t2i_musa",
DiffusionServerArgs(
model_path="Qwen/Qwen-Image",
model_path=hf_cached_model("Qwen/Qwen-Image"),
modality="image",
),
T2I_sampling_params,
run_consistency_check=False,
),
]
ONE_GPU_MUSA_CASES_B: list[DiffusionTestCase] = [
DiffusionTestCase(
"wan2_1_t2v_1.3b_musa",
DiffusionServerArgs(
model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
model_path=hf_cached_model("Wan-AI/Wan2.1-T2V-1.3B-Diffusers"),
modality="video",
custom_validator="video",
),
@@ -38,11 +55,75 @@ ONE_GPU_MUSA_CASES_B: list[DiffusionTestCase] = [
]
TWO_GPU_MUSA_CASES_A: list[DiffusionTestCase] = [
NIGHTLY_1_GPU_MUSA_CASES: list[DiffusionTestCase] = [
DiffusionTestCase(
"zimage_image_t2i_musa",
DiffusionServerArgs(
model_path=hf_cached_model("Tongyi-MAI/Z-Image-Turbo"),
modality="image",
),
T2I_sampling_params,
run_consistency_check=False,
),
DiffusionTestCase(
"qwen_image_layered_i2i_musa",
DiffusionServerArgs(
model_path=hf_cached_model("Qwen/Qwen-Image-Layered"),
modality="image",
),
MULTI_FRAME_I2I_sampling_params,
run_consistency_check=False,
),
DiffusionTestCase(
"fast_hunyuan_video_musa",
DiffusionServerArgs(
model_path=hf_cached_model("FastVideo/FastHunyuan-diffusers"),
modality="video",
custom_validator="video",
),
T2V_sampling_params,
run_consistency_check=False,
),
DiffusionTestCase(
"qwen_image_2512_t2i_musa",
DiffusionServerArgs(
model_path=hf_cached_model("Qwen/Qwen-Image-2512"),
modality="image",
),
T2I_sampling_params,
run_consistency_check=False,
),
DiffusionTestCase(
"qwen_image_edit_t2i_musa",
DiffusionServerArgs(
model_path=hf_cached_model("Qwen/Qwen-Image-Edit"),
modality="image",
),
MUSA_TI2I_sampling_params,
run_consistency_check=False,
),
DiffusionTestCase(
"qwen_image_edit_2509_ti2i_musa",
DiffusionServerArgs(
model_path=hf_cached_model("Qwen/Qwen-Image-Edit-2509"),
modality="image",
),
MULTI_IMAGE_TI2I_sampling_params,
run_consistency_check=False,
),
]
ONE_GPU_NIGHTLY_MUSA_CASES: list[DiffusionTestCase] = (
ONE_GPU_MUSA_CASES + NIGHTLY_1_GPU_MUSA_CASES
)
TWO_GPU_MUSA_CASES: list[DiffusionTestCase] = [
DiffusionTestCase(
"wan2_1_i2v_14b_480P_2gpu_musa",
DiffusionServerArgs(
model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
model_path=hf_cached_model("Wan-AI/Wan2.1-I2V-14B-480P-Diffusers"),
modality="video",
custom_validator="video",
num_gpus=2,