minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[diffusion] CI: make auxiliary coverage explicit and simplify testcases (#20983)

Browse Source
This commit is contained in:
Mick
2026-03-21 20:18:23 +08:00
committed by GitHub
parent a0862f00c2
commit 6dfa8a40bc
4 changed files with 228 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/multimodal_gen/test/server/perf_baselines.json
+82 -82
View File
@@ -103,7 +103,7 @@
"qwen_image_t2i_2_gpus": { "qwen_image_t2i_2_gpus": {
"stages_ms": { "stages_ms": {
"InputValidationStage": 0.04, "InputValidationStage": 0.04,
"TextEncodingStage": 693.2, "TextEncodingStage": 800.0,
"TimestepPreparationStage": 2.84, "TimestepPreparationStage": 2.84,
"LatentPreparationStage": 9.13, "LatentPreparationStage": 9.13,
"DenoisingStage": 24529.77, "DenoisingStage": 24529.77,
@@ -292,7 +292,7 @@
"48": 497.49, "48": 497.49,
"49": 495.69 "49": 495.69
}, },
"expected_e2e_ms": 25832.82, "expected_e2e_ms": 329129.82,
"expected_avg_denoise_ms": 489.43, "expected_avg_denoise_ms": 489.43,
"expected_median_denoise_ms": 497.53 "expected_median_denoise_ms": 497.53
}, },
@@ -1281,7 +1281,7 @@
"TimestepPreparationStage": 58.66, "TimestepPreparationStage": 58.66,
"LatentPreparationStage": 28.55, "LatentPreparationStage": 28.55,
"DmdDenoisingStage": 499.34, "DmdDenoisingStage": 499.34,
"DecodingStage": 1924.01, "DecodingStage": 3712.76,
"per_frame_generation": null "per_frame_generation": null
}, },
"denoise_step_ms": { "denoise_step_ms": {
@@ -1843,7 +1843,7 @@
"flux_2_image_t2i_2_gpus": { "flux_2_image_t2i_2_gpus": {
"stages_ms": { "stages_ms": {
"InputValidationStage": 0.05, "InputValidationStage": 0.05,
"TextEncodingStage": 518.88, "TextEncodingStage": 600.0,
"ImageVAEEncodingStage": 0.0, "ImageVAEEncodingStage": 0.0,
"LatentPreparationStage": 0.45, "LatentPreparationStage": 0.45,
"TimestepPreparationStage": 3.41, "TimestepPreparationStage": 3.41,
@@ -1964,93 +1964,93 @@
}, },
"fsdp-inference": { "fsdp-inference": {
"stages_ms": { "stages_ms": {
"InputValidationStage": 0.04, "InputValidationStage": 0.07,
"TextEncodingStage": 411.12, "TextEncodingStage": 605.9,
"TimestepPreparationStage": 1.44, "LatentPreparationStage": 0.2,
"LatentPreparationStage": 0.1, "TimestepPreparationStage": 61.38,
"DenoisingStage": 1569.61, "DenoisingStage": 2180.15,
"DecodingStage": 41.43 "DecodingStage": 8.15
}, },
"denoise_step_ms": { "denoise_step_ms": {
"0": 165.33, "0": 97.92,
"1": 158.34, "1": 228.71,
"2": 167.65, "2": 267.25,
"3": 179.11, "3": 266.93,
"4": 183.98, "4": 265.8,
"5": 175.08, "5": 263.33,
"6": 178.34, "6": 262.05,
"7": 178.53, "7": 261.54,
"8": 178.08 "8": 261.6
}, },
"expected_e2e_ms": 2103.05, "expected_e2e_ms": 3541.48,
"expected_avg_denoise_ms": 173.83, "expected_avg_denoise_ms": 241.68,
"expected_median_denoise_ms": 178.08 "expected_median_denoise_ms": 262.05
}, },
"hunyuan3d_shape_gen": { "hunyuan3d_shape_gen": {
"stages_ms": { "stages_ms": {
"Hunyuan3DShapeBeforeDenoisingStage": 235.65, "Hunyuan3DShapeBeforeDenoisingStage": 544.59,
"Hunyuan3DShapeDenoisingStage": 3452.51, "Hunyuan3DShapeDenoisingStage": 3306.16,
"Hunyuan3DShapeExportStage": 8819.6, "Hunyuan3DShapeExportStage": 8488.42,
"Hunyuan3DShapeSaveStage": 752.4, "Hunyuan3DShapeSaveStage": 859.23,
"Hunyuan3DPaintPreprocessStage": 218136.45, "Hunyuan3DPaintPreprocessStage": 256020.36,
"Hunyuan3DPaintTexGenStage": 10259.21, "Hunyuan3DPaintTexGenStage": 23764.05,
"Hunyuan3DPaintPostprocessStage": 6387.55 "Hunyuan3DPaintPostprocessStage": 7095.01
}, },
"denoise_step_ms": { "denoise_step_ms": {
"0": 150.72, "0": 137.54,
"1": 26.65, "1": 31.13,
"2": 65.91, "2": 65.96,
"3": 68.09, "3": 65.31,
"4": 76.12, "4": 65.03,
"5": 68.16, "5": 65.08,
"6": 61.19, "6": 65.1,
"7": 68.26, "7": 65.48,
"8": 67.92, "8": 64.99,
"9": 68.26, "9": 65.53,
"10": 68.06, "10": 64.91,
"11": 68.22, "11": 65.44,
"12": 68.11, "12": 64.94,
"13": 68.19, "13": 65.42,
"14": 69.58, "14": 65.1,
"15": 66.91, "15": 65.55,
"16": 68.03, "16": 65.03,
"17": 68.36, "17": 65.46,
"18": 68.49, "18": 64.97,
"19": 67.69, "19": 65.39,
"20": 68.19, "20": 65.08,
"21": 69.1, "21": 65.47,
"22": 67.78, "22": 65.89,
"23": 68.36, "23": 64.66,
"24": 68.19, "24": 65.16,
"25": 68.26, "25": 65.59,
"26": 68.06, "26": 64.95,
"27": 68.25, "27": 65.65,
"28": 68.39, "28": 64.94,
"29": 68.26, "29": 65.49,
"30": 68.05, "30": 65.37,
"31": 68.27, "31": 65.58,
"32": 68.2, "32": 64.85,
"33": 68.19, "33": 65.53,
"34": 68.02, "34": 65.18,
"35": 68.3, "35": 65.53,
"36": 68.2, "36": 64.89,
"37": 68.48, "37": 65.54,
"38": 68.23, "38": 65.29,
"39": 68.36, "39": 65.41,
"40": 67.9, "40": 65.01,
"41": 75.76, "41": 65.54,
"42": 62.04, "42": 65.17,
"43": 66.78, "43": 65.49,
"44": 67.85, "44": 64.98,
"45": 68.11, "45": 65.36,
"46": 67.92, "46": 65.25,
"47": 68.15, "47": 65.38,
"48": 67.89, "48": 65.36,
"49": 68.3 "49": 66.03
}, },
"expected_e2e_ms": 248171.5, "expected_e2e_ms": 300141.39,
"expected_avg_denoise_ms": 68.98, "expected_avg_denoise_ms": 66.06,
"expected_median_denoise_ms": 68.19 "expected_median_denoise_ms": 65.36
}, },
"wan2_1_t2v_1.3b_frame_interp_2x": { "wan2_1_t2v_1.3b_frame_interp_2x": {
"stages_ms": { "stages_ms": {
python/sglang/multimodal_gen/test/server/test_server_common.py
+9 -5
View File
@@ -92,6 +92,9 @@ def diffusion_server(case: DiffusionTestCase) -> ServerContext:
if server_args.ring_degree is not None: if server_args.ring_degree is not None:
extra_args += f" --ring-degree {server_args.ring_degree}" extra_args += f" --ring-degree {server_args.ring_degree}"
if server_args.cfg_parallel:
extra_args += " --enable-cfg-parallel"
# LoRA support # LoRA support
if server_args.lora_path: if server_args.lora_path:
extra_args += f" --lora-path {server_args.lora_path}" extra_args += f" --lora-path {server_args.lora_path}"
@@ -835,7 +838,7 @@ Consider updating perf_baselines.json with the snippets below:
# Dynamic LoRA loading test - tests LayerwiseOffload + set_lora interaction # Dynamic LoRA loading test - tests LayerwiseOffload + set_lora interaction
# Server starts WITHOUT lora_path, then set_lora is called after startup # Server starts WITHOUT lora_path, then set_lora is called after startup
if case.server_args.dynamic_lora_path and not is_gt_gen_mode: if case.run_lora_dynamic_load_check and not is_gt_gen_mode:
self._test_dynamic_lora_loading(diffusion_server, case) self._test_dynamic_lora_loading(diffusion_server, case)
generate_fn = get_generate_fn( generate_fn = get_generate_fn(
@@ -871,15 +874,16 @@ Consider updating perf_baselines.json with the snippets below:
validate_mesh_correctness(mesh_path) validate_mesh_correctness(mesh_path)
# Test /v1/models endpoint for router compatibility # Test /v1/models endpoint for router compatibility
if case.run_models_api_check:
self._test_v1_models_endpoint(diffusion_server, case) self._test_v1_models_endpoint(diffusion_server, case)
if case.run_t2v_input_reference_check:
self._test_t2v_rejects_input_reference(diffusion_server, case) self._test_t2v_rejects_input_reference(diffusion_server, case)
# LoRA API functionality test with E2E validation (only for LoRA-enabled cases) # LoRA API functionality test with E2E validation (only for LoRA-enabled cases)
if case.server_args.lora_path or case.server_args.dynamic_lora_path: if case.run_lora_basic_api_check:
self._test_lora_api_functionality(diffusion_server, case, generate_fn) self._test_lora_api_functionality(diffusion_server, case, generate_fn)
# Test dynamic LoRA switching (requires a second LoRA adapter) if case.run_lora_dynamic_switch_check:
if case.server_args.second_lora_path:
self._test_lora_dynamic_switch_e2e( self._test_lora_dynamic_switch_e2e(
diffusion_server, diffusion_server,
case, case,
@@ -887,7 +891,7 @@ Consider updating perf_baselines.json with the snippets below:
case.server_args.second_lora_path, case.server_args.second_lora_path,
) )
# Test multi-LoRA functionality if case.run_multi_lora_api_check:
self._test_multi_lora_e2e( self._test_multi_lora_e2e(
diffusion_server, diffusion_server,
case, case,
python/sglang/multimodal_gen/test/server/test_server_utils.py
+46 -1
View File
@@ -737,6 +737,28 @@ VALIDATOR_REGISTRY = {
} }
def _extract_async_job_error_message(job: Any) -> str | None:
error = getattr(job, "error", None)
if error is None and isinstance(job, dict):
error = job.get("error")
if error is None:
return None
if isinstance(error, dict):
for key in ("message", "detail", "error"):
value = error.get(key)
if value:
return str(value)
return str(error)
message = getattr(error, "message", None)
if message:
return str(message)
return str(error)
def get_generate_fn( def get_generate_fn(
model_path: str, model_path: str,
modality: str, modality: str,
@@ -796,11 +818,25 @@ def get_generate_fn(
while True: while True:
page = client.videos.list() # type: ignore[attr-defined] page = client.videos.list() # type: ignore[attr-defined]
item = next((v for v in page.data if v.id == video_id), None) item = next((v for v in page.data if v.id == video_id), None)
status = getattr(item, "status", None) if item is not None else None
if item and getattr(item, "status", None) == "completed": if status == "completed":
job_completed = True job_completed = True
break break
if status == "failed":
error_message = (
_extract_async_job_error_message(item) or "unknown error"
)
pytest.fail(
f"{case_id}: video job {video_id} failed early: {error_message}"
)
if status in {"cancelled", "deleted"}:
pytest.fail(
f"{case_id}: video job {video_id} ended with status={status}"
)
if time.time() > deadline: if time.time() > deadline:
break break
@@ -837,6 +873,15 @@ def get_generate_fn(
# Validate output file # Validate output file
expected_width, expected_height = parse_dimensions(size) expected_width, expected_height = parse_dimensions(size)
if (
extra_body is not None
and extra_body.get("enable_upscaling")
and expected_width
and expected_height
):
scale = extra_body.get("upscaling_scale", 4)
expected_width *= scale
expected_height *= scale
validate_video_file( validate_video_file(
tmp_path, expected_filename, expected_width, expected_height tmp_path, expected_filename, expected_width, expected_height
) )
python/sglang/multimodal_gen/test/server/testcase_configs.py
+76 -64
View File
@@ -247,6 +247,37 @@ class DiffusionTestCase:
server_args: DiffusionServerArgs server_args: DiffusionServerArgs
sampling_params: DiffusionSamplingParams sampling_params: DiffusionSamplingParams
run_perf_check: bool = True run_perf_check: bool = True
run_models_api_check: bool = True
run_t2v_input_reference_check: bool = True
run_lora_basic_api_check: bool = False
run_lora_dynamic_load_check: bool = False
run_lora_dynamic_switch_check: bool = False
run_multi_lora_api_check: bool = False
def __post_init__(self) -> None:
has_startup_lora = self.server_args.lora_path is not None
has_dynamic_lora = self.server_args.dynamic_lora_path is not None
has_second_lora = self.server_args.second_lora_path is not None
if self.run_lora_basic_api_check and not (has_startup_lora or has_dynamic_lora):
raise ValueError(
f"{self.id}: run_lora_basic_api_check requires lora_path or dynamic_lora_path"
)
if self.run_lora_dynamic_load_check and not has_dynamic_lora:
raise ValueError(
f"{self.id}: run_lora_dynamic_load_check requires dynamic_lora_path"
)
if self.run_lora_dynamic_switch_check and not has_second_lora:
raise ValueError(
f"{self.id}: run_lora_dynamic_switch_check requires second_lora_path"
)
if self.run_multi_lora_api_check and not (has_startup_lora and has_second_lora):
raise ValueError(
f"{self.id}: run_multi_lora_api_check requires lora_path and second_lora_path"
)
def sample_step_indices( def sample_step_indices(
@@ -459,6 +490,9 @@ ONE_GPU_CASES_A: list[DiffusionTestCase] = [
second_lora_path="tarn59/pixel_art_style_lora_z_image_turbo", second_lora_path="tarn59/pixel_art_style_lora_z_image_turbo",
), ),
T2I_sampling_params, T2I_sampling_params,
run_lora_basic_api_check=True,
run_lora_dynamic_switch_check=True,
run_multi_lora_api_check=True,
), ),
DiffusionTestCase( DiffusionTestCase(
"sana_image_t2i", "sana_image_t2i",
@@ -621,6 +655,8 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
DiffusionSamplingParams( DiffusionSamplingParams(
prompt="csetiarcane Nfj1nx with blue hair, a woman walking in a cyberpunk city at night", prompt="csetiarcane Nfj1nx with blue hair, a woman walking in a cyberpunk city at night",
), ),
run_lora_basic_api_check=True,
run_lora_dynamic_load_check=True,
), ),
# NOTE(mick): flaky # NOTE(mick): flaky
# DiffusionTestCase( # DiffusionTestCase(
@@ -680,43 +716,44 @@ ONE_GPU_CASES_B: list[DiffusionTestCase] = [
), ),
TI2V_sampling_params, TI2V_sampling_params,
), ),
# flaky
# === Helios T2V === # === Helios T2V ===
DiffusionTestCase( # DiffusionTestCase(
"helios_base_t2v", # "helios_base_t2v",
DiffusionServerArgs( # DiffusionServerArgs(
model_path="BestWishYsh/Helios-Base", # model_path="BestWishYsh/Helios-Base",
modality="video", # modality="video",
), # ),
DiffusionSamplingParams( # DiffusionSamplingParams(
prompt=T2V_PROMPT, # prompt=T2V_PROMPT,
output_size="640x384", # output_size="640x384",
num_frames=33, # num_frames=33,
), # ),
), # ),
DiffusionTestCase( # DiffusionTestCase(
"helios_mid_t2v", # "helios_mid_t2v",
DiffusionServerArgs( # DiffusionServerArgs(
model_path="BestWishYsh/Helios-Mid", # model_path="BestWishYsh/Helios-Mid",
modality="video", # modality="video",
), # ),
DiffusionSamplingParams( # DiffusionSamplingParams(
prompt=T2V_PROMPT, # prompt=T2V_PROMPT,
output_size="640x384", # output_size="640x384",
num_frames=33, # num_frames=33,
), # ),
), # ),
DiffusionTestCase( # DiffusionTestCase(
"helios_distilled_t2v", # "helios_distilled_t2v",
DiffusionServerArgs( # DiffusionServerArgs(
model_path="BestWishYsh/Helios-Distilled", # model_path="BestWishYsh/Helios-Distilled",
modality="video", # modality="video",
), # ),
DiffusionSamplingParams( # DiffusionSamplingParams(
prompt=T2V_PROMPT, # prompt=T2V_PROMPT,
output_size="640x384", # output_size="640x384",
num_frames=33, # num_frames=33,
), # ),
), # ),
] ]
# Skip hunyuan3d on AMD: marching_cubes surface extraction produces invalid SDF on ROCm. # Skip hunyuan3d on AMD: marching_cubes surface extraction produces invalid SDF on ROCm.
@@ -800,6 +837,7 @@ TWO_GPU_CASES_A = [
DiffusionSamplingParams( DiffusionSamplingParams(
prompt="Nfj1nx with blue hair, a woman walking in a cyberpunk city at night", prompt="Nfj1nx with blue hair, a woman walking in a cyberpunk city at night",
), ),
run_lora_basic_api_check=True,
), ),
DiffusionTestCase( DiffusionTestCase(
"wan2_1_t2v_14b_2gpu", "wan2_1_t2v_14b_2gpu",
@@ -861,6 +899,7 @@ TWO_GPU_CASES_B = [
lora_path="starsfriday/Wan2.1-Divine-Power-LoRA", lora_path="starsfriday/Wan2.1-Divine-Power-LoRA",
), ),
TI2V_sampling_params, TI2V_sampling_params,
run_lora_basic_api_check=True,
), ),
DiffusionTestCase( DiffusionTestCase(
"wan2_1_i2v_14b_720P_2gpu", "wan2_1_i2v_14b_720P_2gpu",
@@ -937,34 +976,7 @@ if not current_platform.is_hip():
MULTI_IMAGE_TI2I_UPLOAD_sampling_params, MULTI_IMAGE_TI2I_UPLOAD_sampling_params,
) )
) )
# Skip turbowan because Triton requires 81920 shared memory, but AMD only has 65536.
ONE_GPU_CASES_B.append(
DiffusionTestCase(
"turbo_wan2_1_t2v_1.3b",
DiffusionServerArgs(
model_path="IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers",
modality="video",
custom_validator="video",
),
DiffusionSamplingParams(
prompt=T2V_PROMPT,
),
)
)
# Skip turbowan because Triton requires 81920 shared memory, but AMD only has 65536.
TWO_GPU_CASES_A.append(
DiffusionTestCase(
"turbo_wan2_2_i2v_a14b_2gpu",
DiffusionServerArgs(
model_path="IPostYellow/TurboWan2.2-I2V-A14B-Diffusers",
modality="video",
custom_validator="video",
num_gpus=2,
tp_size=2,
),
TURBOWAN_I2V_sampling_params,
)
)
# Load global configuration # Load global configuration
BASELINE_CONFIG = BaselineConfig.load( BASELINE_CONFIG = BaselineConfig.load(