minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[diffusion] CI: improve validation method (#13627)

Browse Source
This commit is contained in:
Mick
2025-11-21 09:13:13 +08:00
committed by GitHub
parent 3ae664d786
commit c0a2513b07
4 changed files with 65 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/multimodal_gen/test/server/perf_baselines.json
+11 -11
View File
@@ -6,7 +6,8 @@
}, },
"tolerances": { "tolerances": {
"e2e": 0.05, "e2e": 0.05,
"stage": 0.05, "denoise_stage": 0.05,
"non_denoise_stage": 0.15,
"denoise_step": 0.15, "denoise_step": 0.15,
"denoise_agg": 0.08 "denoise_agg": 0.08
}, },
@@ -40,7 +41,7 @@
"TimestepPreparationStage": 10.6, "TimestepPreparationStage": 10.6,
"LatentPreparationStage": 11.8, "LatentPreparationStage": 11.8,
"DenoisingStage": 21202.6, "DenoisingStage": 21202.6,
"DecodingStage": 476.12 "DecodingStage": 512.6
}, },
"denoise_step_ms": { "denoise_step_ms": {
"0": 1077.77, "0": 1077.77,
@@ -103,7 +104,7 @@
"TimestepPreparationStage": 1.91, "TimestepPreparationStage": 1.91,
"LatentPreparationStage": 4.92, "LatentPreparationStage": 4.92,
"DenoisingStage": 8154.07, "DenoisingStage": 8154.07,
"DecodingStage": 302.01 "DecodingStage": 386.20
}, },
"denoise_step_ms": { "denoise_step_ms": {
"0": 56.03, "0": 56.03,
@@ -168,8 +169,8 @@
"expected_median_denoise_ms": 718.0, "expected_median_denoise_ms": 718.0,
"stages_ms": { "stages_ms": {
"InputValidationStage": 23, "InputValidationStage": 23,
"ImageEncodingStage": 1350.0, "ImageEncodingStage": 1485.0,
"ImageVAEEncodingStage": 340.0, "ImageVAEEncodingStage": 350.0,
"ConditioningStage": 0.13, "ConditioningStage": 0.13,
"TimestepPreparationStage": 13.78, "TimestepPreparationStage": 13.78,
"LatentPreparationStage": 15.0, "LatentPreparationStage": 15.0,
@@ -232,7 +233,7 @@
"wan2_1_t2v_1.3b": { "wan2_1_t2v_1.3b": {
"stages_ms": { "stages_ms": {
"InputValidationStage": 0.08, "InputValidationStage": 0.08,
"TextEncodingStage": 2140.81, "TextEncodingStage": 2945.3,
"ConditioningStage": 0.01, "ConditioningStage": 0.01,
"TimestepPreparationStage": 1.97, "TimestepPreparationStage": 1.97,
"LatentPreparationStage": 6.85, "LatentPreparationStage": 6.85,
@@ -241,7 +242,7 @@
"per_frame_generation": null "per_frame_generation": null
}, },
"denoise_step_ms": { "denoise_step_ms": {
"0": 1998.19, "0": 2831.7,
"1": 1356.14, "1": 1356.14,
"2": 1368.21, "2": 1368.21,
"3": 1369.72, "3": 1369.72,
@@ -296,7 +297,6 @@
"expected_avg_denoise_ms": 1386.08, "expected_avg_denoise_ms": 1386.08,
"expected_median_denoise_ms": 1374.03 "expected_median_denoise_ms": 1374.03
}, },
"wan2_2_i2v_a14b": { "wan2_2_i2v_a14b": {
"stages_ms": { "stages_ms": {
"InputValidationStage": 33.55, "InputValidationStage": 33.55,
@@ -304,7 +304,7 @@
"ConditioningStage": 0.01, "ConditioningStage": 0.01,
"TimestepPreparationStage": 1.52, "TimestepPreparationStage": 1.52,
"LatentPreparationStage": 7.01, "LatentPreparationStage": 7.01,
"ImageVAEEncodingStage": 1904.33, "ImageVAEEncodingStage": 2068,
"DenoisingStage": 231725.24, "DenoisingStage": 231725.24,
"DecodingStage": 2655.39, "DecodingStage": 2655.39,
"per_frame_generation": null "per_frame_generation": null
@@ -351,7 +351,7 @@
"38": 4897.7, "38": 4897.7,
"39": 4888.4 "39": 4888.4
}, },
"expected_e2e_ms": 238587.95, "expected_e2e_ms": 257132.95,
"expected_avg_denoise_ms": 5791.06, "expected_avg_denoise_ms": 5791.06,
"expected_median_denoise_ms": 4900.59 "expected_median_denoise_ms": 4900.59
}, },
@@ -419,7 +419,7 @@
"stages_ms": { "stages_ms": {
"InputValidationStage": 53.67, "InputValidationStage": 53.67,
"TextEncodingStage": 2228.89, "TextEncodingStage": 2228.89,
"ImageEncodingStage": 1908.14, "ImageEncodingStage": 2694.14,
"ConditioningStage": 0.01, "ConditioningStage": 0.01,
"TimestepPreparationStage": 2.25, "TimestepPreparationStage": 2.25,
"LatentPreparationStage": 5.27, "LatentPreparationStage": 5.27,
python/sglang/multimodal_gen/test/server/test_server_performance.py
+15 -10
View File
@@ -406,7 +406,7 @@ Consider updating perf_baselines.json with the snippets below:
summary = validator.collect_metrics(perf_record, stage_metrics) summary = validator.collect_metrics(perf_record, stage_metrics)
if is_baseline_generation_mode or missing_scenario: if is_baseline_generation_mode or missing_scenario:
self._dump_baseline_scenario(case, summary) self._dump_baseline_for_testcase(case, summary)
if missing_scenario: if missing_scenario:
pytest.fail(f"Testcase '{case.id}' not found in perf_baselines.json") pytest.fail(f"Testcase '{case.id}' not found in perf_baselines.json")
return return
@@ -415,8 +415,9 @@ Consider updating perf_baselines.json with the snippets below:
try: try:
validator.validate(perf_record, stage_metrics, case.num_frames) validator.validate(perf_record, stage_metrics, case.num_frames)
except AssertionError: except AssertionError as e:
self._dump_baseline_scenario(case, summary) logger.error(f"Performance validation failed for {case.id}:\n{e}")
self._dump_baseline_for_testcase(case, summary)
raise raise
if case.modality == "video" and summary.frames_per_second: if case.modality == "video" and summary.frames_per_second:
@@ -504,10 +505,14 @@ Consider updating perf_baselines.json with the snippets below:
threshold = BASELINE_CONFIG.improvement_threshold threshold = BASELINE_CONFIG.improvement_threshold
def is_sig_faster(actual, expected): def is_sig_faster(actual, expected):
if expected == 0: if expected == 0 or expected is None:
return False return False
return actual < expected * (1 - threshold) return actual < expected * (1 - threshold)
def safe_get_metric(metric_dict, key):
val = metric_dict.get(key)
return val if val is not None else float("inf")
# Check for any significant improvement # Check for any significant improvement
if ( if (
is_sig_faster(summary.e2e_ms, scenario.expected_e2e_ms) is_sig_faster(summary.e2e_ms, scenario.expected_e2e_ms)
@@ -521,15 +526,15 @@ Consider updating perf_baselines.json with the snippets below:
# Combine metrics, always taking the better (lower) value # Combine metrics, always taking the better (lower) value
new_stages = { new_stages = {
stage: min( stage: min(
summary.stage_metrics.get(stage, float("inf")), safe_get_metric(summary.stage_metrics, stage),
scenario.stages_ms.get(stage, float("inf")), safe_get_metric(scenario.stages_ms, stage),
) )
for stage in set(summary.stage_metrics) | set(scenario.stages_ms) for stage in set(summary.stage_metrics) | set(scenario.stages_ms)
} }
new_denoise_steps = { new_denoise_steps = {
step: min( step: min(
summary.all_denoise_steps.get(step, float("inf")), safe_get_metric(summary.all_denoise_steps, step),
scenario.denoise_step_ms.get(step, float("inf")), safe_get_metric(scenario.denoise_step_ms, step),
) )
for step in set(summary.all_denoise_steps) | set(scenario.denoise_step_ms) for step in set(summary.all_denoise_steps) | set(scenario.denoise_step_ms)
} }
@@ -567,7 +572,7 @@ Consider updating perf_baselines.json with the snippets below:
} }
self._improved_baselines.append({"id": case.id, "baseline": new_baseline}) self._improved_baselines.append({"id": case.id, "baseline": new_baseline})
def _dump_baseline_scenario( def _dump_baseline_for_testcase(
self, case: DiffusionTestCase, summary: "PerformanceSummary" self, case: DiffusionTestCase, summary: "PerformanceSummary"
) -> None: ) -> None:
"""Dump performance metrics as a JSON scenario for baselines.""" """Dump performance metrics as a JSON scenario for baselines."""
@@ -602,7 +607,7 @@ the "scenarios" section of perf_baselines.json:
"{case.id}": {json.dumps(baseline, indent=4)} "{case.id}": {json.dumps(baseline, indent=4)}
""" """
print(output) logger.error(output)
def test_diffusion_perf( def test_diffusion_perf(
self, self,
python/sglang/multimodal_gen/test/server/test_server_utils.py
+28 -9
View File
@@ -306,14 +306,28 @@ class PerformanceValidator:
os.environ.get("SGLANG_GEN_BASELINE", "0") == "1" os.environ.get("SGLANG_GEN_BASELINE", "0") == "1"
) )
def _assert_le(self, name: str, actual: float, expected: float, tolerance: float): def _assert_le(
"""Assert that actual is less than or equal to expected within a tolerance.""" self,
upper_bound = expected * (1 + tolerance) name: str,
actual: float,
expected: float,
tolerance: float,
min_abs_tolerance_ms: float = 20.0,
):
"""Assert that actual is less than or equal to expected within a tolerance.
Uses the larger of relative tolerance or absolute tolerance to prevent
flaky failures on very fast operations.
"""
rel_limit = expected * (1 + tolerance)
abs_limit = expected + min_abs_tolerance_ms
upper_bound = max(rel_limit, abs_limit)
assert actual <= upper_bound, ( assert actual <= upper_bound, (
f"Validation failed for '{name}'.\n" f"Validation failed for '{name}'.\n"
f" - Actual: {actual:.4f}ms\n" f" Actual: {actual:.4f}ms\n"
f" - Expected: {expected:.4f}ms\n" f" Expected: {expected:.4f}ms\n"
f" - Limit: {upper_bound:.4f}ms (tolerance: {tolerance:.1%})" f" Limit: {upper_bound:.4f}ms "
f"(rel_tol: {tolerance:.1%}, abs_pad: {min_abs_tolerance_ms}ms)"
) )
def validate( def validate(
@@ -417,12 +431,17 @@ class PerformanceValidator:
continue continue
actual = summary.stage_metrics.get(stage) actual = summary.stage_metrics.get(stage)
assert actual is not None, f"Stage {stage} timing missing" assert actual is not None, f"Stage {stage} timing missing"
tolerance = (
self.tolerances.denoise_stage
if stage == "DenoisingStage"
else self.tolerances.non_denoise_stage
)
self._assert_le( self._assert_le(
f"Stage '{stage}'", f"Stage '{stage}'",
actual, actual,
expected, expected,
self.tolerances.stage, tolerance,
min_abs_tolerance_ms=120.0, # relax absolute tolerance for non-denoising stages
) )
@@ -458,7 +477,7 @@ class VideoPerformanceValidator(PerformanceValidator):
"Average Frame Time", "Average Frame Time",
summary.avg_frame_time_ms, summary.avg_frame_time_ms,
expected_frame_time, expected_frame_time,
self.tolerances.stage, self.tolerances.denoise_stage,
) )
python/sglang/multimodal_gen/test/server/testcase_configs.py
+11 -2
View File
@@ -30,7 +30,8 @@ class ToleranceConfig:
"""Tolerance ratios for performance validation.""" """Tolerance ratios for performance validation."""
e2e: float e2e: float
stage: float denoise_stage: float
non_denoise_stage: float
denoise_step: float denoise_step: float
denoise_agg: float denoise_agg: float
@@ -65,7 +66,15 @@ class BaselineConfig:
tol_data = data["tolerances"] tol_data = data["tolerances"]
tolerances = ToleranceConfig( tolerances = ToleranceConfig(
e2e=float(os.getenv("SGLANG_E2E_TOLERANCE", tol_data["e2e"])), e2e=float(os.getenv("SGLANG_E2E_TOLERANCE", tol_data["e2e"])),
stage=float(os.getenv("SGLANG_STAGE_TIME_TOLERANCE", tol_data["stage"])), denoise_stage=float(
os.getenv("SGLANG_STAGE_TIME_TOLERANCE", tol_data["denoise_stage"])
),
non_denoise_stage=float(
os.getenv(
"SGLANG_NON_DENOISE_STAGE_TIME_TOLERANCE",
tol_data["non_denoise_stage"],
)
),
denoise_step=float( denoise_step=float(
os.getenv("SGLANG_DENOISE_STEP_TOLERANCE", tol_data["denoise_step"]) os.getenv("SGLANG_DENOISE_STEP_TOLERANCE", tol_data["denoise_step"])
), ),