minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[diffusion] CI: add minimax-h3 ref2va audio consistency coverage and guard peak vram (#35511)

Browse Source
This commit is contained in:
Mick
2026-08-20 17:22:05 +08:00
committed by GitHub
parent f1b9a1f42a
commit 17313cf4b2
24 changed files with 1617 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/multimodal_gen/runtime/managers/gpu_worker.py
+53 -15
View File
@@ -8,7 +8,7 @@ import os
import tempfile
import time
from contextlib import ExitStack
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from typing import Any, Callable, Iterator, List, Union
import numpy as np
@@ -24,6 +24,7 @@ globally_suppress_loggers()
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.distributed import (
get_replica_group,
get_sp_group,
get_tp_rank,
get_tp_world_size,
@@ -151,6 +152,12 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
self.pipeline: ComposedPipelineBase = None
self.init_device_and_model()
self._load_peak_reserved_mb = (
0.0
if current_platform.is_cpu()
else capture_memory_snapshot().peak_reserved_mb
)
self._runtime_peak_reserved_mb = 0.0
self.sp_group = get_sp_group()
self.sp_cpu_group = self.sp_group.cpu_group
self.tp_group = get_tp_group()
@@ -464,11 +471,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
output_batch = None
forward_failed = False
try:
if (
self.is_output_rank
and not current_platform.is_cpu()
and not current_platform.is_mps()
):
if not current_platform.is_cpu() and not current_platform.is_mps():
torch.get_device_module().reset_peak_memory_stats()
start_time = (
@@ -510,7 +513,6 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
return result
output_batch = self._to_output_batch(result)
self._record_output_peak_memory(output_batch)
output_metrics = self._iter_output_metrics(output_batch)
if self.is_output_rank and output_metrics and not current_platform.is_cpu():
@@ -518,6 +520,19 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
for metrics in output_metrics:
metrics.record_memory_snapshot("after_forward", peak_snapshot)
duration_ms = (time.monotonic() - start_time) * 1000
for metrics in output_metrics:
metrics.total_duration_ms = duration_ms
self._materialize_output_transport(output_batch, req, save_output_paths)
self._record_output_peak_memory(output_batch)
collect_perf = (
req.perf_dump_path is not None or envs.SGLANG_DIFFUSION_STAGE_LOGGING
)
if collect_perf and not req.is_warmup:
self._record_replica_peak_memory(output_metrics)
if (
self.is_output_rank
and not req.suppress_logs
@@ -526,12 +541,6 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
):
self.do_mem_analysis(output_batch)
duration_ms = (time.monotonic() - start_time) * 1000
for metrics in output_metrics:
metrics.total_duration_ms = duration_ms
self._materialize_output_transport(output_batch, req, save_output_paths)
if (
not current_platform.is_cpu()
and output_batch.output is None
@@ -690,9 +699,38 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
return np.asarray(materialized.frames)
def _record_output_peak_memory(self, output_batch: OutputBatch) -> None:
if not self.is_output_rank or current_platform.is_cpu():
if current_platform.is_cpu():
return
output_batch.peak_memory_mb = capture_memory_snapshot().peak_reserved_mb
peak_reserved_mb = capture_memory_snapshot().peak_reserved_mb
self._runtime_peak_reserved_mb = max(
self._runtime_peak_reserved_mb, peak_reserved_mb
)
if self.is_output_rank:
output_batch.peak_memory_mb = peak_reserved_mb
def _record_replica_peak_memory(self, output_metrics: list[Any]) -> None:
"""Record replica-wide loading and runtime allocator peaks."""
if not current_platform.is_cuda():
return
peaks = torch.tensor(
[self._load_peak_reserved_mb, self._runtime_peak_reserved_mb],
dtype=torch.float64,
device=current_platform.get_device(self.local_rank),
)
peaks = get_replica_group().all_reduce(peaks, op=torch.distributed.ReduceOp.MAX)
if not self.is_output_rank:
return
snapshot = capture_memory_snapshot()
load_peak_mb, runtime_peak_mb = peaks.tolist()
for metrics in output_metrics:
metrics.record_memory_snapshot(
"load_peak", replace(snapshot, peak_reserved_mb=load_peak_mb)
)
metrics.record_memory_snapshot(
"runtime_peak", replace(snapshot, peak_reserved_mb=runtime_peak_mb)
)
def _forward_group(self, batch: list[Req]) -> OutputBatch:
assert self.pipeline is not None
python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py
+4 -1
View File
@@ -181,7 +181,9 @@ class ComposedPipelineBase(ABC):
if model_subfolder is None:
model_path = maybe_download_model(
self.model_path, force_diffusers_model=True
self.model_path,
force_diffusers_model=True,
revision=self.server_args.revision,
)
else:
model_subfolder = os.path.normpath(model_subfolder)
@@ -196,6 +198,7 @@ class ComposedPipelineBase(ABC):
model_root = maybe_download_model(
self.model_path,
allow_patterns=[f"{model_subfolder}/**"],
revision=self.server_args.revision,
)
model_path = os.path.join(model_root, model_subfolder)
python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/reference_encoding.py
+5 -1
View File
@@ -35,6 +35,7 @@ from sglang.multimodal_gen.configs.models.vaes.minimax_h3_video import (
MiniMaxH3VideoVAEArchConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import get_world_group
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import (
MINIMAX_H3_SUPPORTED_FPS,
)
@@ -321,7 +322,10 @@ def minimax_h3_encode_reference_audio_rows(
waveform = _audio_resampler(int(source_rate))(waveform)
waveform = waveform.to(device)
with _AudioVAEDeterminismContext():
with (
_AudioVAEDeterminismContext(),
set_forward_context(current_timestep=0, attn_metadata=None),
):
audio_data = model.preprocess(
waveform.unsqueeze(1), MINIMAX_H3_AUDIO_SAMPLE_RATE
)
python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py
+5 -2
View File
@@ -34,7 +34,10 @@ _DEFAULT_REALESRGAN_FILENAMES_BY_SCALE = {
4: "RealESRGAN_x4.pth",
8: "RealESRGAN_x8.pth",
}
_LOW_MEMORY_TILED_UPSCALE_FREE_BYTES = 2 * 1024**3
# RRDBNet full-frame 1024x1024 inference can request several GiB of cuDNN
# workspace beyond its output tensor. Keep enough headroom to avoid using an
# allocator OOM as the normal tiled-dispatch mechanism.
_FULL_FRAME_UPSCALE_MIN_FREE_BYTES = 12 * 1024**3
_REALESRGAN_TILE_SIZE = 256
_REALESRGAN_TILE_PAD = 32
@@ -343,7 +346,7 @@ class UpscalerModel:
free_bytes, _ = torch.cuda.mem_get_info(self.device)
output_bytes = h * w * self.scale * self.scale * 3 * 4
required_free_bytes = max(
_LOW_MEMORY_TILED_UPSCALE_FREE_BYTES,
_FULL_FRAME_UPSCALE_MIN_FREE_BYTES,
output_bytes * 4,
)
return free_bytes < required_free_bytes
python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py
+5
View File
@@ -837,6 +837,7 @@ def maybe_download_model(
is_lora: bool = False,
allow_patterns: list[str] | None = None,
force_diffusers_model: bool = False,
revision: str | None = None,
skip_overlay_resolution: bool = False,
) -> str:
"""
@@ -848,6 +849,7 @@ def maybe_download_model(
download: Whether to download the model from Hugging Face Hub
is_lora: If True, skip model completeness verification (LoRA models don't have transformer/vae directories)
force_diffusers_model: If True, apply diffusers model check. Otherwise it should be a component model
revision: Specific Hugging Face Hub revision to resolve
Returns:
Local path to the model
"""
@@ -917,6 +919,7 @@ def maybe_download_model(
local_dir=local_dir,
local_files_only=True,
max_workers=8,
revision=revision,
)
if _is_revisionless_snapshot_root(local_path):
# A cache miss, so the download below re-resolves and rewrites the ref.
@@ -1009,6 +1012,7 @@ def maybe_download_model(
allow_patterns=allow_patterns,
local_dir=local_dir,
max_workers=8,
revision=revision,
)
if not force_diffusers_model:
@@ -1027,6 +1031,7 @@ def maybe_download_model(
local_dir=local_dir,
max_workers=8,
force_download=True,
revision=revision,
)
if not _verify_diffusers_model_complete(local_path):
raise ValueError(
python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py
+9 -1
View File
@@ -141,7 +141,7 @@ def _run_case(case: DiffusionTestCase) -> dict:
if "per_frame_generation" not in perf.stage_metrics:
perf.stage_metrics["per_frame_generation"] = perf.e2e_ms / sp.num_frames
return {
baseline = {
"stages_ms": {k: round(v, 2) for k, v in perf.stage_metrics.items()},
"denoise_step_ms": {
str(k): round(v, 2) for k, v in perf.all_denoise_steps.items()
@@ -150,6 +150,14 @@ def _run_case(case: DiffusionTestCase) -> dict:
"expected_avg_denoise_ms": round(perf.avg_denoise_ms, 2),
"expected_median_denoise_ms": round(perf.median_denoise_ms, 2),
}
if current_platform.is_cuda():
baseline.update(
{
"load_peak_vram_mb": round(perf.load_peak_vram_mb, 2),
"runtime_peak_vram_mb": round(perf.runtime_peak_vram_mb, 2),
}
)
return baseline
finally:
ctx.cleanup()
python/sglang/multimodal_gen/test/server/conftest.py
+36 -8
View File
@@ -43,10 +43,30 @@ def _write_github_step_summary(content: str):
def _write_results_json(results: list, output_path: str = "diffusion-results.json"):
"""Write performance results to JSON file for CI artifact collection."""
try:
existing = []
if os.path.exists(output_path):
try:
with open(output_path, encoding="utf-8") as f:
loaded = json.load(f)
if isinstance(loaded, list):
existing = loaded
except json.JSONDecodeError:
pass
merged = {
(entry.get("class_name"), entry.get("test_name")): entry
for entry in existing
}
merged.update(
{
(entry.get("class_name"), entry.get("test_name")): entry
for entry in results
}
)
with open(output_path, "w") as f:
json.dump(results, f, indent=2)
json.dump(list(merged.values()), f, indent=2)
print(f"[CONFTEST] Wrote results to {output_path}")
except Exception as e:
except (json.JSONDecodeError, OSError) as e:
print(f"[CONFTEST] Failed to write results JSON: {e}")
@@ -63,15 +83,17 @@ def _generate_diffusion_markdown_report(results: list) -> str:
# Main performance table
markdown = header
markdown += "| Test Suite | Test Name | Modality | E2E (ms) | Avg Denoise (ms) | Median Denoise (ms) |\n"
markdown += "| ---------- | --------- | -------- | -------- | ---------------- | ------------------- |\n"
markdown += "| Test Suite | Test Name | Modality | E2E (ms) | Avg Denoise (ms) | Median Denoise (ms) | Load Peak VRAM (MiB) | Runtime Peak VRAM (MiB) |\n"
markdown += "| ---------- | --------- | -------- | -------- | ---------------- | ------------------- | -------------------- | ----------------------- |\n"
for entry in sorted(results, key=lambda x: (x["class_name"], x["test_name"])):
modality = entry.get("modality", "image")
markdown += (
f"| {entry['class_name']} | {entry['test_name']} | {modality} | "
f"{entry['e2e_ms']:.2f} | {entry['avg_denoise_ms']:.2f} | "
f"{entry['median_denoise_ms']:.2f} |\n"
f"{entry['median_denoise_ms']:.2f} | "
f"{entry.get('load_peak_vram_mb', 0):.0f} | "
f"{entry.get('runtime_peak_vram_mb', 0):.0f} |\n"
)
# Video-specific metrics table (if any video tests)
@@ -111,7 +133,7 @@ def pytest_sessionfinish(session):
# Print to stdout (existing behavior)
print("\n\n" + "=" * 35 + " Performance Summary " + "=" * 35)
print(
f"{'Test Suite':<30} | {'Test Name':<20} | {'E2E (ms)':>12} | {'Avg Denoise (ms)':>18} | {'Median Denoise (ms)':>20}"
f"{'Test Suite':<30} | {'Test Name':<20} | {'E2E (ms)':>12} | {'Avg Denoise (ms)':>18} | {'Median Denoise (ms)':>20} | {'Load Peak (MiB)':>15} | {'Runtime Peak (MiB)':>18}"
)
print(
"-" * 30
@@ -123,15 +145,21 @@ def pytest_sessionfinish(session):
+ "-" * 18
+ "-+-"
+ "-" * 20
+ "-+-"
+ "-" * 15
+ "-+-"
+ "-" * 18
)
for entry in sorted_results:
print(
f"{entry['class_name']:<30} | {entry['test_name']:<20} | {entry['e2e_ms']:>12.2f} | "
f"{entry['avg_denoise_ms']:>18.2f} | {entry['median_denoise_ms']:>20.2f}"
f"{entry['avg_denoise_ms']:>18.2f} | {entry['median_denoise_ms']:>20.2f} | "
f"{entry.get('load_peak_vram_mb', 0):>15.0f} | "
f"{entry.get('runtime_peak_vram_mb', 0):>18.0f}"
)
print("=" * 91)
print("=" * 130)
print("\n\n" + "=" * 36 + " Detailed Reports " + "=" * 37)
for entry in sorted_results:
python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json
+5 -1
View File
@@ -279,5 +279,9 @@
"default_mean_abs_diff_threshold_image": 8.0,
"default_ssim_threshold_video": 0.92,
"default_psnr_threshold_video": 24.0,
"default_mean_abs_diff_threshold_video": 10.0
"default_mean_abs_diff_threshold_video": 10.0,
"default_audio_spectral_similarity_threshold": 0.95,
"default_audio_waveform_correlation_threshold": 0.90,
"default_audio_rms_db_diff_threshold": 2.0,
"default_audio_duration_diff_threshold": 0.1
}
python/sglang/multimodal_gen/test/server/gpu_cases.py
+73 -1
View File
@@ -489,6 +489,9 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
"--pipeline-class-name LingBotWorldCausalDMDPipeline --warmup-mode off"
],
text_encoder_cpu_offload=True,
env_vars={
"PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
},
),
REALTIME_MODEL_sampling_params,
run_component_accuracy_check=False,
@@ -652,6 +655,7 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [
output_size="1344x768",
seconds=5,
output_format="mp4",
expect_audio_output=True,
num_outputs_per_prompt=1,
extras={
"task": "fl2va",
@@ -683,7 +687,7 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [
run_component_accuracy_check=False,
run_models_api_check=False,
run_t2v_input_reference_check=False,
)
),
]
TWO_GPU_CASES = [
@@ -723,6 +727,7 @@ TWO_GPU_CASES = [
output_size="1344x768",
seconds=4,
output_format="mp4",
expect_audio_output=True,
num_outputs_per_prompt=1,
extras={
"task": "t2va",
@@ -744,6 +749,73 @@ TWO_GPU_CASES = [
run_models_api_check=False,
run_t2v_input_reference_check=False,
),
DiffusionTestCase(
"minimax_h3_ref2va_video_audio_2gpu_h100",
DiffusionServerArgs(
model_path="MiniMaxAI/MiniMax-H3",
modality="video",
tp_size=2,
ulysses_degree=1,
extras=[
"--model-variant",
"ref2va",
"--revision",
"42ed227ee7df40d41602854ae760620d6eb651fe",
"--performance-mode",
"memory",
"--layerwise-offload-components",
"dit,text_encoder",
"--component-residency",
"vae=resident",
"--dit-offload-prefetch-size",
"1",
"--dit-layerwise-resident-layers",
"20",
"--enable-torch-compile",
"false",
],
),
DiffusionSamplingParams(
prompt=(
"Follow the motion and appearance of <Video 1> while moving the "
"scene to a quiet moonlit room, and use <Audio 1> as the sound "
"reference with coherent timing."
),
output_size="1344x768",
seconds=4,
output_format="mp4",
expect_audio_output=True,
num_outputs_per_prompt=1,
extras={
"task": "ref2va",
"conditions": [
{
"type": "video_audio",
"uri": (
"https://huggingface.co/MiniMaxAI/MiniMax-H3/resolve/"
"42ed227ee7df40d41602854ae760620d6eb651fe/assets/"
"ref2va.mp4"
),
"role": "reference",
}
],
"target": {
"short_edge": 768,
"aspect_ratio": "16:9",
"duration_seconds": 4.0,
},
"num_inference_steps": 8,
"flow_shift": 12.0,
"audio_flow_shift": 3.0,
"seed": 42,
},
),
run_perf_check=False,
run_consistency_check=True,
run_component_accuracy_check=False,
run_models_api_check=False,
run_t2v_input_reference_check=False,
),
DiffusionTestCase(
"flux2_modelopt_fp8_tp2_t2i",
DiffusionServerArgs(
python/sglang/multimodal_gen/test/server/perf_baselines/5090.json
+12 -2
View File
@@ -11,14 +11,18 @@
"denoise_stage": 0.1,
"non_denoise_stage": 0.5,
"denoise_step": 0.25,
"denoise_agg": 0.15
"denoise_agg": 0.15,
"load_peak_vram": 0.01,
"runtime_peak_vram": 0.02
},
"pr_test": {
"e2e": 0.25,
"denoise_stage": 0.25,
"non_denoise_stage": 0.8,
"denoise_step": 0.3,
"denoise_agg": 0.2
"denoise_agg": 0.2,
"load_peak_vram": 0.01,
"runtime_peak_vram": 0.02
}
},
"improvement_reporting": {
@@ -56,6 +60,8 @@
"expected_e2e_ms": 18255.39,
"expected_avg_denoise_ms": 338.78,
"expected_median_denoise_ms": 345.95,
"load_peak_vram_mb": 1126.0,
"runtime_peak_vram_mb": 13180.0,
"estimated_full_test_time_s": 94.0
},
"wan2_1_t2v_1.3b": {
@@ -78,6 +84,8 @@
"expected_e2e_ms": 23418.73,
"expected_avg_denoise_ms": 426.59,
"expected_median_denoise_ms": 427.94,
"load_peak_vram_mb": 4934.0,
"runtime_peak_vram_mb": 18246.0,
"estimated_full_test_time_s": 160.9
},
"zimage_image_t2i": {
@@ -100,6 +108,8 @@
"expected_e2e_ms": 2492.09,
"expected_avg_denoise_ms": 241.53,
"expected_median_denoise_ms": 265.1,
"load_peak_vram_mb": 13120.0,
"runtime_peak_vram_mb": 17776.0,
"estimated_full_test_time_s": 329.8
},
"lingbot_video_moe_t2v": {
python/sglang/multimodal_gen/test/server/perf_baselines/b200.json
+6 -2
View File
@@ -11,14 +11,18 @@
"denoise_stage": 0.1,
"non_denoise_stage": 0.5,
"denoise_step": 0.25,
"denoise_agg": 0.15
"denoise_agg": 0.15,
"load_peak_vram": 0.01,
"runtime_peak_vram": 0.02
},
"pr_test": {
"e2e": 0.25,
"denoise_stage": 0.25,
"non_denoise_stage": 0.8,
"denoise_step": 0.3,
"denoise_agg": 0.2
"denoise_agg": 0.2,
"load_peak_vram": 0.01,
"runtime_peak_vram": 0.02
}
},
"improvement_reporting": {
python/sglang/multimodal_gen/test/server/perf_baselines/h100.json
+114 -2
View File
@@ -11,14 +11,18 @@
"denoise_stage": 0.1,
"non_denoise_stage": 0.5,
"denoise_step": 0.25,
"denoise_agg": 0.15
"denoise_agg": 0.15,
"load_peak_vram": 0.01,
"runtime_peak_vram": 0.02
},
"pr_test": {
"e2e": 0.25,
"denoise_stage": 0.25,
"non_denoise_stage": 0.8,
"denoise_step": 0.3,
"denoise_agg": 0.2
"denoise_agg": 0.2,
"load_peak_vram": 0.01,
"runtime_peak_vram": 0.02
}
},
"improvement_reporting": {
@@ -99,6 +103,8 @@
"expected_e2e_ms": 12770.15,
"expected_avg_denoise_ms": 247.95,
"expected_median_denoise_ms": 249.01,
"load_peak_vram_mb": 41840.0,
"runtime_peak_vram_mb": 46938.0,
"estimated_full_test_time_s": 133.1
},
"qwen_image_t2i_2_gpus": {
@@ -165,6 +171,8 @@
"expected_e2e_ms": 10192.48,
"expected_avg_denoise_ms": 192.9,
"expected_median_denoise_ms": 190.02,
"load_peak_vram_mb": 41840.0,
"runtime_peak_vram_mb": 46938.0,
"estimated_full_test_time_s": 133.2
},
"ideogram4_fp8_t2i": {
@@ -294,6 +302,8 @@
"expected_e2e_ms": 6693.01,
"expected_avg_denoise_ms": 128.85,
"expected_median_denoise_ms": 130.22,
"load_peak_vram_mb": 23594.0,
"runtime_peak_vram_mb": 28540.0,
"estimated_full_test_time_s": 127.4
},
"flux_2_image_t2i": {
@@ -361,6 +371,8 @@
"expected_e2e_ms": 23139.56,
"expected_avg_denoise_ms": 436.76,
"expected_median_denoise_ms": 445.86,
"load_peak_vram_mb": 64344.0,
"runtime_peak_vram_mb": 70560.0,
"estimated_full_test_time_s": 145.2
},
"flux_2_klein_image_t2i": {
@@ -382,6 +394,8 @@
"expected_e2e_ms": 399.62,
"expected_avg_denoise_ms": 30.0,
"expected_median_denoise_ms": 24.14,
"load_peak_vram_mb": 8504.0,
"runtime_peak_vram_mb": 13732.0,
"estimated_full_test_time_s": 120.5
},
"flux_2_klein_base_image_t2i": {
@@ -449,6 +463,8 @@
"expected_e2e_ms": 6112.68,
"expected_avg_denoise_ms": 116.49,
"expected_median_denoise_ms": 117.28,
"load_peak_vram_mb": 8504.0,
"runtime_peak_vram_mb": 13754.0,
"estimated_full_test_time_s": 124.4
},
"flux_2_ti2i": {
@@ -516,6 +532,8 @@
"expected_e2e_ms": 45445.95,
"expected_avg_denoise_ms": 871.81,
"expected_median_denoise_ms": 889.93,
"load_peak_vram_mb": 64344.0,
"runtime_peak_vram_mb": 70564.0,
"estimated_full_test_time_s": 168.9
},
"flux_2_ti2i_multi_image_cache_dit": {
@@ -583,6 +601,8 @@
"expected_e2e_ms": 24192.8,
"expected_avg_denoise_ms": 461.7,
"expected_median_denoise_ms": 53.69,
"load_peak_vram_mb": 64344.0,
"runtime_peak_vram_mb": 71904.0,
"estimated_full_test_time_s": 148.6
},
"flux_image_t2i_2_gpus": {
@@ -649,6 +669,8 @@
"expected_e2e_ms": 4244.27,
"expected_avg_denoise_ms": 79.98,
"expected_median_denoise_ms": 81.01,
"load_peak_vram_mb": 23284.0,
"runtime_peak_vram_mb": 28148.0,
"estimated_full_test_time_s": 125.5
},
"zimage_image_t2i": {
@@ -674,6 +696,8 @@
"expected_e2e_ms": 906.14,
"expected_avg_denoise_ms": 82.61,
"expected_median_denoise_ms": 90.0,
"load_peak_vram_mb": 13120.0,
"runtime_peak_vram_mb": 17776.0,
"estimated_full_test_time_s": 116.3
},
"zimage_image_t2i_fp8": {
@@ -699,6 +723,8 @@
"expected_e2e_ms": 1131.68,
"expected_avg_denoise_ms": 103.24,
"expected_median_denoise_ms": 108.73,
"load_peak_vram_mb": 11344.0,
"runtime_peak_vram_mb": 15868.0,
"estimated_full_test_time_s": 123.7
},
"zimage_image_t2i_multi_lora": {
@@ -724,6 +750,8 @@
"expected_e2e_ms": 910.85,
"expected_avg_denoise_ms": 83.08,
"expected_median_denoise_ms": 89.93,
"load_peak_vram_mb": 14196.0,
"runtime_peak_vram_mb": 17612.0,
"estimated_full_test_time_s": 162.1
},
"zimage_image_t2i_2_gpus": {
@@ -749,6 +777,8 @@
"expected_e2e_ms": 752.84,
"expected_avg_denoise_ms": 51.13,
"expected_median_denoise_ms": 54.3,
"load_peak_vram_mb": 13120.0,
"runtime_peak_vram_mb": 17602.0,
"estimated_full_test_time_s": 75.0
},
"qwen_image_edit_ti2i": {
@@ -816,6 +846,8 @@
"expected_e2e_ms": 32832.9,
"expected_avg_denoise_ms": 633.05,
"expected_median_denoise_ms": 636.16,
"load_peak_vram_mb": 43244.0,
"runtime_peak_vram_mb": 48250.0,
"estimated_full_test_time_s": 153.6
},
"joyai_image_edit_ti2i": {
@@ -873,6 +905,8 @@
"expected_e2e_ms": 27693.61,
"expected_avg_denoise_ms": 666.89,
"expected_median_denoise_ms": 673.29,
"load_peak_vram_mb": 35324.0,
"runtime_peak_vram_mb": 45124.0,
"estimated_full_test_time_s": 117.6
},
"qwen_image_t2i_cache_dit_enabled": {
@@ -939,6 +973,8 @@
"expected_e2e_ms": 4531.7,
"expected_avg_denoise_ms": 84.06,
"expected_median_denoise_ms": 53.48,
"load_peak_vram_mb": 41840.0,
"runtime_peak_vram_mb": 47034.0,
"estimated_full_test_time_s": 124.9
},
"wan2_1_t2v_1.3b_teacache_enabled": {
@@ -1005,6 +1041,8 @@
"expected_e2e_ms": 4739.06,
"expected_avg_denoise_ms": 76.99,
"expected_median_denoise_ms": 98.28,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 18078.0,
"estimated_full_test_time_s": 126.0
},
"wan2_1_t2v_1.3b": {
@@ -1071,6 +1109,8 @@
"expected_e2e_ms": 8106.87,
"expected_avg_denoise_ms": 143.82,
"expected_median_denoise_ms": 144.57,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 18078.0,
"estimated_full_test_time_s": 129.3
},
"wan2_1_t2v_1.3b_cfg_parallel": {
@@ -1137,6 +1177,8 @@
"expected_e2e_ms": 4435.63,
"expected_avg_denoise_ms": 73.51,
"expected_median_denoise_ms": 74.05,
"load_peak_vram_mb": 5422.0,
"runtime_peak_vram_mb": 11972.0,
"estimated_full_test_time_s": 127.6
},
"turbo_wan2_1_t2v_1.3b": {
@@ -1158,6 +1200,8 @@
"expected_e2e_ms": 959.14,
"expected_avg_denoise_ms": 73.38,
"expected_median_denoise_ms": 72.78,
"load_peak_vram_mb": 26256.0,
"runtime_peak_vram_mb": 34340.0,
"estimated_full_test_time_s": 124.7
},
"ltx_2_two_stage_t2v": {
@@ -1225,6 +1269,8 @@
"expected_e2e_ms": 6313.7,
"expected_avg_denoise_ms": 118.58,
"expected_median_denoise_ms": 109.14,
"load_peak_vram_mb": 43920.0,
"runtime_peak_vram_mb": 57528.0,
"estimated_full_test_time_s": 345.4
},
"wan2_2_ti2v_5b": {
@@ -1291,6 +1337,8 @@
"expected_e2e_ms": 14853.56,
"expected_avg_denoise_ms": 253.29,
"expected_median_denoise_ms": 256.21,
"load_peak_vram_mb": 17610.0,
"runtime_peak_vram_mb": 39856.0,
"estimated_full_test_time_s": 141.7
},
"qwen_image_edit_2509_ti2i": {
@@ -1348,6 +1396,8 @@
"expected_e2e_ms": 38630.0,
"expected_avg_denoise_ms": 938.38,
"expected_median_denoise_ms": 945.14,
"load_peak_vram_mb": 43244.0,
"runtime_peak_vram_mb": 47874.0,
"estimated_full_test_time_s": 160.2
},
"qwen_image_layered_i2i": {
@@ -1412,6 +1462,8 @@
"expected_e2e_ms": 33451.98,
"expected_avg_denoise_ms": 661.05,
"expected_median_denoise_ms": 664.26,
"load_peak_vram_mb": 58204.0,
"runtime_peak_vram_mb": 61846.0,
"estimated_full_test_time_s": 161.5
},
"fastwan2_2_ti2v_5b": {
@@ -1431,6 +1483,8 @@
"expected_e2e_ms": 2444.17,
"expected_avg_denoise_ms": 83.86,
"expected_median_denoise_ms": 112.67,
"load_peak_vram_mb": 17610.0,
"runtime_peak_vram_mb": 39054.0,
"estimated_full_test_time_s": 125.2
},
"fast_hunyuan_video": {
@@ -1454,6 +1508,8 @@
"expected_e2e_ms": 9123.68,
"expected_avg_denoise_ms": 871.39,
"expected_median_denoise_ms": 1010.9,
"load_peak_vram_mb": 27320.0,
"runtime_peak_vram_mb": 35926.0,
"estimated_full_test_time_s": 77.0
},
"wan2_2_i2v_a14b_2gpu": {
@@ -1511,6 +1567,8 @@
"expected_e2e_ms": 73811.61,
"expected_avg_denoise_ms": 1689.73,
"expected_median_denoise_ms": 1714.58,
"load_peak_vram_mb": 6440.0,
"runtime_peak_vram_mb": 26322.0,
"estimated_full_test_time_s": 264.1
},
"wan2_1_i2v_14b_480P_2gpu": {
@@ -1579,6 +1637,8 @@
"expected_e2e_ms": 34055.55,
"expected_avg_denoise_ms": 653.89,
"expected_median_denoise_ms": 656.68,
"load_peak_vram_mb": 35350.0,
"runtime_peak_vram_mb": 42674.0,
"estimated_full_test_time_s": 243.0
},
"wan2_1_i2v_14b_720P_2gpu": {
@@ -1647,6 +1707,8 @@
"expected_e2e_ms": 98831.21,
"expected_avg_denoise_ms": 1923.51,
"expected_median_denoise_ms": 1933.7,
"load_peak_vram_mb": 35350.0,
"runtime_peak_vram_mb": 51230.0,
"estimated_full_test_time_s": 248.9
},
"wan2_2_t2v_a14b_2gpu": {
@@ -1703,6 +1765,8 @@
"expected_e2e_ms": 83297.78,
"expected_avg_denoise_ms": 2029.88,
"expected_median_denoise_ms": 2020.71,
"load_peak_vram_mb": 6440.0,
"runtime_peak_vram_mb": 21524.0,
"estimated_full_test_time_s": 204.3
},
"wan2_1_t2v_14b_2gpu": {
@@ -1769,6 +1833,8 @@
"expected_e2e_ms": 24072.24,
"expected_avg_denoise_ms": 461.16,
"expected_median_denoise_ms": 467.69,
"load_peak_vram_mb": 31246.0,
"runtime_peak_vram_mb": 37788.0,
"estimated_full_test_time_s": 173.2
},
"wan2_2_t2v_a14b_lora_2gpu": {
@@ -1825,6 +1891,8 @@
"expected_e2e_ms": 62134.19,
"expected_avg_denoise_ms": 1437.82,
"expected_median_denoise_ms": 1416.85,
"load_peak_vram_mb": 6784.0,
"runtime_peak_vram_mb": 21450.0,
"estimated_full_test_time_s": 181.8
},
"wan2_1_t2v_1_3b_lora_1gpu": {
@@ -1891,6 +1959,8 @@
"expected_e2e_ms": 8138.0,
"expected_avg_denoise_ms": 146.48,
"expected_median_denoise_ms": 147.68,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 18350.0,
"estimated_full_test_time_s": 129.6
},
"wan2_1_i2v_14b_lora_2gpu": {
@@ -1959,6 +2029,8 @@
"expected_e2e_ms": 98305.51,
"expected_avg_denoise_ms": 1913.15,
"expected_median_denoise_ms": 1922.21,
"load_peak_vram_mb": 36266.0,
"runtime_peak_vram_mb": 51132.0,
"estimated_full_test_time_s": 248.2
},
"flux_2_image_t2i_2_gpus": {
@@ -2026,6 +2098,8 @@
"expected_e2e_ms": 13216.13,
"expected_avg_denoise_ms": 242.72,
"expected_median_denoise_ms": 246.92,
"load_peak_vram_mb": 33110.0,
"runtime_peak_vram_mb": 38262.0,
"estimated_full_test_time_s": 135.3
},
"qwen_image_edit_2511_ti2i": {
@@ -2083,6 +2157,8 @@
"expected_e2e_ms": 22989.11,
"expected_avg_denoise_ms": 551.91,
"expected_median_denoise_ms": 555.79,
"load_peak_vram_mb": 43244.0,
"runtime_peak_vram_mb": 47874.0,
"estimated_full_test_time_s": 143.7
},
"fsdp-inference": {
@@ -2108,6 +2184,8 @@
"expected_e2e_ms": 1042.43,
"expected_avg_denoise_ms": 82.26,
"expected_median_denoise_ms": 81.75,
"load_peak_vram_mb": 7176.0,
"runtime_peak_vram_mb": 12634.0,
"estimated_full_test_time_s": 122.7
},
"hunyuan3d_shape_gen": {
@@ -2175,6 +2253,8 @@
"expected_e2e_ms": 237254.68,
"expected_avg_denoise_ms": 32.68,
"expected_median_denoise_ms": 30.2,
"load_peak_vram_mb": 876.0,
"runtime_peak_vram_mb": 6094.0,
"estimated_full_test_time_s": 420.1
},
"wan2_1_t2v_1.3b_frame_interp_2x": {
@@ -2241,6 +2321,8 @@
"expected_e2e_ms": 8273.83,
"expected_avg_denoise_ms": 146.61,
"expected_median_denoise_ms": 147.3,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 18080.0,
"estimated_full_test_time_s": 129.3
},
"flux_2_image_t2i_upscaling_4x": {
@@ -2308,6 +2390,8 @@
"expected_e2e_ms": 22732.26,
"expected_avg_denoise_ms": 429.1,
"expected_median_denoise_ms": 438.64,
"load_peak_vram_mb": 64344.0,
"runtime_peak_vram_mb": 70620.0,
"estimated_full_test_time_s": 145.1
},
"wan2_1_t2v_1.3b_upscaling_4x": {
@@ -2374,6 +2458,8 @@
"expected_e2e_ms": 8333.66,
"expected_avg_denoise_ms": 144.68,
"expected_median_denoise_ms": 144.74,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 24380.0,
"estimated_full_test_time_s": 129.3
},
"wan2_1_t2v_1.3b_frame_interp_2x_upscaling_4x": {
@@ -2440,6 +2526,8 @@
"expected_e2e_ms": 8099.03,
"expected_avg_denoise_ms": 143.71,
"expected_median_denoise_ms": 144.67,
"load_peak_vram_mb": 7814.0,
"runtime_peak_vram_mb": 24388.0,
"estimated_full_test_time_s": 129.4
},
"ltx_2.3_one_stage_ti2v": {
@@ -2489,6 +2577,8 @@
"expected_e2e_ms": 16722.42,
"expected_avg_denoise_ms": 520.0,
"expected_median_denoise_ms": 494.9,
"load_peak_vram_mb": 46278.0,
"runtime_peak_vram_mb": 49390.0,
"estimated_full_test_time_s": 144.2
},
"ltx_2.3_two_stage_t2v_2gpus": {
@@ -2546,6 +2636,8 @@
"expected_e2e_ms": 6349.58,
"expected_avg_denoise_ms": 158.2,
"expected_median_denoise_ms": 145.66,
"load_peak_vram_mb": 47234.0,
"runtime_peak_vram_mb": 59544.0,
"estimated_full_test_time_s": 160.0
},
"ltx_2_3_two_stage_ti2v_2gpus": {
@@ -2603,6 +2695,8 @@
"expected_e2e_ms": 9687.04,
"expected_avg_denoise_ms": 250.16,
"expected_median_denoise_ms": 242.79,
"load_peak_vram_mb": 47234.0,
"runtime_peak_vram_mb": 60286.0,
"estimated_full_test_time_s": 170.0
},
"longlive2_t2v": {
@@ -2619,6 +2713,8 @@
"expected_e2e_ms": 5648.49,
"expected_avg_denoise_ms": 477.49,
"expected_median_denoise_ms": 56.74,
"load_peak_vram_mb": 16110.0,
"runtime_peak_vram_mb": 45756.0,
"estimated_full_test_time_s": 153.1
},
"longlive2_i2v": {
@@ -2635,6 +2731,8 @@
"expected_e2e_ms": 9086.81,
"expected_avg_denoise_ms": 492.02,
"expected_median_denoise_ms": 151.9,
"load_peak_vram_mb": 16110.0,
"runtime_peak_vram_mb": 45756.0,
"estimated_full_test_time_s": 149.4
},
"lingbot_video_moe_t2v": {
@@ -2651,6 +2749,8 @@
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"load_peak_vram_mb": 42814.0,
"runtime_peak_vram_mb": 73256.0,
"estimated_full_test_time_s": 126.0
},
"ltx_2_3_hq_pipeline": {
@@ -2695,6 +2795,8 @@
"expected_e2e_ms": 15974.21,
"expected_avg_denoise_ms": 802.06,
"expected_median_denoise_ms": 747.62,
"load_peak_vram_mb": 56048.0,
"runtime_peak_vram_mb": 59804.0,
"estimated_full_test_time_s": 363.2
},
"qwen_image_t2i_cache_dit_scm_config_diffusers_1gpu": {
@@ -2801,6 +2903,14 @@
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 55.4
},
"minimax_h3_ref2va_video_audio_2gpu_h100": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 0.0,
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 266.7
},
"minimax_h3_t2va_2gpu_h100": {
"stages_ms": {
"InputValidationStage": 0.05,
@@ -2826,6 +2936,8 @@
"expected_e2e_ms": 19700.0,
"expected_avg_denoise_ms": 2045.39,
"expected_median_denoise_ms": 2332.99,
"load_peak_vram_mb": 45428.0,
"runtime_peak_vram_mb": 63312.0,
"estimated_full_test_time_s": 208.0
},
"mova_360p_tp2": {
python/sglang/multimodal_gen/test/server/test_server_common.py
+225 -28
View File
@@ -26,6 +26,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
from sglang.multimodal_gen.test.server import conftest
from sglang.multimodal_gen.test.server.realtime_consistency import (
RealtimeChunkStats,
pop_realtime_key_frames,
pop_realtime_perf_stats,
validate_realtime_perf_stats,
@@ -51,10 +52,17 @@ from sglang.multimodal_gen.test.test_utils import (
_consistency_gt_filenames,
_get_consistency_gt_dir,
action_gt_exists,
audio_gt_exists,
compare_audio_with_gt,
compare_with_gt,
encode_audio_gt_wav,
extract_audio_pcm_from_video_bytes,
extract_key_frames_from_video,
get_action_consistency_gt_candidates,
get_action_consistency_gt_remote_files,
get_audio_consistency_gt_candidates,
get_audio_consistency_gt_remote_files,
get_audio_consistency_thresholds,
get_consistency_gt_candidates,
get_consistency_gt_remote_files,
get_consistency_threshold_path,
@@ -63,7 +71,9 @@ from sglang.multimodal_gen.test.test_utils import (
gt_exists,
image_bytes_to_numpy,
load_action_consistency_gt,
load_audio_consistency_gt,
load_consistency_gt,
save_audio_gt_artifact,
save_consistency_failure_artifact,
save_missing_consistency_gt_artifact,
wait_for_req_perf_record,
@@ -403,18 +413,7 @@ class DiffusionServerBase:
scenario = BASELINE_CONFIG.scenarios.get(case.id)
missing_scenario = False
if scenario is None:
# Create dummy scenario to allow metric collection
scenario = type(
"DummyScenario",
(),
{
"expected_e2e_ms": 0,
"expected_avg_denoise_ms": 0,
"expected_median_denoise_ms": 0,
"stages_ms": {},
"denoise_step_ms": {},
},
)()
scenario = ScenarioConfig({}, {}, 0, 0, 0)
if not is_baseline_generation_mode:
missing_scenario = True
@@ -450,6 +449,29 @@ class DiffusionServerBase:
)
return
if current_platform.is_cuda():
expected_load_peak_vram_mb = scenario.load_peak_vram_mb
expected_runtime_peak_vram_mb = scenario.runtime_peak_vram_mb
if (
expected_load_peak_vram_mb is None
or expected_runtime_peak_vram_mb is None
):
self._dump_baseline_for_testcase(case, summary, missing_scenario)
pytest.fail(
f"Testcase '{case.id}' is missing a load/runtime peak VRAM "
f"baseline in {get_perf_baseline_path()}"
)
try:
validator.validate_peak_vram(
summary,
expected_load_peak_vram_mb,
expected_runtime_peak_vram_mb,
)
except AssertionError as e:
logger.error(f"Peak VRAM validation failed for {case.id}:\n{e}")
self._dump_baseline_for_testcase(case, summary, missing_scenario)
raise
# only run performance validation if run_perf_check is True
try:
validator.validate(perf_record, case.sampling_params.num_frames)
@@ -458,12 +480,93 @@ class DiffusionServerBase:
self._dump_baseline_for_testcase(case, summary, missing_scenario)
raise
self._record_performance_result(case, summary)
def _validate_realtime_performance(
self,
ctx: ServerContext,
case: DiffusionTestCase,
chunk_stats: list[RealtimeChunkStats],
) -> None:
validate_realtime_perf_stats(
case.id,
chunk_stats,
case.sampling_params.realtime_perf_thresholds,
ignore_initial_chunks=(
case.sampling_params.realtime_perf_ignore_initial_chunks
),
)
if not case.run_perf_check or not current_platform.is_cuda():
return
request_id = next(
(stat.request_id for stat in reversed(chunk_stats) if stat.request_id),
None,
)
if request_id is None:
pytest.fail(f"{case.id}: realtime chunk stats are missing request IDs")
perf_record = wait_for_req_perf_record(
request_id,
ctx.perf_log_path,
timeout=30,
)
if perf_record is None:
pytest.fail(f"{case.id}: realtime request performance record is missing")
scenario = BASELINE_CONFIG.scenarios.get(case.id)
if scenario is None:
pytest.fail(f"Testcase '{case.id}' not found in {get_perf_baseline_path()}")
validator = PerformanceValidator(
scenario=scenario,
tolerances=BASELINE_CONFIG.tolerances,
step_fractions=BASELINE_CONFIG.step_fractions,
)
summary = validator.collect_metrics(perf_record)
self._print_performance_log(case, summary, scenario)
self._record_performance_result(case, summary)
if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1":
logger.info(
"%s realtime peak VRAM baseline: load=%.0fMiB, runtime=%.0fMiB",
case.id,
summary.load_peak_vram_mb,
summary.runtime_peak_vram_mb,
)
return
if scenario.load_peak_vram_mb is None or scenario.runtime_peak_vram_mb is None:
pytest.fail(
f"Testcase '{case.id}' is missing a load/runtime peak VRAM "
f"baseline in {get_perf_baseline_path()}; measured "
f"load={summary.load_peak_vram_mb:.0f}MiB, "
f"runtime={summary.runtime_peak_vram_mb:.0f}MiB"
)
try:
validator.validate_peak_vram(
summary,
scenario.load_peak_vram_mb,
scenario.runtime_peak_vram_mb,
)
except AssertionError as e:
logger.error(f"Peak VRAM validation failed for {case.id}:\n{e}")
raise
def _record_performance_result(
self,
case: DiffusionTestCase,
summary: PerformanceSummary,
) -> None:
result = {
"test_name": case.id,
"modality": case.server_args.modality,
"e2e_ms": summary.e2e_ms,
"avg_denoise_ms": summary.avg_denoise_ms,
"median_denoise_ms": summary.median_denoise_ms,
"load_peak_vram_mb": summary.load_peak_vram_mb,
"runtime_peak_vram_mb": summary.runtime_peak_vram_mb,
"stage_metrics": summary.stage_metrics,
"sampled_steps": summary.sampled_steps,
}
@@ -495,7 +598,9 @@ class DiffusionServerBase:
(
f" e2e={summary.e2e_ms:.2f}ms, "
f"avg_denoise={summary.avg_denoise_ms:.2f}ms, "
f"median_denoise={summary.median_denoise_ms:.2f}ms"
f"median_denoise={summary.median_denoise_ms:.2f}ms, "
f"load_peak_vram={summary.load_peak_vram_mb:.0f}MiB, "
f"runtime_peak_vram={summary.runtime_peak_vram_mb:.0f}MiB"
),
]
if scenario is not None:
@@ -505,6 +610,16 @@ class DiffusionServerBase:
f"avg_denoise={scenario.expected_avg_denoise_ms:.2f}ms, "
f"median_denoise={scenario.expected_median_denoise_ms:.2f}ms"
)
if (
scenario is not None
and scenario.load_peak_vram_mb is not None
and scenario.runtime_peak_vram_mb is not None
):
lines.append(
" peak_vram_baseline: "
f"load={scenario.load_peak_vram_mb:.0f}MiB, "
f"runtime={scenario.runtime_peak_vram_mb:.0f}MiB"
)
if summary.stage_metrics:
stages = ", ".join(
f"{name}={duration:.2f}ms"
@@ -544,6 +659,14 @@ class DiffusionServerBase:
"expected_median_denoise_ms": round(summary.median_denoise_ms, 2),
}
if current_platform.is_cuda():
baseline.update(
{
"load_peak_vram_mb": round(summary.load_peak_vram_mb, 2),
"runtime_peak_vram_mb": round(summary.runtime_peak_vram_mb, 2),
}
)
if measured_full_time is not None:
baseline["estimated_full_test_time_s"] = round(measured_full_time, 1)
@@ -612,6 +735,15 @@ class DiffusionServerBase:
logger.info(
"[Artifact] Saved missing consistency GT: %s", artifact_path
)
if case.sampling_params.expect_audio_output:
audio_path = save_audio_gt_artifact(
os.environ.get("SGLANG_DIFFUSION_ARTIFACT_DIR"),
case.id,
num_gpus,
extract_audio_pcm_from_video_bytes(content),
)
if audio_path is not None:
logger.info("[Artifact] Saved missing audio GT: %s", audio_path)
if _get_consistency_gt_dir() is not None:
names = ", ".join(
get_consistency_gt_candidates(
@@ -739,6 +871,67 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
f"max_mean_abs_diff={result.max_mean_abs_diff:.4f})"
)
if case.sampling_params.expect_audio_output:
self._validate_audio_consistency(case, content)
def _validate_audio_consistency(
self,
case: DiffusionTestCase,
content: bytes,
) -> None:
num_gpus = case.server_args.num_gpus
output_audio = extract_audio_pcm_from_video_bytes(content)
if not audio_gt_exists(case.id, num_gpus):
artifact_path = save_audio_gt_artifact(
os.environ.get("SGLANG_DIFFUSION_ARTIFACT_DIR"),
case.id,
num_gpus,
output_audio,
)
if artifact_path is not None:
logger.info("[Artifact] Saved missing audio GT: %s", artifact_path)
names = ", ".join(get_audio_consistency_gt_candidates(case.id, num_gpus))
pytest.fail(f"Audio GT not found for {case.id}. Expected one of: {names}")
gt_audio = load_audio_consistency_gt(case.id, num_gpus)
result = compare_audio_with_gt(
output_audio,
gt_audio,
get_audio_consistency_thresholds(case.id),
)
if not result.passed:
gt_remote_info = "\n".join(
f" - {filename}: {url}"
for filename, url in get_audio_consistency_gt_remote_files(
case.id, num_gpus
)
)
pytest.fail(
f"Audio consistency check failed for {case.id}:\n"
f" Metrics: spectral_similarity={result.spectral_similarity:.4f}, "
f"waveform_correlation={result.waveform_correlation:.4f}, "
f"rms_db_diff={result.rms_db_diff:.4f}, "
f"duration_diff={result.duration_diff:.4f}s\n"
f" Thresholds: spectral_similarity>="
f"{result.thresholds.spectral_similarity_threshold}, "
f"waveform_correlation>="
f"{result.thresholds.waveform_correlation_threshold}, "
f"rms_db_diff<={result.thresholds.rms_db_diff_threshold}, "
f"duration_diff<={result.thresholds.duration_diff_threshold}s\n"
f" Compared GT files and links:\n{gt_remote_info}"
)
logger.info(
"[Consistency] %s: PASSED audio GT check "
"(spectral_similarity=%.4f, waveform_correlation=%.4f, "
"rms_db_diff=%.4f, duration_diff=%.4fs)",
case.id,
result.spectral_similarity,
result.waveform_correlation,
result.rms_db_diff,
result.duration_diff,
)
def _extract_action_array(
self,
payload: dict[str, Any],
@@ -865,16 +1058,22 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
logger.warning(
f"{case.id}: expected 3 frames, got {len(frames)}, skipping frame save"
)
return
else:
# Save frames (reuse naming from _consistency_gt_filenames)
filenames = _consistency_gt_filenames(case.id, num_gpus, is_video=True)
from PIL import Image
# Save frames (reuse naming from _consistency_gt_filenames)
filenames = _consistency_gt_filenames(case.id, num_gpus, is_video=True)
from PIL import Image
for frame, fn in zip(frames, filenames):
frame_path = out_dir / fn
Image.fromarray(frame).save(frame_path)
logger.info(f"Saved GT frame: {frame_path}")
for frame, fn in zip(frames, filenames):
frame_path = out_dir / fn
Image.fromarray(frame).save(frame_path)
logger.info(f"Saved GT frame: {frame_path}")
if case.sampling_params.expect_audio_output:
audio_path = out_dir / f"{case.id}_{num_gpus}gpu_audio.wav"
audio_path.write_bytes(
encode_audio_gt_wav(extract_audio_pcm_from_video_bytes(content))
)
logger.info("Saved GT audio: %s", audio_path)
else:
# Save image
from sglang.multimodal_gen.test.test_utils import detect_image_format
@@ -1372,15 +1571,13 @@ Pinned revision used by this check: {SGL_TEST_FILES_CI_DATA_REVISION}
failures.append((name, str(exc)))
if is_realtime_case:
chunk_stats = pop_realtime_perf_stats(case.id)
run_case_check(
"performance",
lambda: validate_realtime_perf_stats(
case.id,
pop_realtime_perf_stats(case.id),
case.sampling_params.realtime_perf_thresholds,
ignore_initial_chunks=(
case.sampling_params.realtime_perf_ignore_initial_chunks
),
lambda: self._validate_realtime_performance(
diffusion_server,
case,
chunk_stats,
),
)
else:
python/sglang/multimodal_gen/test/server/test_server_utils.py
+64 -23
View File
@@ -51,6 +51,7 @@ from sglang.multimodal_gen.test.test_utils import (
get_video_frame_count,
is_image_url,
prepare_perf_log,
validate_audio_output,
validate_image,
validate_image_file,
validate_openai_video,
@@ -461,9 +462,7 @@ class ServerManager:
flush=True,
)
self._wait_for_ready(process, stdout_path)
return ServerContext(
context = ServerContext(
port=self.port,
process=process,
model=self.model,
@@ -473,6 +472,13 @@ class ServerManager:
_stdout_fh=stdout_fh,
_log_thread=log_thread,
)
try:
self._wait_for_ready(process, stdout_path)
except BaseException:
context.cleanup()
raise
return context
def _wait_for_ready(self, process: subprocess.Popen, stdout_path: Path) -> None:
"""Wait until model warmup finishes and inference traffic is accepted."""
@@ -539,7 +545,8 @@ class PerformanceValidator:
actual: float,
expected: float,
tolerance: float,
min_abs_tolerance_ms: float = 20.0,
min_abs_tolerance: float = 20.0,
unit: str = "ms",
):
"""Assert that actual is less than or equal to expected within a tolerance.
@@ -555,29 +562,54 @@ class PerformanceValidator:
# Use 100% higher tolerance for AMD (2x the expected value)
amd_tolerance = 1.0 # 100%
upper_bound = calculate_upper_bound(
expected, amd_tolerance, min_abs_tolerance_ms
expected, amd_tolerance, min_abs_tolerance
)
if actual > upper_bound:
logger.warning(
f"[AMD PERF WARNING] Validation would fail for '{name}'.\n"
f" Actual: {actual:.4f}ms\n"
f" Expected: {expected:.4f}ms\n"
f" AMD Limit: {upper_bound:.4f}ms "
f"(rel_tol: {amd_tolerance:.1%}, abs_pad: {min_abs_tolerance_ms}ms)\n"
f" Actual: {actual:.4f}{unit}\n"
f" Expected: {expected:.4f}{unit}\n"
f" AMD Limit: {upper_bound:.4f}{unit} "
f"(rel_tol: {amd_tolerance:.1%}, "
f"abs_pad: {min_abs_tolerance}{unit})\n"
f" Original tolerance was: {tolerance:.1%}"
)
else:
upper_bound = calculate_upper_bound(
expected, tolerance, min_abs_tolerance_ms
)
upper_bound = calculate_upper_bound(expected, tolerance, min_abs_tolerance)
assert actual <= upper_bound, (
f"Validation failed for '{name}'.\n"
f" Actual: {actual:.4f}ms\n"
f" Expected: {expected:.4f}ms\n"
f" Limit: {upper_bound:.4f}ms "
f"(rel_tol: {tolerance:.1%}, abs_pad: {min_abs_tolerance_ms}ms)"
f" Actual: {actual:.4f}{unit}\n"
f" Expected: {expected:.4f}{unit}\n"
f" Limit: {upper_bound:.4f}{unit} "
f"(rel_tol: {tolerance:.1%}, "
f"abs_pad: {min_abs_tolerance}{unit})"
)
def validate_peak_vram(
self,
summary: PerformanceSummary,
expected_load_peak_vram_mb: float,
expected_runtime_peak_vram_mb: float,
) -> None:
assert summary.load_peak_vram_mb > 0, "Load peak VRAM metric missing"
assert summary.runtime_peak_vram_mb > 0, "Runtime peak VRAM metric missing"
self._assert_le(
"Load Peak VRAM",
summary.load_peak_vram_mb,
expected_load_peak_vram_mb,
self.tolerances.load_peak_vram,
min_abs_tolerance=128.0,
unit=" MiB",
)
self._assert_le(
"Runtime Peak VRAM",
summary.runtime_peak_vram_mb,
expected_runtime_peak_vram_mb,
self.tolerances.runtime_peak_vram,
min_abs_tolerance=128.0,
unit=" MiB",
)
def validate(
self, perf_record: RequestPerfRecord, *args, **kwargs
) -> PerformanceSummary:
@@ -640,7 +672,7 @@ class PerformanceValidator:
actual,
expected,
FIRST_DENOISE_STEP_TOLERANCE,
min_abs_tolerance_ms=FIRST_DENOISE_STEP_MIN_ABS_TOLERANCE_MS,
min_abs_tolerance=FIRST_DENOISE_STEP_MIN_ABS_TOLERANCE_MS,
)
continue
@@ -667,15 +699,15 @@ class PerformanceValidator:
)
if stage.endswith("DecodingStage"):
tolerance = max(tolerance, 0.9)
min_abs_tolerance_ms = DECODING_STAGE_MIN_ABS_TOLERANCE_MS
min_abs_tolerance = DECODING_STAGE_MIN_ABS_TOLERANCE_MS
else:
min_abs_tolerance_ms = 120.0
min_abs_tolerance = 120.0
self._assert_le(
f"Stage '{stage}'",
actual,
expected,
tolerance,
min_abs_tolerance_ms=min_abs_tolerance_ms,
min_abs_tolerance=min_abs_tolerance,
)
@@ -696,7 +728,7 @@ class VideoPerformanceValidator(PerformanceValidator):
actual,
expected,
FIRST_DENOISE_STEP_TOLERANCE,
min_abs_tolerance_ms=FIRST_DENOISE_STEP_MIN_ABS_TOLERANCE_MS,
min_abs_tolerance=FIRST_DENOISE_STEP_MIN_ABS_TOLERANCE_MS,
)
continue
@@ -707,7 +739,7 @@ class VideoPerformanceValidator(PerformanceValidator):
actual,
expected,
self.tolerances.denoise_step,
min_abs_tolerance_ms=VIDEO_DENOISE_STEP_MIN_ABS_TOLERANCE_MS,
min_abs_tolerance=VIDEO_DENOISE_STEP_MIN_ABS_TOLERANCE_MS,
)
def validate(
@@ -1022,6 +1054,15 @@ def get_generate_fn(
validate_video_file(
tmp_path, expected_filename, expected_width, expected_height
)
if sampling_params.expect_audio_output:
audio_info = validate_audio_output(tmp_path)
logger.info(
"%s: validated audio output (%s Hz, %s channels, %.3fs)",
case_id,
audio_info.sample_rate,
audio_info.channels,
audio_info.duration_seconds,
)
if expected_frame_count is not None:
actual_count = get_video_frame_count(tmp_path)
@@ -1390,7 +1431,7 @@ def get_generate_fn(
init_payload=init_payload,
events=list(sampling_params.realtime_events),
num_chunks=sampling_params.realtime_num_chunks,
require_chunk_stats=bool(sampling_params.realtime_perf_thresholds),
require_chunk_stats=True,
)
)
record_realtime_perf_stats(case_id, realtime_output.chunk_stats)
python/sglang/multimodal_gen/test/server/testcase_configs.py
+55 -22
View File
@@ -46,6 +46,8 @@ class ToleranceConfig:
non_denoise_stage: float
denoise_step: float
denoise_agg: float
load_peak_vram: float = 0.01
runtime_peak_vram: float = 0.02
@classmethod
def load_profile(cls, all_tolerances: dict, profile_name: str) -> ToleranceConfig:
@@ -86,6 +88,18 @@ class ToleranceConfig:
denoise_agg=float(
os.getenv("SGLANG_DENOISE_AGG_TOLERANCE", tol_data["denoise_agg"])
),
load_peak_vram=float(
os.getenv(
"SGLANG_LOAD_PEAK_VRAM_TOLERANCE",
tol_data.get("load_peak_vram", 0.01),
)
),
runtime_peak_vram=float(
os.getenv(
"SGLANG_RUNTIME_PEAK_VRAM_TOLERANCE",
tol_data.get("runtime_peak_vram", 0.02),
)
),
)
@@ -99,6 +113,25 @@ class ScenarioConfig:
expected_avg_denoise_ms: float
expected_median_denoise_ms: float
estimated_full_test_time_s: float | None = None
load_peak_vram_mb: float | None = None
runtime_peak_vram_mb: float | None = None
@classmethod
def from_dict(cls, cfg: dict[str, Any]) -> ScenarioConfig:
def optional_float(name: str) -> float | None:
value = cfg.get(name)
return float(value) if value is not None else None
return cls(
stages_ms=cfg["stages_ms"],
denoise_step_ms={int(k): v for k, v in cfg["denoise_step_ms"].items()},
expected_e2e_ms=float(cfg["expected_e2e_ms"]),
expected_avg_denoise_ms=float(cfg["expected_avg_denoise_ms"]),
expected_median_denoise_ms=float(cfg["expected_median_denoise_ms"]),
estimated_full_test_time_s=optional_float("estimated_full_test_time_s"),
load_peak_vram_mb=optional_float("load_peak_vram_mb"),
runtime_peak_vram_mb=optional_float("runtime_peak_vram_mb"),
)
@dataclass
@@ -122,16 +155,10 @@ class BaselineConfig:
data.get("tolerances", {}), profile_name
)
scenarios = {}
for name, cfg in data["scenarios"].items():
scenarios[name] = ScenarioConfig(
stages_ms=cfg["stages_ms"],
denoise_step_ms={int(k): v for k, v in cfg["denoise_step_ms"].items()},
expected_e2e_ms=float(cfg["expected_e2e_ms"]),
expected_avg_denoise_ms=float(cfg["expected_avg_denoise_ms"]),
expected_median_denoise_ms=float(cfg["expected_median_denoise_ms"]),
estimated_full_test_time_s=cfg.get("estimated_full_test_time_s"),
)
scenarios = {
name: ScenarioConfig.from_dict(cfg)
for name, cfg in data["scenarios"].items()
}
return cls(
scenarios=scenarios,
@@ -147,18 +174,12 @@ class BaselineConfig:
with path.open("r", encoding="utf-8") as fh:
data = json.load(fh)
scenarios_new = {}
for name, cfg in data["scenarios"].items():
scenarios_new[name] = ScenarioConfig(
stages_ms=cfg["stages_ms"],
denoise_step_ms={int(k): v for k, v in cfg["denoise_step_ms"].items()},
expected_e2e_ms=float(cfg["expected_e2e_ms"]),
expected_avg_denoise_ms=float(cfg["expected_avg_denoise_ms"]),
expected_median_denoise_ms=float(cfg["expected_median_denoise_ms"]),
estimated_full_test_time_s=cfg.get("estimated_full_test_time_s"),
)
self.scenarios.update(scenarios_new)
self.scenarios.update(
{
name: ScenarioConfig.from_dict(cfg)
for name, cfg in data["scenarios"].items()
}
)
return self
@@ -248,6 +269,7 @@ class DiffusionSamplingParams:
# output format
output_format: str | None = None # "png", "jpeg", "mp4", etc.
expect_audio_output: bool = False
num_outputs_per_prompt: int = 1
@@ -408,6 +430,8 @@ class PerformanceSummary:
step_metrics: list[float]
sampled_steps: dict[int, float]
all_denoise_steps: dict[int, float]
load_peak_vram_mb: float = 0.0
runtime_peak_vram_mb: float = 0.0
frames_per_second: float | None = None
total_frames: int | None = None
avg_frame_time_ms: float | None = None
@@ -437,6 +461,13 @@ class PerformanceSummary:
val = item.get("execution_time_ms", 0.0)
stage_metrics[item["name"]] = val
load_peak_vram_mb = float(
record.memory_snapshots.get("load_peak", {}).get("peak_reserved_mb", 0.0)
)
runtime_peak_vram_mb = float(
record.memory_snapshots.get("runtime_peak", {}).get("peak_reserved_mb", 0.0)
)
return PerformanceSummary(
e2e_ms=e2e_ms,
avg_denoise_ms=avg_denoise,
@@ -445,6 +476,8 @@ class PerformanceSummary:
step_metrics=step_durations,
sampled_steps=sampled_steps,
all_denoise_steps=per_step,
load_peak_vram_mb=load_peak_vram_mb,
runtime_peak_vram_mb=runtime_peak_vram_mb,
)
python/sglang/multimodal_gen/test/test_utils.py
+451 -1
View File
@@ -10,6 +10,7 @@ import subprocess
import sys
import tempfile
import time
import wave
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -39,7 +40,7 @@ logger = init_logger(__name__)
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
# publish.
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
SGL_TEST_FILES_CI_DATA_REVISION = "8c3896984319c8d5628bf08df4b596baf2368ec7"
SGL_TEST_FILES_CI_DATA_REVISION = "15b30030ef980756788ab40072f9223fe21a5526"
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
# when it's regenerated on its own cadence.
@@ -100,6 +101,11 @@ DEFAULT_MEAN_ABS_DIFF_THRESHOLD_IMAGE = 8.0
DEFAULT_SSIM_THRESHOLD_VIDEO = 0.92
DEFAULT_PSNR_THRESHOLD_VIDEO = 24.0
DEFAULT_MEAN_ABS_DIFF_THRESHOLD_VIDEO = 10.0
AUDIO_CONSISTENCY_SAMPLE_RATE = 16_000
DEFAULT_AUDIO_SPECTRAL_SIMILARITY_THRESHOLD = 0.95
DEFAULT_AUDIO_WAVEFORM_CORRELATION_THRESHOLD = 0.90
DEFAULT_AUDIO_RMS_DB_DIFF_THRESHOLD = 2.0
DEFAULT_AUDIO_DURATION_DIFF_THRESHOLD = 0.10
_clip_model_cache: dict[str, Any] = {}
_consistency_gt_cache: dict[str, Any] = {}
_official_consistency_gt_outputs_cache: dict[str, frozenset[str]] | None = None
@@ -721,6 +727,162 @@ def validate_video_file(
), f"Video height mismatch: expected {expected_height}, got {actual_height}"
@dataclass(frozen=True)
class AudioStreamInfo:
sample_rate: int
channels: int
duration_seconds: float
def probe_audio_stream(file_path: str) -> AudioStreamInfo:
"""Return metadata for the first audio stream in a media file."""
try:
result = subprocess.run(
[
"ffprobe",
"-v",
"error",
"-show_entries",
"stream=codec_type,sample_rate,channels,duration:format=duration",
"-of",
"json",
file_path,
],
check=True,
capture_output=True,
text=True,
)
except (FileNotFoundError, subprocess.CalledProcessError) as exc:
stderr = getattr(exc, "stderr", None)
raise AssertionError(
f"Unable to inspect audio stream in {file_path}: {stderr or exc}"
) from exc
payload = json.loads(result.stdout)
stream = next(
(
item
for item in payload.get("streams", [])
if item.get("codec_type") == "audio"
),
None,
)
assert stream is not None, f"Media file has no audio stream: {file_path}"
sample_rate = int(stream.get("sample_rate") or 0)
channels = int(stream.get("channels") or 0)
duration = float(
stream.get("duration") or payload.get("format", {}).get("duration") or 0.0
)
assert sample_rate > 0, f"Audio stream has invalid sample rate: {sample_rate}"
assert channels > 0, f"Audio stream has invalid channel count: {channels}"
assert (
math.isfinite(duration) and duration > 0
), f"Audio stream has invalid duration: {duration}"
return AudioStreamInfo(sample_rate, channels, duration)
def extract_audio_pcm(
file_path: str,
sample_rate: int = AUDIO_CONSISTENCY_SAMPLE_RATE,
) -> np.ndarray:
"""Decode the first audio stream as mono float32 PCM."""
try:
result = subprocess.run(
[
"ffmpeg",
"-v",
"error",
"-i",
file_path,
"-map",
"0:a:0",
"-vn",
"-ac",
"1",
"-ar",
str(sample_rate),
"-f",
"f32le",
"pipe:1",
],
check=True,
capture_output=True,
)
except (FileNotFoundError, subprocess.CalledProcessError) as exc:
stderr = getattr(exc, "stderr", b"")
if isinstance(stderr, bytes):
stderr = stderr.decode("utf-8", errors="replace")
raise AssertionError(
f"Unable to decode audio stream in {file_path}: {stderr or exc}"
) from exc
return np.frombuffer(result.stdout, dtype="<f4").copy()
def extract_audio_pcm_from_video_bytes(
video_bytes: bytes,
sample_rate: int = AUDIO_CONSISTENCY_SAMPLE_RATE,
) -> np.ndarray:
with tempfile.NamedTemporaryFile(suffix=".mp4") as tmp:
tmp.write(video_bytes)
tmp.flush()
return extract_audio_pcm(tmp.name, sample_rate)
def validate_audio_output(file_path: str) -> AudioStreamInfo:
"""Validate that a media file contains finite, non-silent audio."""
info = probe_audio_stream(file_path)
audio = extract_audio_pcm(file_path)
assert audio.size > 0, f"Decoded audio stream is empty: {file_path}"
assert np.isfinite(audio).all(), f"Decoded audio contains NaN or inf: {file_path}"
rms = float(np.sqrt(np.mean(np.square(audio, dtype=np.float64))))
assert rms > 1e-5, f"Decoded audio is silent or near-silent: rms={rms:.3e}"
return info
def encode_audio_gt_wav(
audio: np.ndarray,
sample_rate: int = AUDIO_CONSISTENCY_SAMPLE_RATE,
) -> bytes:
pcm = np.round(np.clip(audio, -1.0, 1.0) * 32767.0).astype("<i2")
output = io.BytesIO()
with wave.open(output, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(sample_rate)
wav.writeframes(pcm.tobytes())
return output.getvalue()
def save_audio_gt_artifact(
artifact_dir: str | None,
case_id: str,
num_gpus: int,
audio: np.ndarray,
) -> Path | None:
if not artifact_dir:
return None
path = Path(artifact_dir) / f"{case_id}_{num_gpus}gpu_audio.wav"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(encode_audio_gt_wav(audio))
return path
def decode_audio_gt_wav(
content: bytes,
expected_sample_rate: int = AUDIO_CONSISTENCY_SAMPLE_RATE,
) -> np.ndarray:
with wave.open(io.BytesIO(content), "rb") as wav:
assert wav.getnchannels() == 1, "Audio GT must be mono"
assert wav.getsampwidth() == 2, "Audio GT must use signed 16-bit PCM"
assert wav.getframerate() == expected_sample_rate, (
f"Audio GT sample rate must be {expected_sample_rate}, "
f"got {wav.getframerate()}"
)
pcm = np.frombuffer(wav.readframes(wav.getnframes()), dtype="<i2")
return pcm.astype(np.float32) / 32767.0
def _normalize_consistency_platform(platform: str) -> str:
normalized = platform.strip().lower().replace("_", "-")
normalized = normalized.replace("-", "")
@@ -797,6 +959,24 @@ class ConsistencyThresholds:
mean_abs_diff_threshold: float
@dataclass(frozen=True)
class AudioConsistencyThresholds:
spectral_similarity_threshold: float
waveform_correlation_threshold: float
rms_db_diff_threshold: float
duration_diff_threshold: float
@dataclass(frozen=True)
class AudioConsistencyResult:
passed: bool
spectral_similarity: float
waveform_correlation: float
rms_db_diff: float
duration_diff: float
thresholds: AudioConsistencyThresholds
def get_consistency_thresholds(
case_id: str,
is_video: bool,
@@ -850,6 +1030,130 @@ def get_consistency_thresholds(
)
def get_audio_consistency_thresholds(
case_id: str,
metadata: dict[str, Any] | None = None,
) -> AudioConsistencyThresholds:
if metadata is None:
metadata = _load_threshold_json()
case_meta = metadata.get("cases", {}).get(case_id, {})
return AudioConsistencyThresholds(
spectral_similarity_threshold=float(
case_meta.get(
"audio_spectral_similarity_threshold",
metadata.get(
"default_audio_spectral_similarity_threshold",
DEFAULT_AUDIO_SPECTRAL_SIMILARITY_THRESHOLD,
),
)
),
waveform_correlation_threshold=float(
case_meta.get(
"audio_waveform_correlation_threshold",
metadata.get(
"default_audio_waveform_correlation_threshold",
DEFAULT_AUDIO_WAVEFORM_CORRELATION_THRESHOLD,
),
)
),
rms_db_diff_threshold=float(
case_meta.get(
"audio_rms_db_diff_threshold",
metadata.get(
"default_audio_rms_db_diff_threshold",
DEFAULT_AUDIO_RMS_DB_DIFF_THRESHOLD,
),
)
),
duration_diff_threshold=float(
case_meta.get(
"audio_duration_diff_threshold",
metadata.get(
"default_audio_duration_diff_threshold",
DEFAULT_AUDIO_DURATION_DIFF_THRESHOLD,
),
)
),
)
def _audio_magnitude_spectrogram(audio: np.ndarray) -> np.ndarray:
frame_length = 512
hop_length = 160
if audio.size < frame_length:
audio = np.pad(audio, (0, frame_length - audio.size))
frame_count = 1 + (audio.size - frame_length) // hop_length
frames = np.lib.stride_tricks.sliding_window_view(audio, frame_length)[
: frame_count * hop_length : hop_length
]
window = np.hanning(frame_length).astype(np.float32)
magnitude = np.abs(np.fft.rfft(frames * window, axis=1))
return np.sqrt(magnitude).astype(np.float32, copy=False)
def compare_audio_with_gt(
output_audio: np.ndarray,
gt_audio: np.ndarray,
thresholds: AudioConsistencyThresholds,
sample_rate: int = AUDIO_CONSISTENCY_SAMPLE_RATE,
) -> AudioConsistencyResult:
output_audio = np.asarray(output_audio, dtype=np.float32).reshape(-1)
gt_audio = np.asarray(gt_audio, dtype=np.float32).reshape(-1)
if output_audio.size == 0 or gt_audio.size == 0:
raise ValueError("Audio consistency inputs must be non-empty")
if not np.isfinite(output_audio).all() or not np.isfinite(gt_audio).all():
raise ValueError("Audio consistency inputs must be finite")
duration_diff = abs(output_audio.size - gt_audio.size) / float(sample_rate)
sample_count = min(output_audio.size, gt_audio.size)
output = output_audio[:sample_count]
target = gt_audio[:sample_count]
output_centered = output - float(output.mean())
target_centered = target - float(target.mean())
correlation_denominator = float(
np.linalg.norm(output_centered) * np.linalg.norm(target_centered)
)
waveform_correlation = (
float(np.dot(output_centered, target_centered)) / correlation_denominator
if correlation_denominator > 0
else 0.0
)
output_spectrum = _audio_magnitude_spectrogram(output)
target_spectrum = _audio_magnitude_spectrogram(target)
spectral_denominator = float(
np.linalg.norm(output_spectrum) * np.linalg.norm(target_spectrum)
)
spectral_similarity = (
float(np.vdot(output_spectrum, target_spectrum)) / spectral_denominator
if spectral_denominator > 0
else 0.0
)
output_rms = float(np.sqrt(np.mean(np.square(output, dtype=np.float64))))
target_rms = float(np.sqrt(np.mean(np.square(target, dtype=np.float64))))
rms_db_diff = abs(
20.0 * math.log10(max(output_rms, 1e-12))
- 20.0 * math.log10(max(target_rms, 1e-12))
)
passed = (
spectral_similarity >= thresholds.spectral_similarity_threshold
and waveform_correlation >= thresholds.waveform_correlation_threshold
and rms_db_diff <= thresholds.rms_db_diff_threshold
and duration_diff <= thresholds.duration_diff_threshold
)
return AudioConsistencyResult(
passed=passed,
spectral_similarity=spectral_similarity,
waveform_correlation=waveform_correlation,
rms_db_diff=rms_db_diff,
duration_diff=duration_diff,
thresholds=thresholds,
)
def get_clip_threshold(
case: "DiffusionTestCase",
metadata: dict[str, Any] | None = None,
@@ -1115,6 +1419,30 @@ def get_action_consistency_gt_candidates(case_id: str, num_gpus: int) -> list[st
]
def _audio_consistency_gt_filenames(case_id: str, num_gpus: int) -> list[str]:
case_id = get_consistency_gt_case_id(case_id)
return [f"{case_id}_{num_gpus}gpu_audio.wav"]
def get_audio_consistency_gt_candidate_sets(
case_id: str,
num_gpus: int,
) -> list[list[str]]:
candidates = _audio_consistency_gt_filenames(case_id, num_gpus)
if _is_ascend_consistency_case(case_id) or current_platform.is_npu():
return [candidates]
platform = get_consistency_platform()
return [[f"{platform}/{candidate}" for candidate in candidates], candidates]
def get_audio_consistency_gt_candidates(case_id: str, num_gpus: int) -> list[str]:
return [
candidate
for candidate_set in get_audio_consistency_gt_candidate_sets(case_id, num_gpus)
for candidate in candidate_set
]
def get_consistency_gt_remote_files(
case_id: str, num_gpus: int, is_video: bool, output_format: str | None = None
) -> list[tuple[str, str]]:
@@ -1143,6 +1471,19 @@ def get_action_consistency_gt_remote_files(
]
def get_audio_consistency_gt_remote_files(
case_id: str, num_gpus: int
) -> list[tuple[str, str]]:
files = _find_remote_audio_consistency_gt_files(case_id, num_gpus)
if files:
return files
filenames = get_audio_consistency_gt_candidates(case_id, num_gpus)
return [
(filename, f"{SGL_TEST_FILES_CONSISTENCY_GT_BASE}/{filename}")
for filename in filenames
]
def _remote_consistency_gt_candidates(
base_url: str,
case_id: str,
@@ -1375,6 +1716,35 @@ def _find_remote_action_consistency_gt_files(
return []
def _find_remote_audio_consistency_gt_files(
case_id: str,
num_gpus: int,
) -> list[tuple[str, str]]:
for filenames in get_audio_consistency_gt_candidate_sets(case_id, num_gpus):
for base_url in _remote_consistency_gt_base_urls(case_id):
candidates = [
(filename, f"{base_url}/{filename}") for filename in filenames
]
if _is_official_consistency_gt_base_url(base_url):
candidates = [
(filename, url)
for filename, url in candidates
if _official_consistency_gt_candidate_is_declared(case_id, filename)
]
if not candidates:
continue
uncertain_candidate = None
for filename, url in candidates:
exists = _remote_file_exists(url)
if exists is True:
return [(filename, url)]
if exists is None and uncertain_candidate is None:
uncertain_candidate = (filename, url)
if uncertain_candidate is not None:
return [uncertain_candidate]
return []
def _get_consistency_gt_dir() -> Path | None:
"""Return the local GT directory when configured."""
d = os.environ.get("SGLANG_CONSISTENCY_GT_DIR")
@@ -1402,6 +1772,13 @@ def _get_action_consistency_gt_cache_key(case_id: str, num_gpus: int) -> str:
return f"{platform}:{case_id}:{num_gpus}:action:{source}"
def _get_audio_consistency_gt_cache_key(case_id: str, num_gpus: int) -> str:
gt_dir = _get_consistency_gt_dir()
source = str(gt_dir) if gt_dir is not None else "remote"
platform = get_consistency_platform()
return f"{platform}:{case_id}:{num_gpus}:audio:{source}"
def load_consistency_gt(
case_id: str,
num_gpus: int,
@@ -1534,6 +1911,62 @@ def load_action_consistency_gt(case_id: str, num_gpus: int) -> dict[str, Any]:
return loaded_gt
def _load_remote_gt_bytes(url: str) -> bytes:
last_error: Exception | None = None
for _ in range(3):
try:
resp = requests.get(url, timeout=60)
try:
if resp.status_code == 200:
return resp.content
last_error = FileNotFoundError(f"GT file not found: {url}")
if resp.status_code not in (403, 429) and resp.status_code < 500:
break
finally:
resp.close()
except requests.RequestException as exc:
last_error = exc
raise FileNotFoundError(f"GT file not found: {url}") from last_error
def load_audio_consistency_gt(case_id: str, num_gpus: int) -> np.ndarray:
cache_key = _get_audio_consistency_gt_cache_key(case_id, num_gpus)
cached = _consistency_gt_cache.get(cache_key)
if cached is not None:
return cached
gt_dir = _get_consistency_gt_dir()
if gt_dir is not None:
path = next(
(
gt_dir / filename
for filename in get_audio_consistency_gt_candidates(case_id, num_gpus)
if (gt_dir / filename).exists()
),
None,
)
if path is None:
candidates = get_audio_consistency_gt_candidates(case_id, num_gpus)
raise FileNotFoundError(
f"GT audio not found in {gt_dir}. Tried: {', '.join(candidates)}"
)
content = path.read_bytes()
logger.info("Loaded audio GT for %s from %s", case_id, path)
else:
remote_files = _find_remote_audio_consistency_gt_files(case_id, num_gpus)
if not remote_files:
candidates = get_audio_consistency_gt_candidates(case_id, num_gpus)
raise FileNotFoundError(
f"GT audio not found for {case_id}. Tried: {', '.join(candidates)}"
)
content = _load_remote_gt_bytes(remote_files[0][1])
logger.info("Loaded audio GT for %s from %s", case_id, remote_files[0][1])
loaded_gt = decode_audio_gt_wav(content)
_consistency_gt_cache[cache_key] = loaded_gt
return loaded_gt
def load_gt_embeddings(
case_id: str,
num_gpus: int,
@@ -1585,6 +2018,23 @@ def gt_exists(
return found
def audio_gt_exists(case_id: str, num_gpus: int) -> bool:
gt_dir = _get_consistency_gt_dir()
if gt_dir is not None:
return any(
(gt_dir / candidate).exists()
for candidate in get_audio_consistency_gt_candidates(case_id, num_gpus)
)
cache_key = _get_audio_consistency_gt_cache_key(case_id, num_gpus)
if cache_key in _gt_exists_remote_cache:
return True
found = bool(_find_remote_audio_consistency_gt_files(case_id, num_gpus))
if found:
_gt_exists_remote_cache.add(cache_key)
return found
def action_gt_exists(case_id: str, num_gpus: int) -> bool:
gt_dir = _get_consistency_gt_dir()
if gt_dir is not None:
python/sglang/multimodal_gen/test/unit/test_consistency_metrics.py
+105
View File
@@ -1,4 +1,5 @@
import math
from types import SimpleNamespace
import numpy as np
import pytest
@@ -625,3 +626,107 @@ def test_save_consistency_failure_artifact(tmp_path, monkeypatch):
assert (
tmp_path / "consistency_failures" / "generated" / "unit_image_fail_1gpu.png"
).exists()
def _sine_wave(frequency: float, seconds: float = 1.0) -> np.ndarray:
t = np.arange(
int(test_utils.AUDIO_CONSISTENCY_SAMPLE_RATE * seconds), dtype=np.float32
)
return np.sin(
2.0 * np.pi * frequency * t / float(test_utils.AUDIO_CONSISTENCY_SAMPLE_RATE)
).astype(np.float32)
def _audio_thresholds() -> test_utils.AudioConsistencyThresholds:
return test_utils.AudioConsistencyThresholds(
spectral_similarity_threshold=0.95,
waveform_correlation_threshold=0.90,
rms_db_diff_threshold=1.0,
duration_diff_threshold=0.05,
)
def test_audio_consistency_wav_round_trip_passes():
output = _sine_wave(440.0)
gt = test_utils.decode_audio_gt_wav(test_utils.encode_audio_gt_wav(output))
result = test_utils.compare_audio_with_gt(output, gt, _audio_thresholds())
assert result.passed
assert result.spectral_similarity > 0.999
assert result.waveform_correlation > 0.999
assert result.rms_db_diff < 0.001
assert result.duration_diff == 0.0
def test_save_audio_gt_artifact(tmp_path):
path = test_utils.save_audio_gt_artifact(
str(tmp_path), "unit_audio", 2, _sine_wave(440.0)
)
assert path == tmp_path / "unit_audio_2gpu_audio.wav"
assert test_utils.decode_audio_gt_wav(path.read_bytes()).size > 0
def test_audio_consistency_rejects_wrong_audio():
result = test_utils.compare_audio_with_gt(
_sine_wave(440.0),
_sine_wave(880.0),
_audio_thresholds(),
)
assert not result.passed
assert result.spectral_similarity < 0.95
assert result.waveform_correlation < 0.90
def test_audio_gt_candidates_prefer_platform_directory(monkeypatch):
monkeypatch.setenv(test_utils.CONSISTENCY_PLATFORM_ENV, "h100")
sglang_prefix = test_utils.SGL_TEST_FILES_SGLANG_CONSISTENCY_GT_BASE + "/"
monkeypatch.setattr(
test_utils,
"_remote_file_exists",
lambda url: url.startswith(sglang_prefix),
)
files = test_utils._find_remote_audio_consistency_gt_files("unit_audio", 2)
assert files == [
(
"h100/unit_audio_2gpu_audio.wav",
f"{sglang_prefix}h100/unit_audio_2gpu_audio.wav",
)
]
def test_probe_audio_stream_reads_positive_metadata(monkeypatch):
monkeypatch.setattr(
test_utils.subprocess,
"run",
lambda *args, **kwargs: SimpleNamespace(
stdout=(
'{"streams":[{"codec_type":"audio","sample_rate":"32000",'
'"channels":2,"duration":"4.0"}],"format":{"duration":"4.1"}}'
)
),
)
info = test_utils.probe_audio_stream("output.mp4")
assert info == test_utils.AudioStreamInfo(32000, 2, 4.0)
def test_validate_audio_output_rejects_silence(monkeypatch):
monkeypatch.setattr(
test_utils,
"probe_audio_stream",
lambda path: test_utils.AudioStreamInfo(32000, 2, 4.0),
)
monkeypatch.setattr(
test_utils,
"extract_audio_pcm",
lambda path: np.zeros(test_utils.AUDIO_CONSISTENCY_SAMPLE_RATE),
)
with pytest.raises(AssertionError, match="silent"):
test_utils.validate_audio_output("output.mp4")
python/sglang/multimodal_gen/test/unit/test_disagg_roles.py
+36
View File
@@ -1,7 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
"""Unit tests for disaggregation role-based module filtering."""
import json
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
@@ -244,6 +247,39 @@ class _FakeStage:
class TestPipelineStageRoleFilter(unittest.TestCase):
def test_load_config_passes_server_revision_to_model_download(self):
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = Path(tmp_dir)
(tmp_path / "model_index.json").write_text(
json.dumps(
{
"_class_name": "FakePipeline",
"_diffusers_version": "0.34.0",
"transformer": ["diffusers", "FakeTransformer"],
}
)
)
(tmp_path / "transformer").mkdir()
pipeline = object.__new__(_FakePipeline)
pipeline.model_path = "org/repo"
pipeline.server_args = SimpleNamespace(
model_subfolder=None,
revision="a" * 40,
)
with patch(
"sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base.maybe_download_model",
return_value=str(tmp_path),
) as download_model:
pipeline._load_config()
download_model.assert_called_once_with(
"org/repo",
force_diffusers_model=True,
revision="a" * 40,
)
def test_stage_factory_skips_without_constructing_for_other_role(self):
pipeline = _make_pipeline(RoleType.ENCODER)
python/sglang/multimodal_gen/test/unit/test_health_warmup_gate.py
+32
View File
@@ -134,6 +134,38 @@ class TestServerManagerReadiness(unittest.TestCase):
["http://127.0.0.1:11000/health"] * 2,
)
def test_start_cleans_up_process_when_readiness_fails(self):
manager = ServerManager("test-model", port=11000, wait_deadline=1)
process = SimpleNamespace(pid=123, stdout=None)
with tempfile.TemporaryDirectory() as temp_dir:
log_dir = Path(temp_dir)
with (
mock.patch(
"sglang.multimodal_gen.test.server.test_server_utils.prepare_perf_log",
return_value=(log_dir, log_dir / "perf.jsonl"),
),
mock.patch(
"sglang.multimodal_gen.test.server.test_server_utils.subprocess.Popen",
return_value=process,
),
mock.patch.object(
manager,
"_wait_for_ready",
side_effect=TimeoutError("startup timed out"),
),
mock.patch(
"sglang.multimodal_gen.test.server.test_server_utils.kill_process_tree"
) as kill_process,
mock.patch(
"sglang.multimodal_gen.test.server.test_server_utils.time.sleep"
),
):
with self.assertRaisesRegex(TimeoutError, "startup timed out"):
manager.start()
kill_process.assert_called_once_with(123)
if __name__ == "__main__":
unittest.main()
python/sglang/multimodal_gen/test/unit/test_hf_diffusers_utils.py
+25
View File
@@ -204,6 +204,31 @@ def test_complete_cached_snapshot_is_served_without_download(
assert calls == ["probe"]
def test_cached_snapshot_respects_requested_revision(monkeypatch, tmp_path):
calls = []
def fake_snapshot_download(**kwargs):
calls.append(kwargs)
return str(tmp_path)
monkeypatch.setattr(hf_diffusers_utils, "snapshot_download", fake_snapshot_download)
assert maybe_download_model("org/repo", revision="a" * 40, download=False) == str(
tmp_path
)
assert calls == [
{
"repo_id": "org/repo",
"ignore_patterns": ["*.onnx", "*.msgpack"],
"allow_patterns": None,
"local_dir": None,
"local_files_only": True,
"max_workers": 8,
"revision": "a" * 40,
}
]
def test_partially_populated_cached_snapshot_is_served_without_download(
recording_snapshot_download, tmp_path
):
python/sglang/multimodal_gen/test/unit/test_minimax_h3_media.py
+40
View File
@@ -11,6 +11,7 @@ import numpy as np
import pytest
import torch
from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3 import (
material_io,
reference_encoding,
@@ -256,3 +257,42 @@ def test_audio_decode_is_bounded_float_pcm_without_temp_files(monkeypatch):
assert ffmpeg[ffmpeg.index("-ss") + 1] == "2.25"
assert ffmpeg.index("-ss") < ffmpeg.index("-i")
assert ffmpeg[-3:] == ["-f", "f32le", "pipe:1"]
def test_reference_audio_encode_sets_forward_context(monkeypatch):
class FakeAudioVAE(torch.nn.Module):
attn_proj = True
def __init__(self):
super().__init__()
self.anchor = torch.nn.Parameter(torch.zeros(1))
self.mean_proj = torch.nn.Identity()
def preprocess(self, waveform, _sample_rate):
return waveform
def encoder(self, _audio_data):
return torch.ones(2, 32, 4)
def pre_block(self, hidden_states):
assert get_forward_context().current_timestep == 0
return hidden_states
monkeypatch.setattr(
reference_encoding,
"_load_waveform",
lambda *_args, **_kwargs: (torch.ones(2, 320), 32000),
)
result = reference_encoding.minimax_h3_encode_reference_audio_rows(
FakeAudioVAE(),
"/input/ref.wav",
SimpleNamespace(
latent_channels=32,
latents_mean=[0.0] * 32,
latents_std=[1.0] * 32,
),
)
assert result["rows"].shape == (8, 32)
assert result["ref_audio_t"] == 4
python/sglang/multimodal_gen/test/unit/test_performance_metrics.py
+235
View File
@@ -0,0 +1,235 @@
import json
from unittest.mock import Mock, patch
import pytest
import torch
import sglang.multimodal_gen.runtime.managers.gpu_worker as gpu_worker_module
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.perf_logger import (
MemorySnapshot,
RequestMetrics,
RequestPerfRecord,
)
from sglang.multimodal_gen.test.server.conftest import _write_results_json
from sglang.multimodal_gen.test.server.test_server_utils import PerformanceValidator
from sglang.multimodal_gen.test.server.testcase_configs import (
BaselineConfig,
PerformanceSummary,
ScenarioConfig,
ToleranceConfig,
)
def _perf_record(memory_snapshots: dict[str, dict]) -> RequestPerfRecord:
return RequestPerfRecord(
request_id="request",
commit_hash="commit",
tag="test",
stages=[],
steps=[],
total_duration_ms=1.0,
memory_snapshots=memory_snapshots,
)
def test_performance_summary_separates_load_and_runtime_peaks():
summary = PerformanceSummary.from_req_perf_record(
_perf_record(
{
"after_forward": {"peak_reserved_mb": 1024.0},
"load_peak": {"peak_reserved_mb": 4096.0},
"runtime_peak": {"peak_reserved_mb": 3072.0},
}
),
step_fractions=(),
)
assert summary.load_peak_vram_mb == 4096.0
assert summary.runtime_peak_vram_mb == 3072.0
def test_worker_records_replica_load_and_runtime_peaks():
worker = GPUWorker.__new__(GPUWorker)
worker.local_rank = 0
worker.is_output_rank = True
worker._load_peak_reserved_mb = 4096.0
worker._runtime_peak_reserved_mb = 0.0
output = OutputBatch()
metrics = RequestMetrics("request")
replica_group = Mock()
replica_group.all_reduce.return_value = torch.tensor(
[5120.0, 3584.0], dtype=torch.float64
)
snapshots = [
MemorySnapshot(0.0, 0.0, 2048.0, 3072.0),
MemorySnapshot(0.0, 0.0, 2048.0, 3072.0),
]
with (
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "get_device", return_value=torch.device("cpu")),
patch.object(
gpu_worker_module, "capture_memory_snapshot", side_effect=snapshots
),
patch.object(
gpu_worker_module, "get_replica_group", return_value=replica_group
),
):
worker._record_output_peak_memory(output)
worker._record_replica_peak_memory([metrics])
assert output.peak_memory_mb == 3072.0
assert worker._load_peak_reserved_mb == 4096.0
assert worker._runtime_peak_reserved_mb == 3072.0
assert metrics.memory_snapshots["load_peak"].peak_reserved_mb == 5120.0
assert metrics.memory_snapshots["runtime_peak"].peak_reserved_mb == 3584.0
def test_baseline_config_loads_per_scenario_peak_vram(tmp_path):
path = tmp_path / "baseline.json"
path.write_text(
json.dumps(
{
"tolerances": {
"pr_test": {
"e2e": 0.25,
"denoise_stage": 0.25,
"non_denoise_stage": 0.8,
"denoise_step": 0.3,
"denoise_agg": 0.2,
"load_peak_vram": 0.01,
"runtime_peak_vram": 0.02,
}
},
"sampling": {"step_fractions": [0.0, 1.0]},
"scenarios": {
"case": {
"stages_ms": {},
"denoise_step_ms": {},
"expected_e2e_ms": 1.0,
"expected_avg_denoise_ms": 1.0,
"expected_median_denoise_ms": 1.0,
"load_peak_vram_mb": 1234.5,
"runtime_peak_vram_mb": 2345.6,
}
},
}
),
encoding="utf-8",
)
config = BaselineConfig.load(path)
scenario = config.scenarios["case"]
assert scenario.load_peak_vram_mb == 1234.5
assert scenario.runtime_peak_vram_mb == 2345.6
assert config.tolerances.load_peak_vram == 0.01
assert config.tolerances.runtime_peak_vram == 0.02
def test_peak_vram_validation_uses_independent_tolerances():
validator = PerformanceValidator(
scenario=ScenarioConfig({}, {}, 0.0, 0.0, 0.0),
tolerances=ToleranceConfig(
0.0,
0.0,
0.0,
0.0,
0.0,
load_peak_vram=0.01,
runtime_peak_vram=0.02,
),
step_fractions=(),
)
load_regression = PerformanceSummary(
0.0,
0.0,
0.0,
{},
[],
{},
{},
load_peak_vram_mb=10_129.0,
runtime_peak_vram_mb=10_000.0,
)
runtime_regression = PerformanceSummary(
0.0,
0.0,
0.0,
{},
[],
{},
{},
load_peak_vram_mb=10_000.0,
runtime_peak_vram_mb=10_201.0,
)
with patch.object(current_platform, "is_hip", return_value=False):
with pytest.raises(AssertionError, match="Load Peak VRAM"):
validator.validate_peak_vram(load_regression, 10_000.0, 10_000.0)
with pytest.raises(AssertionError, match="Runtime Peak VRAM"):
validator.validate_peak_vram(runtime_regression, 10_000.0, 10_000.0)
@pytest.mark.parametrize(
("load_peak_vram_mb", "runtime_peak_vram_mb", "message"),
[
(0.0, 10_000.0, "Load peak VRAM metric missing"),
(10_000.0, 0.0, "Runtime peak VRAM metric missing"),
],
)
def test_peak_vram_validation_rejects_missing_metrics(
load_peak_vram_mb, runtime_peak_vram_mb, message
):
validator = PerformanceValidator(
scenario=ScenarioConfig({}, {}, 0.0, 0.0, 0.0),
tolerances=ToleranceConfig(0.0, 0.0, 0.0, 0.0, 0.0),
step_fractions=(),
)
summary = PerformanceSummary(
0.0,
0.0,
0.0,
{},
[],
{},
{},
load_peak_vram_mb=load_peak_vram_mb,
runtime_peak_vram_mb=runtime_peak_vram_mb,
)
with pytest.raises(AssertionError, match=message):
validator.validate_peak_vram(summary, 10_000.0, 10_000.0)
def test_results_json_merges_retry_sessions(tmp_path):
path = tmp_path / "diffusion-results.json"
_write_results_json(
[
{
"class_name": "Suite",
"test_name": "first",
"load_peak_vram_mb": 1.0,
"runtime_peak_vram_mb": 2.0,
}
],
str(path),
)
_write_results_json(
[
{
"class_name": "Suite",
"test_name": "second",
"load_peak_vram_mb": 3.0,
"runtime_peak_vram_mb": 4.0,
}
],
str(path),
)
results = json.loads(path.read_text(encoding="utf-8"))
assert {item["test_name"] for item in results} == {"first", "second"}