diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index 243de3b24..c06f1eabb 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -136,7 +136,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: diffusion-report-1gpu-${{ matrix.part }} - path: python/sglang/multimodal_gen/test/execution_report_*.json + path: | + python/sglang/multimodal_gen/test/execution_report_*.json + python/diffusion-results.json retention-days: 1 - name: Upload diffusion failure artifacts @@ -203,7 +205,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: diffusion-5090-report-${{ github.run_attempt }} - path: python/sglang/multimodal_gen/test/execution_report_*.json + path: | + python/sglang/multimodal_gen/test/execution_report_*.json + python/diffusion-results.json retention-days: 7 - name: Upload diffusion failure artifacts @@ -342,7 +346,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: diffusion-report-2gpu-${{ matrix.part }} - path: python/sglang/multimodal_gen/test/execution_report_*.json + path: | + python/sglang/multimodal_gen/test/execution_report_*.json + python/diffusion-results.json retention-days: 1 - name: Upload diffusion failure artifacts @@ -463,6 +469,17 @@ jobs: path: diffusion-failures/ if-no-files-found: ignore + - name: Upload execution report + if: always() + uses: actions/upload-artifact@v4 + with: + name: diffusion-report-1gpu-b200-${{ github.run_attempt }} + path: | + python/sglang/multimodal_gen/test/execution_report_*.json + python/diffusion-results.json + if-no-files-found: ignore + retention-days: 7 + - uses: ./.github/actions/upload-cuda-coredumps if: failure() diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index b86440759..6cc1a1d31 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -8,7 +8,7 @@ import os import tempfile import time from contextlib import ExitStack -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Callable, Iterator, List, Union import numpy as np @@ -24,6 +24,7 @@ globally_suppress_loggers() from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.distributed import ( + get_replica_group, get_sp_group, get_tp_rank, get_tp_world_size, @@ -151,6 +152,12 @@ class GPUWorker(GPUWorkerPostTrainingMixin): self.pipeline: ComposedPipelineBase = None self.init_device_and_model() + self._load_peak_reserved_mb = ( + 0.0 + if current_platform.is_cpu() + else capture_memory_snapshot().peak_reserved_mb + ) + self._runtime_peak_reserved_mb = 0.0 self.sp_group = get_sp_group() self.sp_cpu_group = self.sp_group.cpu_group self.tp_group = get_tp_group() @@ -464,11 +471,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): output_batch = None forward_failed = False try: - if ( - self.is_output_rank - and not current_platform.is_cpu() - and not current_platform.is_mps() - ): + if not current_platform.is_cpu() and not current_platform.is_mps(): torch.get_device_module().reset_peak_memory_stats() start_time = ( @@ -510,7 +513,6 @@ class GPUWorker(GPUWorkerPostTrainingMixin): return result output_batch = self._to_output_batch(result) - self._record_output_peak_memory(output_batch) output_metrics = self._iter_output_metrics(output_batch) if self.is_output_rank and output_metrics and not current_platform.is_cpu(): @@ -518,6 +520,19 @@ class GPUWorker(GPUWorkerPostTrainingMixin): for metrics in output_metrics: metrics.record_memory_snapshot("after_forward", peak_snapshot) + duration_ms = (time.monotonic() - start_time) * 1000 + for metrics in output_metrics: + metrics.total_duration_ms = duration_ms + + self._materialize_output_transport(output_batch, req, save_output_paths) + self._record_output_peak_memory(output_batch) + + collect_perf = ( + req.perf_dump_path is not None or envs.SGLANG_DIFFUSION_STAGE_LOGGING + ) + if collect_perf and not req.is_warmup: + self._record_replica_peak_memory(output_metrics) + if ( self.is_output_rank and not req.suppress_logs @@ -526,12 +541,6 @@ class GPUWorker(GPUWorkerPostTrainingMixin): ): self.do_mem_analysis(output_batch) - duration_ms = (time.monotonic() - start_time) * 1000 - for metrics in output_metrics: - metrics.total_duration_ms = duration_ms - - self._materialize_output_transport(output_batch, req, save_output_paths) - if ( not current_platform.is_cpu() and output_batch.output is None @@ -690,9 +699,38 @@ class GPUWorker(GPUWorkerPostTrainingMixin): return np.asarray(materialized.frames) def _record_output_peak_memory(self, output_batch: OutputBatch) -> None: - if not self.is_output_rank or current_platform.is_cpu(): + if current_platform.is_cpu(): return - output_batch.peak_memory_mb = capture_memory_snapshot().peak_reserved_mb + peak_reserved_mb = capture_memory_snapshot().peak_reserved_mb + self._runtime_peak_reserved_mb = max( + self._runtime_peak_reserved_mb, peak_reserved_mb + ) + if self.is_output_rank: + output_batch.peak_memory_mb = peak_reserved_mb + + def _record_replica_peak_memory(self, output_metrics: list[Any]) -> None: + """Record replica-wide loading and runtime allocator peaks.""" + if not current_platform.is_cuda(): + return + + peaks = torch.tensor( + [self._load_peak_reserved_mb, self._runtime_peak_reserved_mb], + dtype=torch.float64, + device=current_platform.get_device(self.local_rank), + ) + peaks = get_replica_group().all_reduce(peaks, op=torch.distributed.ReduceOp.MAX) + if not self.is_output_rank: + return + + snapshot = capture_memory_snapshot() + load_peak_mb, runtime_peak_mb = peaks.tolist() + for metrics in output_metrics: + metrics.record_memory_snapshot( + "load_peak", replace(snapshot, peak_reserved_mb=load_peak_mb) + ) + metrics.record_memory_snapshot( + "runtime_peak", replace(snapshot, peak_reserved_mb=runtime_peak_mb) + ) def _forward_group(self, batch: list[Req]) -> OutputBatch: assert self.pipeline is not None diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py b/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py index 0aa5eb792..fd03627ae 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/composed_pipeline_base.py @@ -181,7 +181,9 @@ class ComposedPipelineBase(ABC): if model_subfolder is None: model_path = maybe_download_model( - self.model_path, force_diffusers_model=True + self.model_path, + force_diffusers_model=True, + revision=self.server_args.revision, ) else: model_subfolder = os.path.normpath(model_subfolder) @@ -196,6 +198,7 @@ class ComposedPipelineBase(ABC): model_root = maybe_download_model( self.model_path, allow_patterns=[f"{model_subfolder}/**"], + revision=self.server_args.revision, ) model_path = os.path.join(model_root, model_subfolder) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/reference_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/reference_encoding.py index 22aa3a4ce..98f1f72e8 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/reference_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/reference_encoding.py @@ -35,6 +35,7 @@ from sglang.multimodal_gen.configs.models.vaes.minimax_h3_video import ( MiniMaxH3VideoVAEArchConfig, ) from sglang.multimodal_gen.runtime.distributed.parallel_state import get_world_group +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import ( MINIMAX_H3_SUPPORTED_FPS, ) @@ -321,7 +322,10 @@ def minimax_h3_encode_reference_audio_rows( waveform = _audio_resampler(int(source_rate))(waveform) waveform = waveform.to(device) - with _AudioVAEDeterminismContext(): + with ( + _AudioVAEDeterminismContext(), + set_forward_context(current_timestep=0, attn_metadata=None), + ): audio_data = model.preprocess( waveform.unsqueeze(1), MINIMAX_H3_AUDIO_SAMPLE_RATE ) diff --git a/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py b/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py index a7df28eb5..31ea6b5b4 100644 --- a/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py +++ b/python/sglang/multimodal_gen/runtime/postprocess/realesrgan_upscaler.py @@ -34,7 +34,10 @@ _DEFAULT_REALESRGAN_FILENAMES_BY_SCALE = { 4: "RealESRGAN_x4.pth", 8: "RealESRGAN_x8.pth", } -_LOW_MEMORY_TILED_UPSCALE_FREE_BYTES = 2 * 1024**3 +# RRDBNet full-frame 1024x1024 inference can request several GiB of cuDNN +# workspace beyond its output tensor. Keep enough headroom to avoid using an +# allocator OOM as the normal tiled-dispatch mechanism. +_FULL_FRAME_UPSCALE_MIN_FREE_BYTES = 12 * 1024**3 _REALESRGAN_TILE_SIZE = 256 _REALESRGAN_TILE_PAD = 32 @@ -343,7 +346,7 @@ class UpscalerModel: free_bytes, _ = torch.cuda.mem_get_info(self.device) output_bytes = h * w * self.scale * self.scale * 3 * 4 required_free_bytes = max( - _LOW_MEMORY_TILED_UPSCALE_FREE_BYTES, + _FULL_FRAME_UPSCALE_MIN_FREE_BYTES, output_bytes * 4, ) return free_bytes < required_free_bytes diff --git a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py index d1bd3988d..87c3bea05 100644 --- a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py @@ -837,6 +837,7 @@ def maybe_download_model( is_lora: bool = False, allow_patterns: list[str] | None = None, force_diffusers_model: bool = False, + revision: str | None = None, skip_overlay_resolution: bool = False, ) -> str: """ @@ -848,6 +849,7 @@ def maybe_download_model( download: Whether to download the model from Hugging Face Hub is_lora: If True, skip model completeness verification (LoRA models don't have transformer/vae directories) force_diffusers_model: If True, apply diffusers model check. Otherwise it should be a component model + revision: Specific Hugging Face Hub revision to resolve Returns: Local path to the model """ @@ -917,6 +919,7 @@ def maybe_download_model( local_dir=local_dir, local_files_only=True, max_workers=8, + revision=revision, ) if _is_revisionless_snapshot_root(local_path): # A cache miss, so the download below re-resolves and rewrites the ref. @@ -1009,6 +1012,7 @@ def maybe_download_model( allow_patterns=allow_patterns, local_dir=local_dir, max_workers=8, + revision=revision, ) if not force_diffusers_model: @@ -1027,6 +1031,7 @@ def maybe_download_model( local_dir=local_dir, max_workers=8, force_download=True, + revision=revision, ) if not _verify_diffusers_model_complete(local_path): raise ValueError( diff --git a/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py b/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py index 80b7e36bb..737328f4f 100644 --- a/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py +++ b/python/sglang/multimodal_gen/test/scripts/gen_perf_baselines.py @@ -141,7 +141,7 @@ def _run_case(case: DiffusionTestCase) -> dict: if "per_frame_generation" not in perf.stage_metrics: perf.stage_metrics["per_frame_generation"] = perf.e2e_ms / sp.num_frames - return { + baseline = { "stages_ms": {k: round(v, 2) for k, v in perf.stage_metrics.items()}, "denoise_step_ms": { str(k): round(v, 2) for k, v in perf.all_denoise_steps.items() @@ -150,6 +150,14 @@ def _run_case(case: DiffusionTestCase) -> dict: "expected_avg_denoise_ms": round(perf.avg_denoise_ms, 2), "expected_median_denoise_ms": round(perf.median_denoise_ms, 2), } + if current_platform.is_cuda(): + baseline.update( + { + "load_peak_vram_mb": round(perf.load_peak_vram_mb, 2), + "runtime_peak_vram_mb": round(perf.runtime_peak_vram_mb, 2), + } + ) + return baseline finally: ctx.cleanup() diff --git a/python/sglang/multimodal_gen/test/server/conftest.py b/python/sglang/multimodal_gen/test/server/conftest.py index 40fedbc35..c2aa729de 100644 --- a/python/sglang/multimodal_gen/test/server/conftest.py +++ b/python/sglang/multimodal_gen/test/server/conftest.py @@ -43,10 +43,30 @@ def _write_github_step_summary(content: str): def _write_results_json(results: list, output_path: str = "diffusion-results.json"): """Write performance results to JSON file for CI artifact collection.""" try: + existing = [] + if os.path.exists(output_path): + try: + with open(output_path, encoding="utf-8") as f: + loaded = json.load(f) + if isinstance(loaded, list): + existing = loaded + except json.JSONDecodeError: + pass + + merged = { + (entry.get("class_name"), entry.get("test_name")): entry + for entry in existing + } + merged.update( + { + (entry.get("class_name"), entry.get("test_name")): entry + for entry in results + } + ) with open(output_path, "w") as f: - json.dump(results, f, indent=2) + json.dump(list(merged.values()), f, indent=2) print(f"[CONFTEST] Wrote results to {output_path}") - except Exception as e: + except (json.JSONDecodeError, OSError) as e: print(f"[CONFTEST] Failed to write results JSON: {e}") @@ -63,15 +83,17 @@ def _generate_diffusion_markdown_report(results: list) -> str: # Main performance table markdown = header - markdown += "| Test Suite | Test Name | Modality | E2E (ms) | Avg Denoise (ms) | Median Denoise (ms) |\n" - markdown += "| ---------- | --------- | -------- | -------- | ---------------- | ------------------- |\n" + markdown += "| Test Suite | Test Name | Modality | E2E (ms) | Avg Denoise (ms) | Median Denoise (ms) | Load Peak VRAM (MiB) | Runtime Peak VRAM (MiB) |\n" + markdown += "| ---------- | --------- | -------- | -------- | ---------------- | ------------------- | -------------------- | ----------------------- |\n" for entry in sorted(results, key=lambda x: (x["class_name"], x["test_name"])): modality = entry.get("modality", "image") markdown += ( f"| {entry['class_name']} | {entry['test_name']} | {modality} | " f"{entry['e2e_ms']:.2f} | {entry['avg_denoise_ms']:.2f} | " - f"{entry['median_denoise_ms']:.2f} |\n" + f"{entry['median_denoise_ms']:.2f} | " + f"{entry.get('load_peak_vram_mb', 0):.0f} | " + f"{entry.get('runtime_peak_vram_mb', 0):.0f} |\n" ) # Video-specific metrics table (if any video tests) @@ -111,7 +133,7 @@ def pytest_sessionfinish(session): # Print to stdout (existing behavior) print("\n\n" + "=" * 35 + " Performance Summary " + "=" * 35) print( - f"{'Test Suite':<30} | {'Test Name':<20} | {'E2E (ms)':>12} | {'Avg Denoise (ms)':>18} | {'Median Denoise (ms)':>20}" + f"{'Test Suite':<30} | {'Test Name':<20} | {'E2E (ms)':>12} | {'Avg Denoise (ms)':>18} | {'Median Denoise (ms)':>20} | {'Load Peak (MiB)':>15} | {'Runtime Peak (MiB)':>18}" ) print( "-" * 30 @@ -123,15 +145,21 @@ def pytest_sessionfinish(session): + "-" * 18 + "-+-" + "-" * 20 + + "-+-" + + "-" * 15 + + "-+-" + + "-" * 18 ) for entry in sorted_results: print( f"{entry['class_name']:<30} | {entry['test_name']:<20} | {entry['e2e_ms']:>12.2f} | " - f"{entry['avg_denoise_ms']:>18.2f} | {entry['median_denoise_ms']:>20.2f}" + f"{entry['avg_denoise_ms']:>18.2f} | {entry['median_denoise_ms']:>20.2f} | " + f"{entry.get('load_peak_vram_mb', 0):>15.0f} | " + f"{entry.get('runtime_peak_vram_mb', 0):>18.0f}" ) - print("=" * 91) + print("=" * 130) print("\n\n" + "=" * 36 + " Detailed Reports " + "=" * 37) for entry in sorted_results: diff --git a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json index 0f349b55c..df68fcdff 100644 --- a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json +++ b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json @@ -279,5 +279,9 @@ "default_mean_abs_diff_threshold_image": 8.0, "default_ssim_threshold_video": 0.92, "default_psnr_threshold_video": 24.0, - "default_mean_abs_diff_threshold_video": 10.0 + "default_mean_abs_diff_threshold_video": 10.0, + "default_audio_spectral_similarity_threshold": 0.95, + "default_audio_waveform_correlation_threshold": 0.90, + "default_audio_rms_db_diff_threshold": 2.0, + "default_audio_duration_diff_threshold": 0.1 } diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 59d2f0afd..30d9726a9 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -489,6 +489,9 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [ "--pipeline-class-name LingBotWorldCausalDMDPipeline --warmup-mode off" ], text_encoder_cpu_offload=True, + env_vars={ + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + }, ), REALTIME_MODEL_sampling_params, run_component_accuracy_check=False, @@ -652,6 +655,7 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [ output_size="1344x768", seconds=5, output_format="mp4", + expect_audio_output=True, num_outputs_per_prompt=1, extras={ "task": "fl2va", @@ -683,7 +687,7 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [ run_component_accuracy_check=False, run_models_api_check=False, run_t2v_input_reference_check=False, - ) + ), ] TWO_GPU_CASES = [ @@ -723,6 +727,7 @@ TWO_GPU_CASES = [ output_size="1344x768", seconds=4, output_format="mp4", + expect_audio_output=True, num_outputs_per_prompt=1, extras={ "task": "t2va", @@ -744,6 +749,73 @@ TWO_GPU_CASES = [ run_models_api_check=False, run_t2v_input_reference_check=False, ), + DiffusionTestCase( + "minimax_h3_ref2va_video_audio_2gpu_h100", + DiffusionServerArgs( + model_path="MiniMaxAI/MiniMax-H3", + modality="video", + tp_size=2, + ulysses_degree=1, + extras=[ + "--model-variant", + "ref2va", + "--revision", + "42ed227ee7df40d41602854ae760620d6eb651fe", + "--performance-mode", + "memory", + "--layerwise-offload-components", + "dit,text_encoder", + "--component-residency", + "vae=resident", + "--dit-offload-prefetch-size", + "1", + "--dit-layerwise-resident-layers", + "20", + "--enable-torch-compile", + "false", + ], + ), + DiffusionSamplingParams( + prompt=( + "Follow the motion and appearance of