diff --git a/.github/workflows/diffusion-ci-gt-gen.yml b/.github/workflows/diffusion-ci-gt-gen.yml index 28ed6305f..6bdcf8dad 100644 --- a/.github/workflows/diffusion-ci-gt-gen.yml +++ b/.github/workflows/diffusion-ci-gt-gen.yml @@ -364,7 +364,8 @@ jobs: --target-time 1800 \ --max-time 2400 \ --max-partitions 10 \ - --parametrized-only + --parametrized-only \ + ${{ inputs.case_ids != '' && format('--case-ids {0}', inputs.case_ids) || '' }} multimodal-diffusion-gen-1gpu: needs: compute-diffusion-partitions @@ -480,6 +481,8 @@ jobs: - name: Generate outputs env: + HF_TOKEN: ${{ secrets.MINIMAX_H3_HF_TOKEN || secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} + HUGGING_FACE_HUB_TOKEN: ${{ secrets.MINIMAX_H3_HF_TOKEN || secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} RUNAI_STREAMER_MEMORY_LIMIT: 0 PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-2gpu }} run: | diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index c70ba0e12..c91356a30 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -301,6 +301,8 @@ jobs: - name: Run diffusion server tests timeout-minutes: 240 env: + HF_TOKEN: ${{ secrets.MINIMAX_H3_HF_TOKEN || secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} + HUGGING_FACE_HUB_TOKEN: ${{ secrets.MINIMAX_H3_HF_TOKEN || secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }} RUNAI_STREAMER_MEMORY_LIMIT: 0 CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error == 'true' && '--continue-on-error' || '' }} PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-2gpu }} @@ -435,55 +437,6 @@ jobs: - uses: ./.github/actions/upload-cuda-coredumps if: failure() - multimodal-gen-test-4-h100: - # Temporarily disabled while the 4-gpu-h100 runner is unstable - if: ${{ false }} - runs-on: 4-gpu-h100 - timeout-minutes: 90 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-pr-test-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: inputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: python/sglang/kernels/aot/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - - - name: Run MiniMax-H3 PR smoke test - timeout-minutes: 45 - env: - RUNAI_STREAMER_MEMORY_LIMIT: 0 - SGLANG_TEST_WAIT_SECS: 1800 - SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures - run: | - cd python - python3 sglang/multimodal_gen/test/run_suite.py --suite 4-gpu-h100 - - - name: Upload diffusion failure artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: diffusion-failures-${{ github.job }}-${{ github.run_attempt }} - path: diffusion-failures/ - if-no-files-found: ignore - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - multimodal-gen-unit-test: if: | ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) && diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index f79e7c54b..b79e3f816 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -658,6 +658,61 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [ ] TWO_GPU_CASES = [ + DiffusionTestCase( + "minimax_h3_t2va_2gpu_h100", + DiffusionServerArgs( + model_path="MiniMaxAI/MiniMax-H3", + modality="video", + tp_size=2, + ulysses_degree=1, + extras=[ + "--model-variant", + "fl2va", + "--performance-mode", + "memory", + "--layerwise-offload-components", + "dit,text_encoder,vae", + "--dit-offload-prefetch-size", + "1", + "--dit-layerwise-resident-layers", + "20", + "--enable-torch-compile", + "false", + ], + ), + DiffusionSamplingParams( + prompt=( + "A static night view of a narrow London alley in soft rain, wet " + "pavement reflecting a yellow streetlamp, the blue K. West sign " + "glowing above a doorway, cardboard boxes near the wall, a pale " + "parked car in the distance, and a slender glam-rock figure " + "holding a guitar under the lamp, brick storefronts, muted teal " + "and amber colors, subtle rain shimmer only." + ), + output_size="1344x768", + seconds=4, + output_format="mp4", + num_outputs_per_prompt=1, + extras={ + "task": "t2va", + "conditions": [], + "target": { + "short_edge": 768, + "aspect_ratio": "16:9", + "duration_seconds": 4.0, + }, + "num_inference_steps": 8, + "flow_shift": 12.0, + "audio_flow_shift": 3.0, + "seed": 42, + }, + ), + run_perf_check=True, + run_consistency_check=True, + run_component_accuracy_check=False, + run_models_api_check=False, + run_t2v_input_reference_check=False, + ), DiffusionTestCase( "flux2_modelopt_fp8_tp2_t2i", DiffusionServerArgs( diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index 4bb6422b4..09f28b7a1 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -2790,6 +2790,33 @@ "expected_median_denoise_ms": 0.0, "estimated_full_test_time_s": 55.4 }, + "minimax_h3_t2va_2gpu_h100": { + "stages_ms": { + "InputValidationStage": 0.05, + "MiniMaxH3PartitionAdmissionStage": 0.03, + "MiniMaxH3TextEncodingStage": 620.13, + "MiniMaxH3VisualEncodingStage": 0.11, + "MiniMaxH3AudioEncodingStage": 0.03, + "MiniMaxH3LatentPreparationStage": 18.72, + "MiniMaxH3TimestepPreparationStage": 0.34, + "MiniMaxH3DenoisingStage": 16839.0, + "MiniMaxH3DecodingStage": 29741.66, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 211.7, + "1": 2427.55, + "2": 2344.0, + "3": 2332.99, + "4": 2332.69, + "5": 2343.56, + "6": 2326.79 + }, + "expected_e2e_ms": 47538.57, + "expected_avg_denoise_ms": 2045.61, + "expected_median_denoise_ms": 2332.99, + "estimated_full_test_time_s": 235.9 + }, "mova_360p_tp2": { "stages_ms": {}, "denoise_step_ms": {}, diff --git a/python/sglang/multimodal_gen/test/server/test_server_common.py b/python/sglang/multimodal_gen/test/server/test_server_common.py index a707a2cde..d2dbe9225 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_common.py +++ b/python/sglang/multimodal_gen/test/server/test_server_common.py @@ -64,6 +64,7 @@ from sglang.multimodal_gen.test.test_utils import ( load_action_consistency_gt, load_consistency_gt, save_consistency_failure_artifact, + save_missing_consistency_gt_artifact, wait_for_req_perf_record, ) @@ -610,6 +611,24 @@ class DiffusionServerBase: if not gt_exists( case.id, num_gpus, is_video=is_video, output_format=output_format ): + if is_video: + output_frames = pop_realtime_key_frames(case.id) + if output_frames is None: + output_frames = extract_key_frames_from_video(content) + else: + output_frames = [image_bytes_to_numpy(content)] + artifact_path = save_missing_consistency_gt_artifact( + artifact_dir=os.environ.get("SGLANG_DIFFUSION_ARTIFACT_DIR"), + case_id=case.id, + num_gpus=num_gpus, + output_frames=output_frames, + is_video=is_video, + output_format=output_format, + ) + if artifact_path is not None: + logger.info( + "[Artifact] Saved missing consistency GT: %s", artifact_path + ) if _get_consistency_gt_dir() is not None: names = ", ".join( get_consistency_gt_candidates( diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 09300734e..4ba87b886 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -39,12 +39,12 @@ logger = init_logger(__name__) # NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows # publish. SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion" -SGL_TEST_FILES_CI_DATA_REVISION = "6d689f4833e8c106ff0d853865f50496d1f2b453" +SGL_TEST_FILES_CI_DATA_REVISION = "739c6c9b7cb972149cc3472cf19fe4bb29cf15c3" # The NPU pin is kept as a separate branch so ascend GT can be bumped independently # when it's regenerated on its own cadence. if current_platform.is_npu(): - SGL_TEST_FILES_CI_DATA_REVISION = "6d689f4833e8c106ff0d853865f50496d1f2b453" + SGL_TEST_FILES_CI_DATA_REVISION = "d180ad38872dff3d1ad03e4610cffcda874d3eb8" SGL_TEST_FILES_CONSISTENCY_GT_ROOT = ( "https://raw.githubusercontent.com/" @@ -2003,6 +2003,30 @@ def _save_generated_artifact_images( return generated_files +def save_missing_consistency_gt_artifact( + artifact_dir: str | Path | None, + case_id: str, + num_gpus: int, + output_frames: list[np.ndarray], + is_video: bool, + output_format: str | None = None, +) -> Path | None: + if not artifact_dir: + return None + + out_dir = Path(artifact_dir) / "missing_consistency_gt" + out_dir.mkdir(parents=True, exist_ok=True) + filenames = _consistency_gt_filenames( + case_id, + num_gpus, + is_video=is_video, + output_format=output_format, + ) + for frame, filename in zip(output_frames, filenames): + Image.fromarray(_ensure_rgb_uint8_image(frame)).save(out_dir / filename) + return out_dir + + def _write_consistency_failure_index( out_dir: Path, records: list[dict[str, Any]], diff --git a/scripts/ci/utils/diffusion/compute_diffusion_partitions.py b/scripts/ci/utils/diffusion/compute_diffusion_partitions.py index 013635a05..bdfe2ac03 100755 --- a/scripts/ci/utils/diffusion/compute_diffusion_partitions.py +++ b/scripts/ci/utils/diffusion/compute_diffusion_partitions.py @@ -255,6 +255,11 @@ def main(): action="store_true", help="Only partition DiffusionTestCase parametrized cases.", ) + parser.add_argument( + "--case-ids", + nargs="*", + help="Only schedule partitions containing these parametrized case IDs.", + ) args = parser.parse_args() script_dir = Path(__file__).resolve().parent @@ -285,6 +290,19 @@ def main(): ) validate_suite_case_coverage(suites) + requested_case_ids = set(args.case_ids or []) + if requested_case_ids: + known_case_ids = { + case.case_id + for suite_name, suite_info in suites.items() + if suite_name in SUITE_OUTPUT_NAMES + for case in suite_info.cases + } + unknown_case_ids = sorted(requested_case_ids - known_case_ids) + if unknown_case_ids: + print(f"Error: Unknown case IDs: {' '.join(unknown_case_ids)}") + sys.exit(1) + print("=== Diffusion Partition Computation ===") print(f"Min partition time: {args.min_time}s ({args.min_time/60:.1f} min)") print(f"Target partition time: {args.target_time}s ({args.target_time/60:.1f} min)") @@ -316,7 +334,19 @@ def main(): ) output_name = SUITE_OUTPUT_NAMES[suite_name] - output_github_value(f"matrix-{output_name}", build_matrix(partition_count)) + matrix = build_matrix(partition_count) + if requested_case_ids: + matrix = { + "include": [ + {"part": idx} + for idx, partition in enumerate(partitions) + if any( + item.kind == "case" and item.item_id in requested_case_ids + for item in partition + ) + ] + } + output_github_value(f"matrix-{output_name}", matrix) output_github_scalar(f"partition-count-{output_name}", str(partition_count)) output_github_value( f"plan-{output_name}", build_partition_plan(suite_name, partitions)