name: PR Test - Multimodal Gen

on:
  workflow_call:
    inputs:
      multimodal_gen:
        required: true
        type: string
      sgl_kernel:
        required: true
        type: string
      runner_config:
        required: true
        type: string
      runs_on_map:
        required: true
        type: string
      continue_on_error:
        required: false
        type: string
        default: 'false'
      git_ref:
        required: false
        type: string
        default: ''
      test_parallel_dispatch:
        required: false
        type: string
        default: 'false'
      caller_needs_failure:
        required: false
        type: string
        default: 'false'
      skip_pr_test_health_check:
        required: false
        type: string
        default: 'false'

# Workflow-level env is NOT inherited from the caller in reusable workflows.
# The github context (including github.event_name) IS inherited from the caller.
env:
  SGLANG_IS_IN_CI: true
  NCCL_NVLS_ENABLE: "0"
  SGLANG_CUDA_COREDUMP: "1"
  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
  SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == 'true' }}

jobs:
  compute-diffusion-partitions:
    if: inputs.multimodal_gen == 'true'
    runs-on: ubuntu-latest
    outputs:
      matrix-1gpu: ${{ steps.compute.outputs.matrix-1gpu }}
      matrix-2gpu: ${{ steps.compute.outputs.matrix-2gpu }}
      partition-count-1gpu: ${{ steps.compute.outputs['partition-count-1gpu'] }}
      partition-count-2gpu: ${{ steps.compute.outputs['partition-count-2gpu'] }}
      plan-1gpu: ${{ steps.compute.outputs.plan-1gpu }}
      plan-2gpu: ${{ steps.compute.outputs.plan-2gpu }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git_ref || github.sha }}

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Compute partitions
        id: compute
        run: |
          python scripts/ci/utils/diffusion/compute_diffusion_partitions.py --min-time 1200 --target-time 1800 --max-time 2400 --max-partitions 10

  multimodal-gen-test-1-gpu:
    needs: compute-diffusion-partitions
    if: |
      always() &&
      needs.compute-diffusion-partitions.result == 'success' &&
      needs.compute-diffusion-partitions.outputs.matrix-1gpu != '{"include":[]}' &&
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
      inputs.multimodal_gen == 'true'
    runs-on: 1-gpu-h100
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-1gpu) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git_ref || github.sha }}

      - uses: ./.github/actions/check-pr-test-health

      - uses: ./.github/actions/check-maintenance

      - name: Download artifacts
        if: inputs.sgl_kernel == 'true'
        uses: actions/download-artifact@v4
        with:
          path: python/sglang/kernels/aot/dist/
          merge-multiple: true
          pattern: wheel-python3.10-cuda*

      - name: Install dependencies
        timeout-minutes: 20
        run: |
          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion
      - name: Run diffusion server tests
        timeout-minutes: 240
        env:
          RUNAI_STREAMER_MEMORY_LIMIT: 0
          DIFFUSION_CONTINUE_ON_ERROR: ${{ inputs.continue_on_error }}
          DIFFUSION_PARTITION_ID: ${{ matrix.part }}
          DIFFUSION_TOTAL_PARTITIONS: ${{ needs.compute-diffusion-partitions.outputs['partition-count-1gpu'] }}
          DIFFUSION_PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-1gpu }}
          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
        run: |
          python3 test/run_suite.py \
            --hw cuda \
            --suite base-b-test-diffusion-1-gpu-h100 \
            --timeout-per-file 14400

      - name: Upload execution report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: diffusion-report-1gpu-${{ matrix.part }}
          path: |
            python/sglang/multimodal_gen/test/execution_report_*.json
            python/diffusion-results.json
          retention-days: 7

      - name: Upload diffusion failure artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: diffusion-failures-1gpu-${{ matrix.part }}-${{ github.run_attempt }}
          path: diffusion-failures/
          if-no-files-found: ignore
          retention-days: 7

      - uses: ./.github/actions/upload-cuda-coredumps
        if: failure()
        with:
          artifact-suffix: ${{ matrix.part }}

  multimodal-gen-test-1-5090:
    if: |
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
      inputs.multimodal_gen == 'true'
    runs-on: 1-gpu-5090
    timeout-minutes: 180
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git_ref || github.sha }}

      - uses: ./.github/actions/check-pr-test-health

      - uses: ./.github/actions/check-maintenance

      - name: Download artifacts
        if: inputs.sgl_kernel == 'true'
        uses: actions/download-artifact@v4
        with:
          path: python/sglang/kernels/aot/dist/
          merge-multiple: true
          pattern: wheel-python3.10-cuda*

      - name: Install dependencies
        timeout-minutes: 20
        run: |
          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion

      - name: Run 5090 diffusion canary tests
        timeout-minutes: 120
        env:
          RUNAI_STREAMER_MEMORY_LIMIT: 0
          DIFFUSION_CONTINUE_ON_ERROR: ${{ inputs.continue_on_error }}
          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
        run: |
          python3 test/run_suite.py \
            --hw cuda \
            --suite base-b-test-diffusion-1-gpu-5090 \
            --timeout-per-file 7200

      - name: Upload execution report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: diffusion-5090-report-${{ github.run_attempt }}
          path: |
            python/sglang/multimodal_gen/test/execution_report_*.json
            python/diffusion-results.json
          retention-days: 7

      - name: Upload diffusion failure artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: diffusion-failures-5090-${{ github.run_attempt }}
          path: diffusion-failures/
          if-no-files-found: ignore
          retention-days: 7

      - uses: ./.github/actions/upload-cuda-coredumps
        if: failure()
        with:
          artifact-suffix: 1-gpu-5090

  bcg-diffusion:
    if: |
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
      inputs.multimodal_gen == 'true'
    runs-on: 1-gpu-h100
    timeout-minutes: 90
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git_ref || github.sha }}

      - uses: ./.github/actions/check-pr-test-health

      - uses: ./.github/actions/check-maintenance

      - name: Download artifacts
        if: inputs.sgl_kernel == 'true'
        uses: actions/download-artifact@v4
        with:
          path: python/sglang/kernels/aot/dist/
          merge-multiple: true
          pattern: wheel-python3.10-cuda*

      - name: Install dependencies
        timeout-minutes: 20
        run: |
          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion

      - name: Run BCG diffusion tests
        timeout-minutes: 60
        env:
          RUNAI_STREAMER_MEMORY_LIMIT: 0
          DIFFUSION_CONTINUE_ON_ERROR: ${{ inputs.continue_on_error }}
          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-bcg-artifacts
        run: |
          python3 test/run_suite.py \
            --hw cuda \
            --suite base-b-test-diffusion-bcg-1-gpu-h100 \
            --timeout-per-file 3600

      - name: Upload BCG diffusion artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: diffusion-bcg-artifacts-${{ github.run_attempt }}
          path: diffusion-bcg-artifacts/
          if-no-files-found: ignore
          retention-days: 7

      - uses: ./.github/actions/upload-cuda-coredumps
        if: failure()
        with:
          artifact-suffix: bcg-diffusion

  multimodal-gen-test-2-gpu:
    needs: compute-diffusion-partitions
    if: |
      always() &&
      needs.compute-diffusion-partitions.result == 'success' &&
      needs.compute-diffusion-partitions.outputs.matrix-2gpu != '{"include":[]}' &&
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
      inputs.multimodal_gen == 'true'
    runs-on: 2-gpu-h100
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.compute-diffusion-partitions.outputs.matrix-2gpu) }}
    steps:
      - name: Record retry deadline
        run: echo "SGLANG_DIFFUSION_RETRY_DEADLINE=$(( $(date +%s) + 2400 ))" >> "$GITHUB_ENV"

      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git_ref || github.sha }}

      - uses: ./.github/actions/check-pr-test-health

      - uses: ./.github/actions/check-maintenance

      - name: Download artifacts
        if: inputs.sgl_kernel == 'true'
        uses: actions/download-artifact@v4
        with:
          path: python/sglang/kernels/aot/dist/
          merge-multiple: true
          pattern: wheel-python3.10-cuda*

      - name: Install dependencies
        timeout-minutes: 20
        run: |
          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion

      - name: Run diffusion server tests
        timeout-minutes: 45
        env:
          HF_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }}
          HUGGING_FACE_HUB_TOKEN: ${{ secrets.SGLANG_DIFFUSION_CI_HF_TOKEN || secrets.HF_TOKEN }}
          RUNAI_STREAMER_MEMORY_LIMIT: 0
          DIFFUSION_CONTINUE_ON_ERROR: ${{ inputs.continue_on_error }}
          DIFFUSION_PARTITION_ID: ${{ matrix.part }}
          DIFFUSION_TOTAL_PARTITIONS: ${{ needs.compute-diffusion-partitions.outputs['partition-count-2gpu'] }}
          DIFFUSION_PARTITION_PLAN_JSON: ${{ needs.compute-diffusion-partitions.outputs.plan-2gpu }}
          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
        run: |
          python3 test/run_suite.py \
            --hw cuda \
            --suite base-b-test-diffusion-2-gpu-h100 \
            --timeout-per-file 14400

      - name: Upload execution report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: diffusion-report-2gpu-${{ matrix.part }}
          path: |
            python/sglang/multimodal_gen/test/execution_report_*.json
            python/diffusion-results.json
          retention-days: 7

      - name: Upload diffusion failure artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: diffusion-failures-2gpu-${{ matrix.part }}-${{ github.run_attempt }}
          path: diffusion-failures/
          if-no-files-found: ignore
          retention-days: 7

      - uses: ./.github/actions/upload-cuda-coredumps
        if: failure()
        with:
          artifact-suffix: ${{ matrix.part }}

  multimodal-gen-component-accuracy:
    if: |
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
      inputs.multimodal_gen == 'true'
    runs-on: 2-gpu-h100
    timeout-minutes: 240
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git_ref || github.sha }}

      - uses: ./.github/actions/check-pr-test-health

      - uses: ./.github/actions/check-maintenance

      - name: Download artifacts
        if: inputs.sgl_kernel == 'true'
        uses: actions/download-artifact@v4
        with:
          path: python/sglang/kernels/aot/dist/
          merge-multiple: true
          pattern: wheel-python3.10-cuda*

      - name: Install dependencies
        timeout-minutes: 20
        run: |
          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion

      - name: Run diffusion component accuracy tests
        timeout-minutes: 240
        env:
          RUNAI_STREAMER_MEMORY_LIMIT: 0
          DIFFUSION_CONTINUE_ON_ERROR: ${{ inputs.continue_on_error }}
        run: |
          python3 test/run_suite.py \
            --hw cuda \
            --suite base-b-test-diffusion-component-2-gpu-h100 \
            --timeout-per-file 14400

      - uses: ./.github/actions/upload-cuda-coredumps
        if: always()
        with:
          artifact-suffix: component-accuracy

  multimodal-gen-test-1-b200:
    if: |
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
      inputs.multimodal_gen == 'true'
    runs-on: ${{ fromJson(inputs.runs_on_map)[inputs.runner_config] }}
    timeout-minutes: 240
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git_ref || github.sha }}

      - uses: ./.github/actions/check-pr-test-health

      - uses: ./.github/actions/check-maintenance

      - name: Download artifacts
        if: inputs.sgl_kernel == 'true'
        uses: actions/download-artifact@v4
        with:
          path: python/sglang/kernels/aot/dist/
          merge-multiple: true
          pattern: wheel-python3.10-cuda*

      - name: Install dependencies
        timeout-minutes: 20
        env:
          SGLANG_CI_EARLY_LD_LIBRARY_PATH: "1"
        run: |
          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion

      - name: Run diffusion server tests
        timeout-minutes: 240
        env:
          RUNAI_STREAMER_MEMORY_LIMIT: 0
          DIFFUSION_CONTINUE_ON_ERROR: ${{ inputs.continue_on_error }}
          SGLANG_DIFFUSION_ARTIFACT_DIR: ${{ github.workspace }}/diffusion-failures
        run: |
          python3 test/run_suite.py \
            --hw cuda \
            --suite base-b-test-diffusion-1-gpu-b200 \
            --timeout-per-file 14400

      - name: Upload diffusion failure artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: diffusion-failures-${{ github.job }}-${{ github.run_attempt }}
          path: diffusion-failures/
          if-no-files-found: ignore

      - name: Upload execution report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: diffusion-report-1gpu-b200-${{ github.run_attempt }}
          path: |
            python/sglang/multimodal_gen/test/execution_report_*.json
            python/diffusion-results.json
          if-no-files-found: ignore
          retention-days: 7

      - uses: ./.github/actions/upload-cuda-coredumps
        if: failure()

  multimodal-gen-unit-test:
    if: |
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
      inputs.multimodal_gen == 'true'
    runs-on: 1-gpu-h100
    timeout-minutes: 120
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git_ref || github.sha }}

      - uses: ./.github/actions/check-pr-test-health

      - uses: ./.github/actions/check-maintenance

      - name: Download artifacts
        if: inputs.sgl_kernel == 'true'
        uses: actions/download-artifact@v4
        with:
          path: python/sglang/kernels/aot/dist/
          merge-multiple: true
          pattern: wheel-python3.10-cuda*

      - name: Install dependencies
        timeout-minutes: 20
        run: |
          CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion

      - name: Run diffusion unit tests
        timeout-minutes: 60
        run: |
          python3 test/run_suite.py \
            --hw cuda \
            --suite base-b-test-diffusion-unit-1-gpu-h100 \
            --timeout-per-file 3600

  diffusion-coverage-check:
    needs: [multimodal-gen-test-1-gpu, multimodal-gen-test-2-gpu]
    if: |
      always() &&
      inputs.multimodal_gen == 'true' &&
      (
        needs.multimodal-gen-test-1-gpu.result == 'success' ||
        needs.multimodal-gen-test-1-gpu.result == 'failure' ||
        needs.multimodal-gen-test-2-gpu.result == 'success' ||
        needs.multimodal-gen-test-2-gpu.result == 'failure'
      )
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git_ref || github.sha }}

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Download all execution reports
        uses: actions/download-artifact@v4
        with:
          path: reports/
          pattern: diffusion-report-*
          merge-multiple: true

      - name: Verify coverage
        run: |
          python scripts/ci/utils/diffusion/verify_diffusion_coverage.py --reports-dir reports/