[diffusion] CI: add performance tracking job to nightly (#21091)

.github/workflows/diffusion-ci-gt-gen.yml

@@ -112,4 +112,4 @@ jobs:
       - name: Publish GT images to sglang-bot/sglang-ci-data
         env:
           GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-        run: python scripts/ci/utils/publish_diffusion_gt.py --source-dir gt_images
+        run: python scripts/ci/utils/diffusion/publish_diffusion_gt.py --source-dir gt_images

.github/workflows/nightly-test-nvidia.yml

@@ -26,6 +26,7 @@ on:
           - 'nightly-test-perf-4-gpu-b200'
           - 'nightly-test-perf-8-gpu-b200'
           - 'nightly-test-kernel-1-gpu-h100'
+          - 'nightly-test-diffusion-comparison'
   workflow_call:
     inputs:
       ref:
@@ -511,7 +512,7 @@ jobs:
       - name: Collect diffusion performance metrics
         if: always()
         run: |
-          python3 scripts/ci/utils/save_diffusion_metrics.py \
+          python3 scripts/ci/utils/diffusion/save_diffusion_metrics.py \
             --gpu-config 1-gpu-h100 \
             --run-id ${{ github.run_id }} \
             --output python/diffusion-metrics-1gpu-partition-${{ matrix.part }}.json \
@@ -571,7 +572,7 @@ jobs:
       - name: Collect diffusion performance metrics
         if: always()
         run: |
-          python3 scripts/ci/utils/save_diffusion_metrics.py \
+          python3 scripts/ci/utils/diffusion/save_diffusion_metrics.py \
             --gpu-config 2-gpu-h100 \
             --run-id ${{ github.run_id }} \
             --output python/diffusion-metrics-2gpu-partition-${{ matrix.part }}.json \
@@ -649,6 +650,69 @@ jobs:
       - uses: ./.github/actions/upload-cuda-coredumps
         if: always()
 
+  # Diffusion cross-framework comparison
+  nightly-test-diffusion-comparison:
+    if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-diffusion-comparison')
+    runs-on: 4-gpu-h100
+    timeout-minutes: 240
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/cuda/ci_install_dependency.sh diffusion
+
+      - name: Run cross-framework comparison
+        env:
+          GITHUB_SHA: ${{ github.sha }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          PYTHONUNBUFFERED: "1"
+        timeout-minutes: 210
+        run: |
+          python3 -u scripts/ci/utils/diffusion/run_comparison.py \
+            --output comparison-results.json
+
+      - name: Generate dashboard
+        if: always()
+        env:
+          GH_PAT_FOR_NIGHTLY_CI_DATA: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python3 scripts/ci/utils/diffusion/generate_diffusion_dashboard.py \
+            --results comparison-results.json \
+            --output dashboard.md \
+            --charts-dir comparison-charts \
+            --fetch-history \
+            --step-summary
+
+      - name: Publish to sglang-ci-data
+        if: always()
+        env:
+          GH_PAT_FOR_NIGHTLY_CI_DATA: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
+        run: |
+          python3 scripts/ci/utils/diffusion/publish_comparison_results.py \
+            --results comparison-results.json \
+            --dashboard dashboard.md \
+            --charts-dir comparison-charts
+
+      - name: Upload comparison artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffusion-comparison-${{ github.run_id }}
+          path: |
+            comparison-results.json
+            dashboard.md
+            comparison-charts/
+            comparison-logs/
+          retention-days: 90
+          if-no-files-found: ignore
+
+      - uses: ./.github/actions/upload-cuda-coredumps
+        if: always()
+
   # Consolidate performance metrics from all jobs
   consolidate-metrics:
     if: github.repository == 'sgl-project/sglang' && always()
@@ -710,6 +774,7 @@ jobs:
       - nightly-test-multimodal-server-2-gpu
       - nightly-test-perf-4-gpu-b200
       - nightly-test-specialized-8-gpu-b200
+      - nightly-test-diffusion-comparison
       - consolidate-metrics
     runs-on: ubuntu-latest
     steps: