[CI] skip srt rust extension builds for diffusion-only PRs (#40293)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>

.github/workflows/pr-test-multimodal-gen.yml

@@ -9,10 +9,6 @@ on:
       sgl_kernel:
         required: true
         type: string
-      rust_ext_artifact:
-        description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, falls back to the cache; a miss there compiles during install.'
-        type: string
-        default: ''
       runner_config:
         required: true
         type: string
@@ -107,10 +103,6 @@ jobs:
           merge-multiple: true
           pattern: wheel-python3.10-cuda*
 
-      - uses: ./.github/actions/download-rust-ext
-        with:
-          artifact_name: ${{ inputs.rust_ext_artifact }}
-
       - name: Install dependencies
         timeout-minutes: 20
         run: |
@@ -178,10 +170,6 @@ jobs:
           merge-multiple: true
           pattern: wheel-python3.10-cuda*
 
-      - uses: ./.github/actions/download-rust-ext
-        with:
-          artifact_name: ${{ inputs.rust_ext_artifact }}
-
       - name: Install dependencies
         timeout-minutes: 20
         run: |
@@ -247,10 +235,6 @@ jobs:
           merge-multiple: true
           pattern: wheel-python3.10-cuda*
 
-      - uses: ./.github/actions/download-rust-ext
-        with:
-          artifact_name: ${{ inputs.rust_ext_artifact }}
-
       - name: Install dependencies
         timeout-minutes: 20
         run: |
@@ -313,10 +297,6 @@ jobs:
           merge-multiple: true
           pattern: wheel-python3.10-cuda*
 
-      - uses: ./.github/actions/download-rust-ext
-        with:
-          artifact_name: ${{ inputs.rust_ext_artifact }}
-
       - name: Install dependencies
         timeout-minutes: 20
         run: |
@@ -387,10 +367,6 @@ jobs:
           merge-multiple: true
           pattern: wheel-python3.10-cuda*
 
-      - uses: ./.github/actions/download-rust-ext
-        with:
-          artifact_name: ${{ inputs.rust_ext_artifact }}
-
       - name: Install dependencies
         timeout-minutes: 20
         run: |
@@ -436,10 +412,6 @@ jobs:
           merge-multiple: true
           pattern: wheel-python3.10-cuda*
 
-      - uses: ./.github/actions/download-rust-ext
-        with:
-          artifact_name: ${{ inputs.rust_ext_artifact }}
-
       - name: Install dependencies
         timeout-minutes: 20
         env:
@@ -505,10 +477,6 @@ jobs:
           merge-multiple: true
           pattern: wheel-python3.10-cuda*
 
-      - uses: ./.github/actions/download-rust-ext
-        with:
-          artifact_name: ${{ inputs.rust_ext_artifact }}
-
       - name: Install dependencies
         timeout-minutes: 20
         run: |