fix: unifying docker image build pipeline (#18814)

.github/workflows/release-docker-cu13.yml

@@ -1,122 +0,0 @@
-name: Build and Push CUDA 13 Docker Images
-
-# release this manually via workflow_dispatch for now
-on:
-    workflow_dispatch:
-    schedule:
-      - cron: "0 0 * * *"
-jobs:
-    build-dev:
-        if: ${{ github.repository == 'sgl-project/sglang' }}
-        runs-on: ${{ matrix.runner }}
-        strategy:
-            matrix:
-                include:
-                    - runner: x64-docker-build-node
-                      platform: linux/amd64
-                      build_type: all
-                      grace_blackwell: 0
-                      tag: dev-x86-cu13
-                      version: 13.0.1
-                    - runner: arm-docker-build-node
-                      platform: linux/arm64
-                      build_type: all
-                      grace_blackwell: 1
-                      tag: dev-arm64-cu13
-                      version: 13.0.1
-        steps:
-            - name: Delete huge unnecessary tools folder
-              run: rm -rf /opt/hostedtoolcache
-
-            - name: Checkout repository
-              uses: actions/checkout@v4
-
-            - name: Free disk space
-              uses: jlumbroso/free-disk-space@main
-              with:
-                  tool-cache: true
-                  docker-images: true
-                  android: true
-                  dotnet: true
-                  haskell: true
-                  large-packages: true
-                  swap-storage: true
-
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-
-            - name: Login to Docker Hub
-              uses: docker/login-action@v2
-              with:
-                  username: ${{ secrets.DOCKERHUB_USERNAME }}
-                  password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-            - name: Build and Push Dev Image
-              run: |
-                  docker buildx build \
-                    --platform ${{ matrix.platform }} \
-                    --push \
-                    --target framework \
-                    -f docker/Dockerfile \
-                    --build-arg CUDA_VERSION=${{ matrix.version }} \
-                    --build-arg BUILD_TYPE=${{ matrix.build_type }} \
-                    --build-arg CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) \
-                    --build-arg GRACE_BLACKWELL=${{ matrix.grace_blackwell }} \
-                    --build-arg USE_LATEST_SGLANG=1 \
-                    --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
-                    -t lmsysorg/sglang:${{ matrix.tag }} \
-                    --no-cache \
-                    .
-
-    create-manifests:
-        runs-on: ubuntu-22.04
-        needs: [build-dev]
-        if: ${{ github.repository == 'sgl-project/sglang' }}
-        strategy:
-            matrix:
-                variant:
-                    - tag: dev-cu13
-                      x86_tag: dev-x86-cu13
-                      arm64_tag: dev-arm64-cu13
-        steps:
-            - uses: docker/setup-buildx-action@v3
-
-            - uses: docker/login-action@v2
-              with:
-                  username: ${{ secrets.DOCKERHUB_USERNAME }}
-                  password: ${{ secrets.DOCKERHUB_TOKEN }}
-            - run: |
-                  docker buildx imagetools create \
-                    -t lmsysorg/sglang:${{ matrix.variant.tag }} \
-                    -t lmsysorg/sglang:nightly-${{ matrix.variant.tag }}-$(date +%Y%m%d)-${GITHUB_SHA:0:8} \
-                    lmsysorg/sglang:${{ matrix.variant.x86_tag }} \
-                    lmsysorg/sglang:${{ matrix.variant.arm64_tag }}
-
-            - name: Cleanup Old Nightly Builds
-              run: |
-                  # Get JWT token for Docker Hub API
-                  TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "${{ secrets.DOCKERHUB_USERNAME }}", "password": "${{ secrets.DOCKERHUB_TOKEN }}"}' https://hub.docker.com/v2/users/login/ | jq -r .token)
-
-                  # Get all tags for the repository
-                  TAGS_RESPONSE=$(curl -s -H "Authorization: JWT $TOKEN" "https://hub.docker.com/v2/repositories/lmsysorg/sglang/tags/?page_size=100")
-
-                  # Extract tags that match our pattern and sort by last_updated timestamp (most recent first)
-                  TAGS=$(echo "$TAGS_RESPONSE" | jq -r '.results[] | select(.name | startswith("nightly-${{ matrix.variant.tag }}-")) | "\(.last_updated)|\(.name)"' | sort -r | cut -d'|' -f2)
-
-                  # Count total tags and keep only the 14 most recent
-                  TAG_COUNT=$(echo "$TAGS" | wc -l)
-                  if [ "$TAG_COUNT" -gt 14 ]; then
-                    echo "Found $TAG_COUNT nightly builds, keeping only the 14 most recent"
-                    TAGS_TO_DELETE=$(echo "$TAGS" | tail -n +15)
-                    echo "Tags to delete: $TAGS_TO_DELETE"
-
-                    # Delete old tags
-                    for tag in $TAGS_TO_DELETE; do
-                      echo "Deleting tag: $tag"
-                      curl -X DELETE \
-                        -H "Authorization: JWT $TOKEN" \
-                        "https://hub.docker.com/v2/repositories/lmsysorg/sglang/tags/$tag/"
-                    done
-                  else
-                    echo "Only $TAG_COUNT nightly builds found, no cleanup needed"
-                  fi

.github/workflows/release-docker-dev-pr.yml

@@ -1,112 +0,0 @@
-name: Build PR Development Docker Images
-
-on:
-  workflow_dispatch:
-    inputs:
-      pr_number:
-        description: 'PR number to build from (works with both internal and fork PRs)'
-        required: true
-        type: string
-
-concurrency:
-  group: release-docker-dev-pr-${{ github.event.inputs.pr_number }}
-  cancel-in-progress: true
-
-jobs:
-  build-dev:
-    if: ${{ github.repository == 'sgl-project/sglang' }}
-    environment: "prod"
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      matrix:
-        include:
-          - runner: x64-docker-build-node
-            platform: linux/amd64
-            build_type: all
-            grace_blackwell: 0
-            arch_tag: x86
-            version: 12.9.1
-          - runner: arm-docker-build-node
-            platform: linux/arm64
-            build_type: all
-            grace_blackwell: 1
-            arch_tag: arm64
-            version: 12.9.1
-    steps:
-      - name: Delete huge unnecessary tools folder
-        run: rm -rf /opt/hostedtoolcache
-
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: refs/pull/${{ inputs.pr_number }}/head
-
-      - name: Free disk space
-        uses: jlumbroso/free-disk-space@main
-        with:
-          tool-cache: true
-          docker-images: true
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: true
-          swap-storage: true
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build and Push Dev Image
-        run: |
-          tag=dev-${{ matrix.arch_tag }}-pr-${{ inputs.pr_number }}
-
-          docker buildx build \
-            --platform ${{ matrix.platform }} \
-            --push \
-            -f docker/Dockerfile \
-            --target framework \
-            --build-arg CUDA_VERSION=${{ matrix.version }} \
-            --build-arg BUILD_TYPE=${{ matrix.build_type }} \
-            --build-arg CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) \
-            --build-arg GRACE_BLACKWELL=${{ matrix.grace_blackwell }} \
-            --build-arg BRANCH_TYPE=local \
-            --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
-            -t lmsysorg/sglang:${tag} \
-            --no-cache \
-            .
-
-  create-manifests:
-    runs-on: ubuntu-22.04
-    needs: [build-dev]
-    if: ${{ github.repository == 'sgl-project/sglang' }}
-    environment: "prod"
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Create multi-arch manifest
-        run: |
-          # Create PR dev manifest
-          docker buildx imagetools create \
-            -t lmsysorg/sglang:dev-pr-${{ inputs.pr_number }} \
-            lmsysorg/sglang:dev-x86-pr-${{ inputs.pr_number }} \
-            lmsysorg/sglang:dev-arm64-pr-${{ inputs.pr_number }}
-
-          echo "✓ Built Docker image: lmsysorg/sglang:dev-pr-${{ inputs.pr_number }}"
-          echo ""
-          echo "Usage:"
-          echo "  docker pull lmsysorg/sglang:dev-pr-${{ inputs.pr_number }}"

.github/workflows/release-docker-dev.yml

@@ -2,9 +2,22 @@ name: Build and Push Development Docker Images
 
 on:
   workflow_dispatch:
+    inputs:
+      pr_number:
+        description: "PR number to build from (leave empty to use current branch)"
+        required: false
+        default: ""
+      tag:
+        description: "Custom tag suffix (overrides pr_number in tag). E.g. 'my-test' → dev-x86-my-test, dev-cu13-my-test, etc."
+        required: false
+        default: ""
   schedule:
     - cron: "0 0 * * *"
 
+concurrency:
+  group: release-docker-dev-${{ inputs.tag || inputs.pr_number || 'nightly' }}
+  cancel-in-progress: true
+
 jobs:
   build-dev:
     if: ${{ github.repository == 'sgl-project/sglang' }}
@@ -16,20 +29,34 @@ jobs:
             platform: linux/amd64
             build_type: all
             grace_blackwell: 0
-            tag: dev-x86
+            arch_tag: x86
             version: 12.9.1
           - runner: arm-docker-build-node
             platform: linux/arm64
             build_type: all
             grace_blackwell: 1
-            tag: dev-arm64
+            arch_tag: arm64
             version: 12.9.1
+          - runner: x64-docker-build-node
+            platform: linux/amd64
+            build_type: all
+            grace_blackwell: 0
+            arch_tag: x86-cu13
+            version: 13.0.1
+          - runner: arm-docker-build-node
+            platform: linux/arm64
+            build_type: all
+            grace_blackwell: 1
+            arch_tag: arm64-cu13
+            version: 13.0.1
     steps:
       - name: Delete huge unnecessary tools folder
         run: rm -rf /opt/hostedtoolcache
 
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.pr_number && format('refs/pull/{0}/head', inputs.pr_number) || github.ref }}
 
       - name: Free disk space
         uses: jlumbroso/free-disk-space@main
@@ -53,18 +80,37 @@ jobs:
 
       - name: Build and Push Dev Image
         run: |
+          # Tag suffix: custom tag > pr number > none
+          SUFFIX=""
+          if [ -n "${{ inputs.tag }}" ]; then
+            SUFFIX="-${{ inputs.tag }}"
+          elif [ -n "${{ inputs.pr_number }}" ]; then
+            SUFFIX="-pr-${{ inputs.pr_number }}"
+          fi
+
+          TAG="dev-${{ matrix.arch_tag }}${SUFFIX}"
+
+          # Nightly (schedule) installs latest release; manual dispatch builds from checked-out source
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            SOURCE_ARG="--build-arg USE_LATEST_SGLANG=1"
+          else
+            SOURCE_ARG="--build-arg BRANCH_TYPE=local"
+          fi
+
+          echo "Building lmsysorg/sglang:${TAG}"
+
           docker buildx build \
             --platform ${{ matrix.platform }} \
             --push \
-            -f docker/Dockerfile \
             --target framework \
+            -f docker/Dockerfile \
             --build-arg CUDA_VERSION=${{ matrix.version }} \
             --build-arg BUILD_TYPE=${{ matrix.build_type }} \
             --build-arg CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) \
             --build-arg GRACE_BLACKWELL=${{ matrix.grace_blackwell }} \
-            --build-arg USE_LATEST_SGLANG=1 \
+            ${SOURCE_ARG} \
             --build-arg INSTALL_FLASHINFER_JIT_CACHE=1 \
-            -t lmsysorg/sglang:${{ matrix.tag }} \
+            -t lmsysorg/sglang:${TAG} \
             --no-cache \
             .
 
@@ -75,9 +121,12 @@ jobs:
     strategy:
       matrix:
         variant:
-          - tag: dev
+          - base: dev
-            x86_tag: dev-x86
+            x86: x86
-            arm64_tag: dev-arm64
+            arm64: arm64
+          - base: dev-cu13
+            x86: x86-cu13
+            arm64: arm64-cu13
     steps:
       - uses: docker/setup-buildx-action@v3
 
@@ -85,37 +134,56 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - run: |
+
-          SHORT_SHA="${{ github.sha }}"
+      - name: Create multi-arch manifest
+        run: |
+          SUFFIX=""
+          if [ -n "${{ inputs.tag }}" ]; then
+            SUFFIX="-${{ inputs.tag }}"
+          elif [ -n "${{ inputs.pr_number }}" ]; then
+            SUFFIX="-pr-${{ inputs.pr_number }}"
+          fi
+
+          TAG="${{ matrix.variant.base }}${SUFFIX}"
+          X86_TAG="dev-${{ matrix.variant.x86 }}${SUFFIX}"
+          ARM64_TAG="dev-${{ matrix.variant.arm64 }}${SUFFIX}"
+
+          # For nightly (no suffix), also stamp a dated tag
+          EXTRA_TAG=""
+          if [ -z "${SUFFIX}" ]; then
+            SHORT_SHA="${{ github.sha }}"
+            EXTRA_TAG="-t lmsysorg/sglang:nightly-${TAG}-$(date +%Y%m%d)-${SHORT_SHA:0:8}"
+          fi
+
           docker buildx imagetools create \
-            -t lmsysorg/sglang:${{ matrix.variant.tag }} \
+            -t lmsysorg/sglang:${TAG} \
-            -t lmsysorg/sglang:nightly-${{ matrix.variant.tag }}-$(date +%Y%m%d)-${SHORT_SHA:0:8} \
+            ${EXTRA_TAG} \
-            lmsysorg/sglang:${{ matrix.variant.x86_tag }} \
+            lmsysorg/sglang:${X86_TAG} \
-            lmsysorg/sglang:${{ matrix.variant.arm64_tag }}
+            lmsysorg/sglang:${ARM64_TAG}
+
+          echo "✓ Published lmsysorg/sglang:${TAG}"
 
       - name: Cleanup Old Nightly Builds
+        if: ${{ !inputs.tag && !inputs.pr_number }}
         run: |
-          # Get JWT token for Docker Hub API
+          TOKEN=$(curl -s -H "Content-Type: application/json" \
-          TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "${{ secrets.DOCKERHUB_USERNAME }}", "password": "${{ secrets.DOCKERHUB_TOKEN }}"}' https://hub.docker.com/v2/users/login/ | jq -r .token)
+            -X POST -d '{"username": "${{ secrets.DOCKERHUB_USERNAME }}", "password": "${{ secrets.DOCKERHUB_TOKEN }}"}' \
+            https://hub.docker.com/v2/users/login/ | jq -r .token)
 
-          # Get all tags for the repository
+          TAGS_RESPONSE=$(curl -s -H "Authorization: JWT $TOKEN" \
-          TAGS_RESPONSE=$(curl -s -H "Authorization: JWT $TOKEN" "https://hub.docker.com/v2/repositories/lmsysorg/sglang/tags/?page_size=100")
+            "https://hub.docker.com/v2/repositories/lmsysorg/sglang/tags/?page_size=100")
 
-          # Extract tags that match our pattern and sort by last_updated timestamp (most recent first)
+          TAGS=$(echo "$TAGS_RESPONSE" | jq -r \
-          TAGS=$(echo "$TAGS_RESPONSE" | jq -r '.results[] | select(.name | startswith("nightly-${{ matrix.variant.tag }}-")) | "\(.last_updated)|\(.name)"' | sort -r | cut -d'|' -f2)
+            '.results[] | select(.name | test("^nightly-${{ matrix.variant.base }}-[0-9]")) | "\(.last_updated)|\(.name)"' \
+            | sort -r | cut -d'|' -f2)
 
-          # Count total tags and keep only the 14 most recent
           TAG_COUNT=$(echo "$TAGS" | wc -l)
           if [ "$TAG_COUNT" -gt 14 ]; then
             echo "Found $TAG_COUNT nightly builds, keeping only the 14 most recent"
             TAGS_TO_DELETE=$(echo "$TAGS" | tail -n +15)
-            echo "Tags to delete: $TAGS_TO_DELETE"
-
-            # Delete old tags
             for tag in $TAGS_TO_DELETE; do
               echo "Deleting tag: $tag"
-              curl -X DELETE \
+              curl -X DELETE -H "Authorization: JWT $TOKEN" \
-                -H "Authorization: JWT $TOKEN" \
                 "https://hub.docker.com/v2/repositories/lmsysorg/sglang/tags/$tag/"
             done
           else