Add local ZIP uploader for whl releases (#32489)

2026-07-27 00:03:59 -07:00
parent 5cc273a780
commit 082b2a10b6
3 changed files with 483 additions and 0 deletions
@@ -4,6 +4,70 @@ This directory contains scripts to automate version bumping for SGLang releases.

 ## Scripts

+### `upload_zip_to_whl.sh`
+
+Uploads a local ZIP file to a new
+[`sgl-project/whl`](https://github.com/sgl-project/whl) GitHub Release and
+prints its direct download URL. The ZIP bytes remain in the Release rather than
+the Git tree. The script adds the direct link to the flat
+[`others/index.html`](https://docs.sglang.io/whl/others/) catalog on the
+`gh-pages` branch and adds an `others/` entry to the root index. Existing
+package-specific PEP 503 indexes are unchanged.
+
+Prerequisites:
+
+- Install [GitHub CLI](https://cli.github.com/).
+- Authenticate once with `gh auth login`.
+- Use a GitHub account with write access to `sgl-project/whl`.
+- Install Git and Python 3.
+- Keep each ZIP file smaller than 2 GiB.
+
+**Usage:**
+
+```bash
+scripts/release/upload_zip_to_whl.sh <zip-path> <version> [release-tag] [release-title]
+```
+
+For example:
+
+```bash
+scripts/release/upload_zip_to_whl.sh ~/Downloads/model-cache.zip v1.2.0
+```
+
+This creates the tag and release `zip-v1.2.0`, preserves the filename
+`model-cache.zip`, and prints output similar to:
+
+```text
+Release: https://github.com/sgl-project/whl/releases/tag/zip-v1.2.0
+Asset:   https://github.com/sgl-project/whl/releases/download/zip-v1.2.0/model-cache.zip
+SHA256:  <checksum>
+Index:   https://docs.sglang.io/whl/others/
+wget https://github.com/sgl-project/whl/releases/download/zip-v1.2.0/model-cache.zip
+```
+
+The optional third and fourth arguments override the default `zip-<version>` tag
+and `ZIP <version>: <filename>` title:
+
+```bash
+scripts/release/upload_zip_to_whl.sh archive.zip 20260726 \
+  special-build-20260726 "Special build 20260726"
+```
+
+The version and tag may contain ASCII letters, digits, `.`, `_`, and `-`. Every
+upload must use a new tag. The script never overwrites or deletes a Release,
+asset, tag, or index entry. ZIP filenames may contain spaces but not backslashes
+or control characters.
+
+The root and `others` index changes are pushed in one commit. If another process
+updates `gh-pages` concurrently, the script reclones the latest branch and
+retries up to three times.
+
+If Release creation succeeds but index publication fails, rerun the exact same
+command. The script resumes only when the existing Release's filename, byte
+size, GitHub asset digest, and SHA256 marker match the local file; any mismatch
+is treated as a version conflict. After three failed index pushes, use the
+`Release` URL printed by the script to inspect the uploaded asset.
+
 ### `bump_sglang_version.py`
 Updates SGLang version across all relevant files following the pattern from [PR #10468](https://github.com/sgl-project/sglang/pull/10468).

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+import argparse
+import html
+import pathlib
+import re
+
+ROOT_LINK = '<a href="others/">others</a><br>'
+OTHERS_HEADER = "<!DOCTYPE html>\n<h1>SGLang Other Files</h1>\n"
+ASSET_URL_PREFIX = "https://github.com/sgl-project/whl/releases/download/"
+SHA256_PATTERN = re.compile(r"[0-9a-fA-F]{64}")
+TAG_PATTERN = re.compile(r"[A-Za-z0-9._-]+")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Update the sgl-project/whl indexes for a special file."
+    )
+    parser.add_argument("--repo-dir", type=pathlib.Path, required=True)
+    parser.add_argument("--asset-url", required=True)
+    parser.add_argument("--filename", required=True)
+    parser.add_argument("--tag", required=True)
+    parser.add_argument("--sha256", required=True)
+    args = parser.parse_args()
+
+    if not args.repo_dir.is_dir():
+        parser.error(f"repository directory does not exist: {args.repo_dir}")
+    if not (args.repo_dir / "index.html").is_file():
+        parser.error(f"root index does not exist: {args.repo_dir / 'index.html'}")
+    if SHA256_PATTERN.fullmatch(args.sha256) is None:
+        parser.error("--sha256 must contain exactly 64 hexadecimal characters")
+    if TAG_PATTERN.fullmatch(args.tag) is None:
+        parser.error("--tag may contain only ASCII letters, digits, '.', '_', and '-'")
+    if not args.asset_url.startswith(ASSET_URL_PREFIX):
+        parser.error(f"--asset-url must start with {ASSET_URL_PREFIX}")
+
+    return args
+
+
+def update_root_index(repo_dir: pathlib.Path) -> bool:
+    index_path = repo_dir / "index.html"
+    content = index_path.read_text(encoding="utf-8")
+    lines = content.splitlines(keepends=True)
+    root_link_count = sum(line.rstrip("\r\n") == ROOT_LINK for line in lines)
+    if root_link_count == 1:
+        return False
+
+    if root_link_count > 1:
+        content = "".join(line for line in lines if line.rstrip("\r\n") != ROOT_LINK)
+    if content and not content.endswith("\n"):
+        content += "\n"
+    index_path.write_text(f"{content}{ROOT_LINK}\n", encoding="utf-8")
+    return True
+
+
+def update_others_index(
+    repo_dir: pathlib.Path,
+    asset_url: str,
+    filename: str,
+    tag: str,
+    sha256: str,
+) -> bool:
+    index_dir = repo_dir / "others"
+    index_path = index_dir / "index.html"
+    escaped_url = html.escape(asset_url, quote=True)
+    escaped_filename = html.escape(filename, quote=True)
+    escaped_tag = html.escape(tag, quote=True)
+    identity = f'href="{escaped_url}#sha256='
+    entry = (
+        f'<a href="{escaped_url}#sha256={sha256.lower()}">'
+        f"{escaped_filename}</a> ({escaped_tag})<br>\n"
+    )
+
+    if index_path.exists():
+        content = index_path.read_text(encoding="utf-8")
+        if not content.startswith(OTHERS_HEADER):
+            raise ValueError(
+                f"{index_path} does not start with the expected SGLang header"
+            )
+    else:
+        content = OTHERS_HEADER
+
+    if identity in content:
+        return False
+
+    index_dir.mkdir(parents=True, exist_ok=True)
+    updated = f"{OTHERS_HEADER}{entry}{content[len(OTHERS_HEADER):]}"
+    index_path.write_text(updated, encoding="utf-8")
+    return True
+
+
+def main() -> None:
+    args = parse_args()
+    root_changed = update_root_index(args.repo_dir)
+    others_changed = update_others_index(
+        repo_dir=args.repo_dir,
+        asset_url=args.asset_url,
+        filename=args.filename,
+        tag=args.tag,
+        sha256=args.sha256,
+    )
+    print("updated" if root_changed or others_changed else "unchanged")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,313 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly REPOSITORY="sgl-project/whl"
+readonly TARGET_BRANCH="gh-pages"
+readonly MAX_ASSET_BYTES=2147483648
+readonly INDEX_URL="https://docs.sglang.io/whl/others/"
+readonly MAX_INDEX_PUSH_ATTEMPTS=3
+
+API_OUTPUT=""
+RELEASE_URL=""
+ASSET_URL=""
+TEMP_ROOT=""
+
+usage() {
+  cat >&2 <<'EOF'
+Usage: scripts/release/upload_zip_to_whl.sh <zip-path> <version> [release-tag] [release-title]
+
+Example:
+  scripts/release/upload_zip_to_whl.sh ~/Downloads/model-cache.zip v1.2.0
+  scripts/release/upload_zip_to_whl.sh archive.zip 20260617 custom-tag "Custom release"
+EOF
+}
+
+die() {
+  printf 'Error: %s\n' "$*" >&2
+  exit 1
+}
+
+warn() {
+  printf 'Warning: %s\n' "$*" >&2
+}
+
+cleanup() {
+  if [[ -n "${TEMP_ROOT:-}" && -d "$TEMP_ROOT" ]]; then
+    rm -rf -- "$TEMP_ROOT"
+  fi
+}
+
+api_get_optional() {
+  local endpoint="$1"
+  local lookup_status
+
+  API_OUTPUT=""
+  if API_OUTPUT=$(gh api "$endpoint" 2>&1); then
+    return 0
+  else
+    lookup_status=$?
+  fi
+
+  if [[ "$API_OUTPUT" == *"HTTP 404"* ]]; then
+    API_OUTPUT=""
+    return 1
+  fi
+
+  printf '%s\n' "$API_OUTPUT" >&2
+  warn "GitHub API request failed for ${endpoint} (gh exited ${lookup_status})"
+  return 2
+}
+
+compute_sha256() {
+  local file_path="$1"
+
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum "$file_path" | awk '{print $1}'
+    return
+  fi
+
+  if command -v shasum >/dev/null 2>&1; then
+    shasum -a 256 "$file_path" | awk '{print $1}'
+    return
+  fi
+
+  die "Neither sha256sum nor shasum is installed"
+}
+
+load_and_validate_release() {
+  local endpoint="$1"
+  local expected_name="$2"
+  local expected_size="$3"
+  local expected_checksum="$4"
+  local release_body
+  local asset_rows
+  local remote_name
+  local remote_size
+  local remote_url
+  local remote_digest
+  local matched_size=""
+  local matched_url=""
+  local matched_digest=""
+  local match_count=0
+  local checksum_marker
+
+  RELEASE_URL=$(gh api "$endpoint" --jq '.html_url') ||
+    die "Unable to read the Release URL"
+  release_body=$(gh api "$endpoint" --jq '.body // ""') ||
+    die "Unable to read the Release body"
+  asset_rows=$(
+    gh api "$endpoint" \
+      --jq '.assets[] | [.name, .size, .browser_download_url, (.digest // "")] | @tsv'
+  ) || die "Unable to read the Release assets"
+
+  while IFS=$'\t' read -r remote_name remote_size remote_url remote_digest; do
+    if [[ "$remote_name" == "$expected_name" ]]; then
+      ((match_count += 1))
+      matched_size="$remote_size"
+      matched_url="$remote_url"
+      matched_digest="$remote_digest"
+    fi
+  done <<<"$asset_rows"
+
+  checksum_marker="SHA256: \`${expected_checksum}\`"
+  [[ "$release_body" == *"$checksum_marker"* ]] ||
+    die "Existing Release checksum does not match the local ZIP"
+  [[ "$match_count" -eq 1 ]] ||
+    die "Release must contain exactly one asset named ${expected_name}"
+  [[ "$matched_size" == "$expected_size" ]] ||
+    die "Existing Release asset size does not match the local ZIP"
+  [[ "$matched_digest" == "sha256:${expected_checksum}" ]] ||
+    die "Existing Release asset digest does not match the local ZIP"
+  [[ -n "$matched_url" ]] ||
+    die "Existing Release asset has no download URL"
+
+  ASSET_URL="$matched_url"
+}
+
+publish_indexes() {
+  local helper_path="$1"
+  local asset_url="$2"
+  local filename="$3"
+  local release_tag="$4"
+  local checksum="$5"
+  local github_login
+  local attempt
+  local attempt_dir
+  local index_status
+
+  github_login=$(gh api user --jq '.login') ||
+    die "Unable to read the authenticated GitHub username"
+  [[ -n "$github_login" ]] ||
+    die "GitHub returned an empty authenticated username"
+
+  for ((attempt = 1; attempt <= MAX_INDEX_PUSH_ATTEMPTS; attempt += 1)); do
+    attempt_dir="${TEMP_ROOT}/whl-${attempt}"
+
+    if ! git clone --quiet --depth 1 --branch "$TARGET_BRANCH" \
+      "https://github.com/${REPOSITORY}.git" "$attempt_dir"; then
+      warn "Index clone attempt ${attempt}/${MAX_INDEX_PUSH_ATTEMPTS} failed"
+      continue
+    fi
+
+    python3 "$helper_path" \
+      --repo-dir "$attempt_dir" \
+      --asset-url "$asset_url" \
+      --filename "$filename" \
+      --tag "$release_tag" \
+      --sha256 "$checksum" >/dev/null ||
+      die "Unable to update the local whl indexes"
+
+    index_status=$(
+      git -C "$attempt_dir" status --porcelain -- index.html others/index.html
+    )
+    if [[ -z "$index_status" ]]; then
+      return 0
+    fi
+
+    git -C "$attempt_dir" config user.name "$github_login"
+    git -C "$attempt_dir" config \
+      user.email "${github_login}@users.noreply.github.com"
+    git -C "$attempt_dir" add -- index.html others/index.html
+    git -C "$attempt_dir" commit --quiet \
+      -m "Add ${filename} to others index for ${release_tag}" ||
+      die "Unable to commit the local whl index update"
+
+    if git -c credential.helper= \
+      -c credential.helper='!gh auth git-credential' \
+      -C "$attempt_dir" push --quiet origin "HEAD:${TARGET_BRANCH}"; then
+      return 0
+    fi
+
+    warn "Index push attempt ${attempt}/${MAX_INDEX_PUSH_ATTEMPTS} failed; retrying from the latest branch tip"
+  done
+
+  die "Release is available, but the whl indexes could not be pushed after ${MAX_INDEX_PUSH_ATTEMPTS} attempts; rerun the same command to resume"
+}
+
+if [[ $# -lt 2 || $# -gt 4 ]]; then
+  usage
+  exit 2
+fi
+
+input_path="$1"
+version="$2"
+
+[[ -e "$input_path" ]] || die "File does not exist: ${input_path}"
+[[ -f "$input_path" ]] || die "Path is not a regular file: ${input_path}"
+[[ -s "$input_path" ]] || die "ZIP file is empty: ${input_path}"
+
+zip_name=$(basename -- "$input_path")
+zip_dir=$(dirname -- "$input_path")
+zip_dir=$(cd "$zip_dir" && pwd -P) || die "Cannot resolve ZIP directory"
+zip_path="${zip_dir}/${zip_name}"
+
+case "$zip_name" in
+  *.[zZ][iI][pP]) ;;
+  *) die "File must have a .zip extension: ${zip_name}" ;;
+esac
+
+if [[ "$zip_name" == *\\* ]] ||
+  printf '%s' "$zip_name" | LC_ALL=C grep -q '[[:cntrl:]]'; then
+  die "ZIP filename cannot contain backslashes or control characters"
+fi
+
+file_size=$(wc -c <"$zip_path")
+file_size=${file_size//[[:space:]]/}
+[[ "$file_size" =~ ^[0-9]+$ ]] ||
+  die "Unable to determine ZIP file size: ${zip_path}"
+((file_size < MAX_ASSET_BYTES)) ||
+  die "ZIP file must be smaller than 2 GiB (${MAX_ASSET_BYTES} bytes)"
+
+[[ -n "$version" ]] || die "Version cannot be empty"
+[[ "$version" =~ ^[A-Za-z0-9._-]+$ ]] ||
+  die "Version may contain only ASCII letters, digits, '.', '_', and '-'"
+
+release_tag="${3:-zip-${version}}"
+release_title="${4:-ZIP ${version}: ${zip_name}}"
+[[ "$release_tag" =~ ^[A-Za-z0-9._-]+$ ]] ||
+  die "Release tag may contain only ASCII letters, digits, '.', '_', and '-'"
+[[ -n "$release_title" ]] || die "Release title cannot be empty"
+if printf '%s' "$release_title" | LC_ALL=C grep -q '[[:cntrl:]]'; then
+  die "Release title cannot contain control characters"
+fi
+
+command -v gh >/dev/null 2>&1 ||
+  die "GitHub CLI is required; install it from https://cli.github.com/"
+command -v git >/dev/null 2>&1 || die "Git is required"
+command -v python3 >/dev/null 2>&1 || die "Python 3 is required"
+gh auth status --hostname github.com >/dev/null 2>&1 ||
+  die "GitHub CLI is not authenticated; run: gh auth login"
+
+push_permission=$(gh api "repos/${REPOSITORY}" --jq '.permissions.push // false') ||
+  die "Unable to read repository permissions for ${REPOSITORY}"
+[[ "$push_permission" == "true" ]] ||
+  die "The authenticated GitHub account needs write access to ${REPOSITORY}"
+
+checksum=$(compute_sha256 "$zip_path")
+[[ "$checksum" =~ ^[0-9A-Fa-f]{64}$ ]] ||
+  die "Unable to compute a valid SHA256 checksum"
+
+release_endpoint="repos/${REPOSITORY}/releases/tags/${release_tag}"
+tag_endpoint="repos/${REPOSITORY}/git/ref/tags/${release_tag}"
+release_exists=false
+tag_exists=false
+
+if api_get_optional "$release_endpoint"; then
+  release_exists=true
+else
+  api_status=$?
+  [[ "$api_status" -eq 1 ]] ||
+    die "Unable to determine whether Release ${release_tag} exists"
+fi
+
+if api_get_optional "$tag_endpoint"; then
+  tag_exists=true
+else
+  api_status=$?
+  [[ "$api_status" -eq 1 ]] ||
+    die "Unable to determine whether tag ${release_tag} exists"
+fi
+
+if [[ "$release_exists" == "false" && "$tag_exists" == "false" ]]; then
+  release_notes=$(
+    printf 'Uploaded asset: `%s`\n\nSHA256: `%s`' "$zip_name" "$checksum"
+  )
+
+  gh release create "$release_tag" "$zip_path" \
+    --repo "$REPOSITORY" \
+    --target "$TARGET_BRANCH" \
+    --title "$release_title" \
+    --notes "$release_notes" \
+    --latest=false >/dev/null ||
+    die "GitHub failed to create Release ${release_tag}"
+elif [[ "$release_exists" == "true" && "$tag_exists" == "true" ]]; then
+  warn "Release ${release_tag} already exists; validating it before resuming the index update"
+else
+  die "Release/tag state for ${release_tag} is inconsistent; choose a new version or repair it manually"
+fi
+
+load_and_validate_release \
+  "$release_endpoint" "$zip_name" "$file_size" "$checksum"
+
+script_dir=$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) ||
+  die "Cannot resolve the script directory"
+index_helper="${script_dir}/update_others_whl_index.py"
+[[ -f "$index_helper" ]] || die "Missing index helper: ${index_helper}"
+
+TEMP_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/sglang-whl-upload.XXXXXX") ||
+  die "Unable to create a temporary directory"
+[[ -n "$TEMP_ROOT" && -d "$TEMP_ROOT" ]] ||
+  die "mktemp did not create a valid temporary directory"
+trap cleanup EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+publish_indexes "$index_helper" "$ASSET_URL" "$zip_name" "$release_tag" "$checksum"
+
+printf 'Release: %s\n' "$RELEASE_URL"
+printf 'Asset:   %s\n' "$ASSET_URL"
+printf 'SHA256:  %s\n' "$checksum"
+printf 'Index:   %s\n' "$INDEX_URL"
+printf 'wget %q\n' "$ASSET_URL"