diff --git a/scripts/release/README.md b/scripts/release/README.md index 484637695..52d203bd6 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -4,6 +4,70 @@ This directory contains scripts to automate version bumping for SGLang releases. ## Scripts +### `upload_zip_to_whl.sh` + +Uploads a local ZIP file to a new +[`sgl-project/whl`](https://github.com/sgl-project/whl) GitHub Release and +prints its direct download URL. The ZIP bytes remain in the Release rather than +the Git tree. The script adds the direct link to the flat +[`others/index.html`](https://docs.sglang.io/whl/others/) catalog on the +`gh-pages` branch and adds an `others/` entry to the root index. Existing +package-specific PEP 503 indexes are unchanged. + +Prerequisites: + +- Install [GitHub CLI](https://cli.github.com/). +- Authenticate once with `gh auth login`. +- Use a GitHub account with write access to `sgl-project/whl`. +- Install Git and Python 3. +- Keep each ZIP file smaller than 2 GiB. + +**Usage:** + +```bash +scripts/release/upload_zip_to_whl.sh [release-tag] [release-title] +``` + +For example: + +```bash +scripts/release/upload_zip_to_whl.sh ~/Downloads/model-cache.zip v1.2.0 +``` + +This creates the tag and release `zip-v1.2.0`, preserves the filename +`model-cache.zip`, and prints output similar to: + +```text +Release: https://github.com/sgl-project/whl/releases/tag/zip-v1.2.0 +Asset: https://github.com/sgl-project/whl/releases/download/zip-v1.2.0/model-cache.zip +SHA256: +Index: https://docs.sglang.io/whl/others/ +wget https://github.com/sgl-project/whl/releases/download/zip-v1.2.0/model-cache.zip +``` + +The optional third and fourth arguments override the default `zip-` tag +and `ZIP : ` title: + +```bash +scripts/release/upload_zip_to_whl.sh archive.zip 20260726 \ + special-build-20260726 "Special build 20260726" +``` + +The version and tag may contain ASCII letters, digits, `.`, `_`, and `-`. Every +upload must use a new tag. The script never overwrites or deletes a Release, +asset, tag, or index entry. ZIP filenames may contain spaces but not backslashes +or control characters. + +The root and `others` index changes are pushed in one commit. If another process +updates `gh-pages` concurrently, the script reclones the latest branch and +retries up to three times. + +If Release creation succeeds but index publication fails, rerun the exact same +command. The script resumes only when the existing Release's filename, byte +size, GitHub asset digest, and SHA256 marker match the local file; any mismatch +is treated as a version conflict. After three failed index pushes, use the +`Release` URL printed by the script to inspect the uploaded asset. + ### `bump_sglang_version.py` Updates SGLang version across all relevant files following the pattern from [PR #10468](https://github.com/sgl-project/sglang/pull/10468). diff --git a/scripts/release/update_others_whl_index.py b/scripts/release/update_others_whl_index.py new file mode 100755 index 000000000..9599697c6 --- /dev/null +++ b/scripts/release/update_others_whl_index.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +import argparse +import html +import pathlib +import re + +ROOT_LINK = 'others
' +OTHERS_HEADER = "\n

SGLang Other Files

\n" +ASSET_URL_PREFIX = "https://github.com/sgl-project/whl/releases/download/" +SHA256_PATTERN = re.compile(r"[0-9a-fA-F]{64}") +TAG_PATTERN = re.compile(r"[A-Za-z0-9._-]+") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Update the sgl-project/whl indexes for a special file." + ) + parser.add_argument("--repo-dir", type=pathlib.Path, required=True) + parser.add_argument("--asset-url", required=True) + parser.add_argument("--filename", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--sha256", required=True) + args = parser.parse_args() + + if not args.repo_dir.is_dir(): + parser.error(f"repository directory does not exist: {args.repo_dir}") + if not (args.repo_dir / "index.html").is_file(): + parser.error(f"root index does not exist: {args.repo_dir / 'index.html'}") + if SHA256_PATTERN.fullmatch(args.sha256) is None: + parser.error("--sha256 must contain exactly 64 hexadecimal characters") + if TAG_PATTERN.fullmatch(args.tag) is None: + parser.error("--tag may contain only ASCII letters, digits, '.', '_', and '-'") + if not args.asset_url.startswith(ASSET_URL_PREFIX): + parser.error(f"--asset-url must start with {ASSET_URL_PREFIX}") + + return args + + +def update_root_index(repo_dir: pathlib.Path) -> bool: + index_path = repo_dir / "index.html" + content = index_path.read_text(encoding="utf-8") + lines = content.splitlines(keepends=True) + root_link_count = sum(line.rstrip("\r\n") == ROOT_LINK for line in lines) + if root_link_count == 1: + return False + + if root_link_count > 1: + content = "".join(line for line in lines if line.rstrip("\r\n") != ROOT_LINK) + if content and not content.endswith("\n"): + content += "\n" + index_path.write_text(f"{content}{ROOT_LINK}\n", encoding="utf-8") + return True + + +def update_others_index( + repo_dir: pathlib.Path, + asset_url: str, + filename: str, + tag: str, + sha256: str, +) -> bool: + index_dir = repo_dir / "others" + index_path = index_dir / "index.html" + escaped_url = html.escape(asset_url, quote=True) + escaped_filename = html.escape(filename, quote=True) + escaped_tag = html.escape(tag, quote=True) + identity = f'href="{escaped_url}#sha256=' + entry = ( + f'' + f"{escaped_filename} ({escaped_tag})
\n" + ) + + if index_path.exists(): + content = index_path.read_text(encoding="utf-8") + if not content.startswith(OTHERS_HEADER): + raise ValueError( + f"{index_path} does not start with the expected SGLang header" + ) + else: + content = OTHERS_HEADER + + if identity in content: + return False + + index_dir.mkdir(parents=True, exist_ok=True) + updated = f"{OTHERS_HEADER}{entry}{content[len(OTHERS_HEADER):]}" + index_path.write_text(updated, encoding="utf-8") + return True + + +def main() -> None: + args = parse_args() + root_changed = update_root_index(args.repo_dir) + others_changed = update_others_index( + repo_dir=args.repo_dir, + asset_url=args.asset_url, + filename=args.filename, + tag=args.tag, + sha256=args.sha256, + ) + print("updated" if root_changed or others_changed else "unchanged") + + +if __name__ == "__main__": + main() diff --git a/scripts/release/upload_zip_to_whl.sh b/scripts/release/upload_zip_to_whl.sh new file mode 100755 index 000000000..983e87ac1 --- /dev/null +++ b/scripts/release/upload_zip_to_whl.sh @@ -0,0 +1,313 @@ +#!/usr/bin/env bash + +set -euo pipefail + +readonly REPOSITORY="sgl-project/whl" +readonly TARGET_BRANCH="gh-pages" +readonly MAX_ASSET_BYTES=2147483648 +readonly INDEX_URL="https://docs.sglang.io/whl/others/" +readonly MAX_INDEX_PUSH_ATTEMPTS=3 + +API_OUTPUT="" +RELEASE_URL="" +ASSET_URL="" +TEMP_ROOT="" + +usage() { + cat >&2 <<'EOF' +Usage: scripts/release/upload_zip_to_whl.sh [release-tag] [release-title] + +Example: + scripts/release/upload_zip_to_whl.sh ~/Downloads/model-cache.zip v1.2.0 + scripts/release/upload_zip_to_whl.sh archive.zip 20260617 custom-tag "Custom release" +EOF +} + +die() { + printf 'Error: %s\n' "$*" >&2 + exit 1 +} + +warn() { + printf 'Warning: %s\n' "$*" >&2 +} + +cleanup() { + if [[ -n "${TEMP_ROOT:-}" && -d "$TEMP_ROOT" ]]; then + rm -rf -- "$TEMP_ROOT" + fi +} + +api_get_optional() { + local endpoint="$1" + local lookup_status + + API_OUTPUT="" + if API_OUTPUT=$(gh api "$endpoint" 2>&1); then + return 0 + else + lookup_status=$? + fi + + if [[ "$API_OUTPUT" == *"HTTP 404"* ]]; then + API_OUTPUT="" + return 1 + fi + + printf '%s\n' "$API_OUTPUT" >&2 + warn "GitHub API request failed for ${endpoint} (gh exited ${lookup_status})" + return 2 +} + +compute_sha256() { + local file_path="$1" + + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$file_path" | awk '{print $1}' + return + fi + + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$file_path" | awk '{print $1}' + return + fi + + die "Neither sha256sum nor shasum is installed" +} + +load_and_validate_release() { + local endpoint="$1" + local expected_name="$2" + local expected_size="$3" + local expected_checksum="$4" + local release_body + local asset_rows + local remote_name + local remote_size + local remote_url + local remote_digest + local matched_size="" + local matched_url="" + local matched_digest="" + local match_count=0 + local checksum_marker + + RELEASE_URL=$(gh api "$endpoint" --jq '.html_url') || + die "Unable to read the Release URL" + release_body=$(gh api "$endpoint" --jq '.body // ""') || + die "Unable to read the Release body" + asset_rows=$( + gh api "$endpoint" \ + --jq '.assets[] | [.name, .size, .browser_download_url, (.digest // "")] | @tsv' + ) || die "Unable to read the Release assets" + + while IFS=$'\t' read -r remote_name remote_size remote_url remote_digest; do + if [[ "$remote_name" == "$expected_name" ]]; then + ((match_count += 1)) + matched_size="$remote_size" + matched_url="$remote_url" + matched_digest="$remote_digest" + fi + done <<<"$asset_rows" + + checksum_marker="SHA256: \`${expected_checksum}\`" + [[ "$release_body" == *"$checksum_marker"* ]] || + die "Existing Release checksum does not match the local ZIP" + [[ "$match_count" -eq 1 ]] || + die "Release must contain exactly one asset named ${expected_name}" + [[ "$matched_size" == "$expected_size" ]] || + die "Existing Release asset size does not match the local ZIP" + [[ "$matched_digest" == "sha256:${expected_checksum}" ]] || + die "Existing Release asset digest does not match the local ZIP" + [[ -n "$matched_url" ]] || + die "Existing Release asset has no download URL" + + ASSET_URL="$matched_url" +} + +publish_indexes() { + local helper_path="$1" + local asset_url="$2" + local filename="$3" + local release_tag="$4" + local checksum="$5" + local github_login + local attempt + local attempt_dir + local index_status + + github_login=$(gh api user --jq '.login') || + die "Unable to read the authenticated GitHub username" + [[ -n "$github_login" ]] || + die "GitHub returned an empty authenticated username" + + for ((attempt = 1; attempt <= MAX_INDEX_PUSH_ATTEMPTS; attempt += 1)); do + attempt_dir="${TEMP_ROOT}/whl-${attempt}" + + if ! git clone --quiet --depth 1 --branch "$TARGET_BRANCH" \ + "https://github.com/${REPOSITORY}.git" "$attempt_dir"; then + warn "Index clone attempt ${attempt}/${MAX_INDEX_PUSH_ATTEMPTS} failed" + continue + fi + + python3 "$helper_path" \ + --repo-dir "$attempt_dir" \ + --asset-url "$asset_url" \ + --filename "$filename" \ + --tag "$release_tag" \ + --sha256 "$checksum" >/dev/null || + die "Unable to update the local whl indexes" + + index_status=$( + git -C "$attempt_dir" status --porcelain -- index.html others/index.html + ) + if [[ -z "$index_status" ]]; then + return 0 + fi + + git -C "$attempt_dir" config user.name "$github_login" + git -C "$attempt_dir" config \ + user.email "${github_login}@users.noreply.github.com" + git -C "$attempt_dir" add -- index.html others/index.html + git -C "$attempt_dir" commit --quiet \ + -m "Add ${filename} to others index for ${release_tag}" || + die "Unable to commit the local whl index update" + + if git -c credential.helper= \ + -c credential.helper='!gh auth git-credential' \ + -C "$attempt_dir" push --quiet origin "HEAD:${TARGET_BRANCH}"; then + return 0 + fi + + warn "Index push attempt ${attempt}/${MAX_INDEX_PUSH_ATTEMPTS} failed; retrying from the latest branch tip" + done + + die "Release is available, but the whl indexes could not be pushed after ${MAX_INDEX_PUSH_ATTEMPTS} attempts; rerun the same command to resume" +} + +if [[ $# -lt 2 || $# -gt 4 ]]; then + usage + exit 2 +fi + +input_path="$1" +version="$2" + +[[ -e "$input_path" ]] || die "File does not exist: ${input_path}" +[[ -f "$input_path" ]] || die "Path is not a regular file: ${input_path}" +[[ -s "$input_path" ]] || die "ZIP file is empty: ${input_path}" + +zip_name=$(basename -- "$input_path") +zip_dir=$(dirname -- "$input_path") +zip_dir=$(cd "$zip_dir" && pwd -P) || die "Cannot resolve ZIP directory" +zip_path="${zip_dir}/${zip_name}" + +case "$zip_name" in + *.[zZ][iI][pP]) ;; + *) die "File must have a .zip extension: ${zip_name}" ;; +esac + +if [[ "$zip_name" == *\\* ]] || + printf '%s' "$zip_name" | LC_ALL=C grep -q '[[:cntrl:]]'; then + die "ZIP filename cannot contain backslashes or control characters" +fi + +file_size=$(wc -c <"$zip_path") +file_size=${file_size//[[:space:]]/} +[[ "$file_size" =~ ^[0-9]+$ ]] || + die "Unable to determine ZIP file size: ${zip_path}" +((file_size < MAX_ASSET_BYTES)) || + die "ZIP file must be smaller than 2 GiB (${MAX_ASSET_BYTES} bytes)" + +[[ -n "$version" ]] || die "Version cannot be empty" +[[ "$version" =~ ^[A-Za-z0-9._-]+$ ]] || + die "Version may contain only ASCII letters, digits, '.', '_', and '-'" + +release_tag="${3:-zip-${version}}" +release_title="${4:-ZIP ${version}: ${zip_name}}" +[[ "$release_tag" =~ ^[A-Za-z0-9._-]+$ ]] || + die "Release tag may contain only ASCII letters, digits, '.', '_', and '-'" +[[ -n "$release_title" ]] || die "Release title cannot be empty" +if printf '%s' "$release_title" | LC_ALL=C grep -q '[[:cntrl:]]'; then + die "Release title cannot contain control characters" +fi + +command -v gh >/dev/null 2>&1 || + die "GitHub CLI is required; install it from https://cli.github.com/" +command -v git >/dev/null 2>&1 || die "Git is required" +command -v python3 >/dev/null 2>&1 || die "Python 3 is required" +gh auth status --hostname github.com >/dev/null 2>&1 || + die "GitHub CLI is not authenticated; run: gh auth login" + +push_permission=$(gh api "repos/${REPOSITORY}" --jq '.permissions.push // false') || + die "Unable to read repository permissions for ${REPOSITORY}" +[[ "$push_permission" == "true" ]] || + die "The authenticated GitHub account needs write access to ${REPOSITORY}" + +checksum=$(compute_sha256 "$zip_path") +[[ "$checksum" =~ ^[0-9A-Fa-f]{64}$ ]] || + die "Unable to compute a valid SHA256 checksum" + +release_endpoint="repos/${REPOSITORY}/releases/tags/${release_tag}" +tag_endpoint="repos/${REPOSITORY}/git/ref/tags/${release_tag}" +release_exists=false +tag_exists=false + +if api_get_optional "$release_endpoint"; then + release_exists=true +else + api_status=$? + [[ "$api_status" -eq 1 ]] || + die "Unable to determine whether Release ${release_tag} exists" +fi + +if api_get_optional "$tag_endpoint"; then + tag_exists=true +else + api_status=$? + [[ "$api_status" -eq 1 ]] || + die "Unable to determine whether tag ${release_tag} exists" +fi + +if [[ "$release_exists" == "false" && "$tag_exists" == "false" ]]; then + release_notes=$( + printf 'Uploaded asset: `%s`\n\nSHA256: `%s`' "$zip_name" "$checksum" + ) + + gh release create "$release_tag" "$zip_path" \ + --repo "$REPOSITORY" \ + --target "$TARGET_BRANCH" \ + --title "$release_title" \ + --notes "$release_notes" \ + --latest=false >/dev/null || + die "GitHub failed to create Release ${release_tag}" +elif [[ "$release_exists" == "true" && "$tag_exists" == "true" ]]; then + warn "Release ${release_tag} already exists; validating it before resuming the index update" +else + die "Release/tag state for ${release_tag} is inconsistent; choose a new version or repair it manually" +fi + +load_and_validate_release \ + "$release_endpoint" "$zip_name" "$file_size" "$checksum" + +script_dir=$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) || + die "Cannot resolve the script directory" +index_helper="${script_dir}/update_others_whl_index.py" +[[ -f "$index_helper" ]] || die "Missing index helper: ${index_helper}" + +TEMP_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/sglang-whl-upload.XXXXXX") || + die "Unable to create a temporary directory" +[[ -n "$TEMP_ROOT" && -d "$TEMP_ROOT" ]] || + die "mktemp did not create a valid temporary directory" +trap cleanup EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +publish_indexes "$index_helper" "$ASSET_URL" "$zip_name" "$release_tag" "$checksum" + +printf 'Release: %s\n' "$RELEASE_URL" +printf 'Asset: %s\n' "$ASSET_URL" +printf 'SHA256: %s\n' "$checksum" +printf 'Index: %s\n' "$INDEX_URL" +printf 'wget %q\n' "$ASSET_URL"