From aa5c7e887c7e1ef1cba10176710ce709882ef594 Mon Sep 17 00:00:00 2001 From: snadampal <87143774+snadampal@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:19:22 -0500 Subject: [PATCH] [Feat] docker: Add AWS EFA image variants built on published images (#41006) Signed-off-by: Sunita Nadampalli --- .github/workflows/release-docker-efa.yml | 287 ++++++++++++++++++ docker/efa.Dockerfile | 93 ++++++ .../advanced_features/pd_disaggregation.mdx | 47 +++ docs/docs/get-started/install.mdx | 1 + .../storage/mooncake_store/mooncake_store.py | 8 +- .../mem_cache/test_mooncake_store_config.py | 42 ++- 6 files changed, 475 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/release-docker-efa.yml create mode 100644 docker/efa.Dockerfile diff --git a/.github/workflows/release-docker-efa.yml b/.github/workflows/release-docker-efa.yml new file mode 100644 index 0000000000..0be7210840 --- /dev/null +++ b/.github/workflows/release-docker-efa.yml @@ -0,0 +1,287 @@ +name: Release Docker EFA Images +# +# Layers docker/efa.Dockerfile on top of already-published images, so the main +# image workflows and docker/Dockerfile stay untouched. Runs after each base +# workflow succeeds: +# +# Release Docker Images (tag push) +# lmsysorg/sglang:v{version} -> v{version}-efa, latest-efa, +# v{version}-cu130-efa, latest-cu130-efa +# Release Docker Runtime Images (tag push) +# lmsysorg/sglang:v{version}-runtime -> v{version}-runtime-efa, latest-runtime-efa, +# v{version}-cu130-runtime-efa, latest-cu130-runtime-efa +# Build and Push Development Docker Images (schedule or workflow_dispatch) +# lmsysorg/sglang:nightly-dev-{date}-{short_sha} -> nightly-dev-efa-{date}-{short_sha}, dev-efa +# +# The EFA image inherits the base's SGLang commit and, for nightlies, its exact +# {date}-{short_sha} suffix, so nightly-dev-efa-X matches nightly-dev-X. +# +# A manual dev run publishes a nightly-dev-* tag only when it has no tag or +# pr_number input; PR and custom-tag builds publish dev- instead and +# are skipped here, since there is no dated nightly to pair with. +# +# Manually dispatched release runs are not followed: their inputs (version, +# image_repo) are not in the workflow_run payload. Use this workflow's own +# workflow_dispatch for those, and for one-off builds. +on: + workflow_run: + workflows: + - Release Docker Images + - Release Docker Runtime Images + - Build and Push Development Docker Images + types: [completed] + workflow_dispatch: + inputs: + base_tag: + description: "Published lmsysorg/sglang tag to layer EFA on (e.g. v0.5.21-runtime, nightly-dev-20261013-7224fce1)" + required: true + output_tags: + description: "Comma-separated tags to publish (e.g. v0.5.21-runtime-efa,latest-runtime-efa). Overwrites existing tags." + required: true + image_repo: + description: "Docker Hub repo to push the EFA image to; the base is always pulled from lmsysorg/sglang. Use lmsysorg/sglang-staging for testing." + required: false + default: "lmsysorg/sglang" + +concurrency: + # One group per base run (re-runs of it share the id), so only a re-run + # supersedes an in-progress EFA build. A coarser key would let one base run + # cancel another's EFA build: e.g. a manual PR build of the dev image, which + # resolves to skip, would cancel the EFA build for a manual nightly run, and + # that day would get no nightly-dev-efa-* at all. + group: >- + release-docker-efa-${{ github.event_name == 'workflow_dispatch' && inputs.output_tags || + github.event.workflow_run.id }} + cancel-in-progress: true + +jobs: + resolve: + if: >- + github.repository == 'sgl-project/sglang' && + (github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + (github.event.workflow_run.event == 'push' || + github.event.workflow_run.event == 'schedule' || + github.event.workflow_run.event == 'workflow_dispatch'))) + runs-on: ubuntu-latest + outputs: + skip: ${{ steps.resolve.outputs.skip }} + nightly: ${{ steps.resolve.outputs.nightly }} + base_image: ${{ steps.resolve.outputs.base_image }} + out_images: ${{ steps.resolve.outputs.out_images }} + primary_image: ${{ steps.resolve.outputs.primary_image }} + environment: ${{ steps.resolve.outputs.environment }} + steps: + - name: Resolve base image and output tags + id: resolve + env: + # Inject via env so a crafted input or branch name cannot break out + # of the script. + EVENT_NAME: ${{ github.event_name }} + SOURCE_WORKFLOW: ${{ github.event.workflow_run.name }} + SOURCE_EVENT: ${{ github.event.workflow_run.event }} + SOURCE_REF: ${{ github.event.workflow_run.head_branch }} + SOURCE_SHA: ${{ github.event.workflow_run.head_sha }} + SOURCE_STARTED: ${{ github.event.workflow_run.run_started_at }} + INPUT_BASE_TAG: ${{ inputs.base_tag }} + INPUT_OUTPUT_TAGS: ${{ inputs.output_tags }} + INPUT_IMAGE_REPO: ${{ inputs.image_repo }} + run: | + set -euo pipefail + + # Whole-string checks. A '^...$' regex anchors per LINE, so a + # multi-line value would pass and then inject extra GITHUB_OUTPUT lines. + validate_tag() { + case "$2" in + ""|[!a-zA-Z0-9_]*|*[!a-zA-Z0-9._-]*) + echo "::error::${1} '${2}' is not a valid Docker tag" + exit 1 + ;; + esac + } + + # Bases always come from the published repo. REPO is only where the + # EFA image is pushed; workflow_dispatch can point it at e.g. + # lmsysorg/sglang-staging, which has no nightly or release bases. + BASE_REPO="lmsysorg/sglang" + REPO="lmsysorg/sglang" + ENVIRONMENT="" + NIGHTLY="false" + if [ "${EVENT_NAME}" = "workflow_dispatch" ]; then + [ -n "${INPUT_IMAGE_REPO}" ] && REPO="${INPUT_IMAGE_REPO}" + case "${REPO}" in + ""|*[!a-z0-9._/-]*|/*|*/) + echo "::error::'${REPO}' is not a valid Docker Hub repo" + exit 1 + ;; + esac + BASE_TAG="${INPUT_BASE_TAG}" + IFS=',' read -ra OUT_TAGS <<< "${INPUT_OUTPUT_TAGS}" + else + case "${SOURCE_WORKFLOW}/${SOURCE_EVENT}" in + "Release Docker Images/push"|"Release Docker Runtime Images/push") + # For tag pushes head_branch is the tag name, e.g. v0.5.21. + # Same check as the base release workflows; validate_tag below + # rejects anything unusable as a tag. + VERSION="${SOURCE_REF#v}" + if ! echo "${VERSION}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+'; then + echo "::error::Cannot derive a release version from '${SOURCE_REF}'" + exit 1 + fi + if [ "${SOURCE_WORKFLOW}" = "Release Docker Images" ]; then + FLAVOR="" + else + FLAVOR="-runtime" + fi + BASE_TAG="v${VERSION}${FLAVOR}" + OUT_TAGS=("v${VERSION}${FLAVOR}-efa" "latest${FLAVOR}-efa" + "v${VERSION}-cu130${FLAVOR}-efa" "latest-cu130${FLAVOR}-efa") + # Match the base release workflows, which publish from `prod`. + ENVIRONMENT="prod" + ;; + "Build and Push Development Docker Images/schedule"|"Build and Push Development Docker Images/workflow_dispatch") + # The base nightly tag is nightly-dev-{date}-{short_sha}, where + # {date} is taken when its manifest is created. Look the tag up by + # commit instead of recomputing the date, so a run that crosses + # midnight still finds it and the EFA tag reuses the same suffix. + # + # Only accept a tag pushed after this base run started, i.e. by + # this run. Otherwise a manual PR or custom-tag build dispatched + # from a commit that already has a nightly would re-publish that + # older nightly, and could move dev-efa back to it. Comparing the + # first 19 characters (YYYY-MM-DDTHH:MM:SS) of both ISO 8601 + # timestamps orders them correctly as strings. + SHORT_SHA="${SOURCE_SHA:0:8}" + case "${SHORT_SHA}" in + *[!0-9a-f]*|"") echo "::error::Bad head_sha '${SOURCE_SHA}'"; exit 1 ;; + esac + BASE_TAG="" + for attempt in 1 2 3; do + BASE_TAG="$(curl -fsS --retry 3 \ + "https://hub.docker.com/v2/repositories/${BASE_REPO}/tags/?name=-${SHORT_SHA}&page_size=100" \ + | jq -r --arg sha "${SHORT_SHA}" --arg since "${SOURCE_STARTED:0:19}" \ + '[.results[] + | select(.name | test("^nightly-dev-[0-9]{8}-" + $sha + "$")) + | select(.last_updated[0:19] >= $since) + | .name] | first // empty')" || true + [ -n "${BASE_TAG}" ] && break + sleep 30 + done + if [ -z "${BASE_TAG}" ]; then + if [ "${SOURCE_EVENT}" = "schedule" ]; then + # A successful scheduled run always publishes this tag, so + # its absence is a real problem. + echo "::error::No nightly-dev--${SHORT_SHA} tag pushed by this run found in ${BASE_REPO}" + exit 1 + fi + # Manual run that published no dated nightly here: a PR or + # custom-tag build, build_only, or another image_repo. + echo "No nightly-dev--${SHORT_SHA} tag pushed by this run in ${BASE_REPO}; nothing to do" + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + SUFFIX="${BASE_TAG#nightly-dev-}" + OUT_TAGS=("nightly-dev-efa-${SUFFIX}" "dev-efa") + NIGHTLY="true" + ;; + *) + echo "Nothing to do for '${SOURCE_WORKFLOW}' (${SOURCE_EVENT})" + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + ;; + esac + fi + + validate_tag base_tag "${BASE_TAG}" + [ "${#OUT_TAGS[@]}" -gt 0 ] || { echo "::error::No output tags"; exit 1; } + OUT_IMAGES="" + for t in "${OUT_TAGS[@]}"; do + t="$(echo "${t}" | xargs)" + validate_tag output_tag "${t}" + OUT_IMAGES="${OUT_IMAGES:+${OUT_IMAGES} }${REPO}:${t}" + done + + echo "Base: ${BASE_REPO}:${BASE_TAG}" + echo "Outputs: ${OUT_IMAGES}" + { + echo "skip=false" + echo "nightly=${NIGHTLY}" + echo "base_image=${BASE_REPO}:${BASE_TAG}" + echo "out_images=${OUT_IMAGES}" + echo "primary_image=${OUT_IMAGES%% *}" + echo "environment=${ENVIRONMENT}" + } >> "$GITHUB_OUTPUT" + + build-and-publish: + needs: resolve + if: needs.resolve.outputs.skip == 'false' + # Same runner as patch-docker-dev.yml, which also layers onto published + # multi-arch images; it has the disk for both platforms' base layers. + runs-on: x64-docker-build-node + environment: ${{ needs.resolve.outputs.environment || null }} + env: + BASE_IMAGE: ${{ needs.resolve.outputs.base_image }} + OUT_IMAGES: ${{ needs.resolve.outputs.out_images }} + PRIMARY_IMAGE: ${{ needs.resolve.outputs.primary_image }} + steps: + - name: Cleanup workspace (remove root-owned files from prior runs) + run: sudo rm -rf "$GITHUB_WORKSPACE"/* || true + + # Always the default branch's efa.Dockerfile: older release tags do not + # have it, and the layer does not depend on the SGLang source. + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push EFA image (amd64 + arm64) + run: | + set -euo pipefail + TAG_ARGS=() + for img in ${OUT_IMAGES}; do TAG_ARGS+=(-t "${img}"); done + # Each platform layers onto its own arch of the base manifest list, so + # the output stays a list. A single-arch base fails here on purpose: + # the output would not be pullable on the other arch. + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --no-cache \ + -f docker/efa.Dockerfile \ + --build-arg BASE_IMAGE="${BASE_IMAGE}" \ + --build-arg EFA_IMAGE_TAG="${PRIMARY_IMAGE}" \ + "${TAG_ARGS[@]}" \ + --push \ + docker/ + + - name: Write summary + run: | + { + echo "### Published EFA image" + echo "- **Base image:** \`${BASE_IMAGE}\`" + for img in ${OUT_IMAGES}; do echo "- \`${img}\`"; done + echo "- **Platforms:** \`linux/amd64,linux/arm64\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Clean up + if: always() + run: docker buildx prune --filter "until=72h" -f || true + + cleanup-nightly: + needs: [resolve, build-and-publish] + # Any run that published a nightly-dev-efa-* tag, scheduled or manual. + if: needs.resolve.outputs.nightly == 'true' + uses: ./.github/workflows/_docker-cleanup-nightly.yml + with: + # The cleanup regex is "^-[0-9]", so this never touches the + # nightly-dev--* tags of the base workflow. + tag_prefixes: '["nightly-dev-efa"]' + secrets: inherit diff --git a/docker/efa.Dockerfile b/docker/efa.Dockerfile new file mode 100644 index 0000000000..1780ca6f0d --- /dev/null +++ b/docker/efa.Dockerfile @@ -0,0 +1,93 @@ +# AWS EFA variant of a published CUDA 13 SGLang image: adds the EFA userspace +# stack, the EFA Mooncake wheel (amd64 only), and EFA defaults for Mooncake, +# NIXL and NCCL. Published by .github/workflows/release-docker-efa.yml. +# +# docker build -f docker/efa.Dockerfile \ +# --build-arg BASE_IMAGE=lmsysorg/sglang:latest-runtime -t sglang:efa docker/ +# +# The host still provides the EFA kernel module and memlock limits. +ARG BASE_IMAGE=lmsysorg/sglang:latest-runtime +FROM ${BASE_IMAGE} + +ARG TARGETARCH +ARG EFA_INSTALLER_VERSION=1.50.0 +ARG EFA_INSTALLER_SHA256=fa6dff8593d866866c13cb4640d9059835cd4efa427971f100ab40c97bef2841 + +# Install the AWS EFA userspace stack. The distro libfabric1 stays (libopenmpi +# depends on it); the ENV below makes the AWS copy win at runtime. +RUN set -eux; \ + EFA_INSTALLER_ARCHIVE="/tmp/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz"; \ + curl -fsSL --retry 3 --retry-all-errors \ + "https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" \ + -o "${EFA_INSTALLER_ARCHIVE}"; \ + echo "${EFA_INSTALLER_SHA256} ${EFA_INSTALLER_ARCHIVE}" | sha256sum -c -; \ + tar -xzf "${EFA_INSTALLER_ARCHIVE}" -C /tmp; \ + # The installer apt-installs dependencies; the base image purged the lists. + apt-get update -y; \ + # Not --minimal: it skips libfabric, which the EFA data path needs. + (cd /tmp/aws-efa-installer && \ + ./efa_installer.sh -y --skip-kmod --skip-limit-conf --no-verify); \ + test -x /opt/amazon/efa/bin/fi_info; \ + /opt/amazon/efa/bin/fi_info --version; \ + # `-l` lists providers without EFA hardware, so this works in a build. + /opt/amazon/efa/bin/fi_info -l | grep -q '^efa:'; \ + ldconfig -p | grep -m1 'libfabric\.so\.1' | grep -q '/opt/amazon/efa/lib/'; \ + # NIXL's plugin needs FABRIC_1.7, which the distro libfabric 1.17.0 lacks. + NIXL_LIBFABRIC_PLUGINS="$(find "$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" \ + -name libplugin_LIBFABRIC.so)"; \ + test -n "${NIXL_LIBFABRIC_PLUGINS}"; \ + for plugin in ${NIXL_LIBFABRIC_PLUGINS}; do \ + ldd "${plugin}" | grep -E '^[[:space:]]*libfabric\.so\.1 => /opt/amazon/efa/lib/'; \ + done; \ + # Plugins that NCCL_NET_PLUGIN=ofi and NCCL_TUNER_PLUGIN=ofi resolve to. + ldconfig -p | grep -q '/opt/amazon/ofi-nccl/lib/libnccl-net-ofi\.so$'; \ + ldconfig -p | grep -q '/opt/amazon/ofi-nccl/lib/libnccl-tuner-ofi\.so$'; \ + apt-get clean; \ + rm -rf "${EFA_INSTALLER_ARCHIVE}" /tmp/aws-efa-installer \ + /var/lib/apt/lists/* + +# LD_LIBRARY_PATH is required: under `--gpus` the NVIDIA toolkit reruns +# ldconfig, which puts the distro libfabric first in the cache. +ENV LD_LIBRARY_PATH="/opt/amazon/efa/lib:${LD_LIBRARY_PATH}" \ + PATH="/opt/amazon/efa/bin:${PATH}" \ + NCCL_NET_PLUGIN=ofi \ + NCCL_TUNER_PLUGIN=ofi + +# Swap in the EFA Mooncake wheel at the base image's Mooncake version. Both +# wheels own the `mooncake` module; it is published for x86_64 only. +RUN --mount=type=cache,target=/root/.cache/pip \ + set -eux; \ + if [ "${TARGETARCH}" = "amd64" ]; then \ + pkg_version() { python3 -c 'import importlib.metadata as m, sys; print(m.version(sys.argv[1]))' "$1"; }; \ + MOONCAKE_VERSION="$(pkg_version mooncake-transfer-engine-cuda13)"; \ + test -n "${MOONCAKE_VERSION}"; \ + python3 -m pip uninstall -y mooncake-transfer-engine-cuda13; \ + python3 -m pip install --no-deps \ + "mooncake-transfer-engine-efa-cuda13==${MOONCAKE_VERSION}"; \ + test "$(pkg_version mooncake-transfer-engine-efa-cuda13)" = "${MOONCAKE_VERSION}"; \ + # `! cmd` is exempt from `set -e`, so check explicitly. + if pkg_version mooncake-transfer-engine-cuda13 >/dev/null 2>&1; then \ + echo "mooncake-transfer-engine-cuda13 is still installed" >&2; \ + exit 1; \ + fi; \ + else \ + echo "TARGETARCH=${TARGETARCH}: mooncake-transfer-engine-efa-cuda13 is \ +x86_64-only, keeping the standard wheel (Mooncake EFA protocol unavailable)"; \ + fi + +# Only amd64 has the EFA Mooncake wheel; other arches use tcp. ENV cannot be +# conditional, so this is "tcp if TARGETARCH is not amd64, else efa". +ARG ARCH_IF_NOT_AMD64=${TARGETARCH#amd64} +ARG PROTOCOL_IF_NOT_AMD64=${ARCH_IF_NOT_AMD64:+tcp} +ENV MOONCAKE_PROTOCOL=${PROTOCOL_IF_NOT_AMD64:-efa} \ + SGLANG_DISAGGREGATION_NIXL_BACKEND=LIBFABRIC + +# Replace the inherited image tag with this image's. The ARG has its own name +# because the inherited SGLANG_IMAGE_TAG ENV would shadow it. +ARG BASE_IMAGE +ARG EFA_IMAGE_TAG= +ENV SGLANG_IMAGE_TAG=${EFA_IMAGE_TAG:-${SGLANG_IMAGE_TAG}} +LABEL org.opencontainers.image.version="${SGLANG_IMAGE_TAG}" \ + ai.sglang.image.tag="${SGLANG_IMAGE_TAG}" \ + ai.sglang.efa.base-image="${BASE_IMAGE}" \ + ai.sglang.efa.installer-version="${EFA_INSTALLER_VERSION}" diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx index 154b4070b9..18881d3ce0 100644 --- a/docs/docs/advanced_features/pd_disaggregation.mdx +++ b/docs/docs/advanced_features/pd_disaggregation.mdx @@ -111,6 +111,49 @@ uv pip install mooncake-transfer-engine ### AWS EFA +#### EFA Docker images + +Each published CUDA 13 image has an EFA variant for EFA-enabled AWS EC2 instances, built from the same SGLang commit with `docker/efa.Dockerfile`: + +| Base image | EFA image | +|---|---| +| `latest`, `v` | `latest-efa`, `v-efa` | +| `latest-runtime`, `v-runtime` | `latest-runtime-efa`, `v-runtime-efa` | +| `nightly-dev--` | `nightly-dev-efa--`, plus the moving `dev-efa` | + +Each EFA image matches its base and adds: + +- The AWS EFA userspace stack: libfabric with the EFA provider, rdma-core, and the aws-ofi-nccl plugin. The host does not need its own EFA userspace install. +- On x86_64, the EFA build of Mooncake with `MOONCAKE_PROTOCOL=efa`. On arm64 the standard Mooncake package is kept and `MOONCAKE_PROTOCOL=tcp` is set, because the EFA build is x86_64-only. +- `SGLANG_DISAGGREGATION_NIXL_BACKEND=LIBFABRIC`, so NIXL transfers use EFA on both architectures. +- `NCCL_NET_PLUGIN=ofi` and `NCCL_TUNER_PLUGIN=ofi`, so NCCL uses aws-ofi-nccl. + +To add EFA to another CUDA 13 SGLang image, build `docker/efa.Dockerfile` on top of it: + +```bash +docker build -f docker/efa.Dockerfile \ + --build-arg BASE_IMAGE=lmsysorg/sglang:latest-runtime \ + -t sglang:efa docker/ +``` + +The host still needs the EFA kernel driver. Pass the EFA devices into the container and lift the locked-memory limit: + +```bash +HF_CACHE_DIR="/path/to/huggingface-cache" +docker run -it --gpus all \ + --network host \ + --ipc=host \ + $(for d in /dev/infiniband/uverbs*; do echo -n "--device $d "; done) \ + --ulimit memlock=-1 \ + -v "$HF_CACHE_DIR:/root/.cache/huggingface" \ + --env HF_TOKEN \ + lmsysorg/sglang:latest-runtime-efa bash +``` + +Inside the container, `fi_info -p efa -t FI_EP_RDM` lists the EFA devices. + +#### Install into your own environment + On an x86_64 AWS EFA instance, replace the standard CUDA 13 Mooncake package with the EFA build. The two packages install the same `mooncake` Python module, so do not install them side by side. The following commands replace the package in an SGLang CUDA 13 environment while preserving its existing Python dependencies: @@ -131,6 +174,10 @@ fi_info -p efa -t FI_EP_RDM The EFA wheel expects a compatible AWS EFA userspace installation on the host and does not bundle `libfabric` or `libefa`. The current wheel is available only for x86_64. It is built without Mooncake EP/PG and the NVLink/MNNVL components, so use the standard Mooncake CUDA package for those features. +#### CUDA VMM limitations + +These limitations apply whenever `MOONCAKE_PROTOCOL=efa`, including in the EFA Docker images. + The current libfabric EFA provider cannot transfer CUDA VMM-backed buffers. Until [libfabric issue #12853](https://github.com/ofiwg/libfabric/issues/12853) is resolved, apply these settings to both prefill and decode workers: - Do not set `SGLANG_MOONCAKE_CUSTOM_MEM_POOL` to `NVLINK` or `BAREX`. diff --git a/docs/docs/get-started/install.mdx b/docs/docs/get-started/install.mdx index 9065a2ddab..6466f6ab67 100644 --- a/docs/docs/get-started/install.mdx +++ b/docs/docs/get-started/install.mdx @@ -36,6 +36,7 @@ The examples use `latest`. To use another image, replace the tag in the pull and - Nightly: Built daily from the latest `main`, including changes not yet in a stable release. Choose a tag from the [nightly image list](https://hub.docker.com/r/lmsysorg/sglang/tags?name=nightly). - Runtime: A smaller image of the same stable release, with inference dependencies and fewer development utilities. Use the `latest-runtime` tag. +- AWS EFA: Any of the images above plus the AWS EFA userspace stack, for EFA-enabled EC2 instances. Add `-efa` to the tag: `latest-efa`, `latest-runtime-efa`, or the dated `nightly-dev-efa--` (moving tag `dev-efa`). See [AWS EFA](/docs/advanced_features/pd_disaggregation#aws-efa) for what the images include and how to run them. For reproducible deployments, use a specific release or dated nightly tag from [Docker Hub](https://hub.docker.com/r/lmsysorg/sglang/tags). diff --git a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py index e66b9ed301..917c1e721a 100644 --- a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py +++ b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py @@ -157,7 +157,10 @@ class MooncakeStoreConfig: "global_segment_size", envs.MOONCAKE_GLOBAL_SEGMENT_SIZE.default ) ), - protocol=config.get("protocol", envs.MOONCAKE_PROTOCOL.default), + # Fall back to MOONCAKE_PROTOCOL (not its default) so an image or + # deployment that sets it, e.g. MOONCAKE_PROTOCOL=efa, is honored + # when the config file omits "protocol". + protocol=config.get("protocol", envs.MOONCAKE_PROTOCOL.get()), device_name=config.get("device_name", envs.MOONCAKE_DEVICE.default), master_server_address=config.get( "master_server_address", envs.MOONCAKE_MASTER.default @@ -236,7 +239,8 @@ class MooncakeStoreConfig: "global_segment_size", envs.MOONCAKE_GLOBAL_SEGMENT_SIZE.default ) ), - protocol=extra_config.get("protocol", envs.MOONCAKE_PROTOCOL.default), + # See from_file(): honor MOONCAKE_PROTOCOL when "protocol" is omitted. + protocol=extra_config.get("protocol", envs.MOONCAKE_PROTOCOL.get()), device_name=extra_config.get("device_name", envs.MOONCAKE_DEVICE.default), master_server_address=extra_config.get( "master_server_address", envs.MOONCAKE_MASTER.default diff --git a/test/registered/unit/mem_cache/test_mooncake_store_config.py b/test/registered/unit/mem_cache/test_mooncake_store_config.py index 6b724cf82b..35f0e1ae4e 100644 --- a/test/registered/unit/mem_cache/test_mooncake_store_config.py +++ b/test/registered/unit/mem_cache/test_mooncake_store_config.py @@ -1,4 +1,4 @@ -"""Unit tests for MooncakeStoreConfig local_hostname resolution. +"""Unit tests for MooncakeStoreConfig local_hostname and protocol resolution. Regression for sgl-project/sglang#23457: runtime attach must resolve per-node local_hostname from process environment instead of a shared default. @@ -118,5 +118,45 @@ class TestMooncakeStoreConfigLocalHostname(CustomTestCase): os.unlink(config_path) +class TestMooncakeStoreConfigProtocol(CustomTestCase): + _BASE_CONFIG = {"master_server_address": "127.0.0.1:50051"} + + def _load_from_extra_config(self, extra_config=None): + config = {**self._BASE_CONFIG, **(extra_config or {})} + return MooncakeStoreConfig.load_from_extra_config(config) + + def _load_from_file(self, extra_config=None): + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fin: + json.dump({**self._BASE_CONFIG, **(extra_config or {})}, fin) + config_path = fin.name + try: + with temp_set_env( + allow_sglang=True, SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=config_path + ): + return MooncakeStoreConfig.from_file() + finally: + os.unlink(config_path) + + def test_load_from_extra_config_uses_protocol_env_when_omitted(self): + with temp_set_env(MOONCAKE_PROTOCOL="efa"): + cfg = self._load_from_extra_config() + self.assertEqual(cfg.protocol, "efa") + + def test_load_from_extra_config_prefers_explicit_protocol_over_env(self): + with temp_set_env(MOONCAKE_PROTOCOL="efa"): + cfg = self._load_from_extra_config({"protocol": "rdma"}) + self.assertEqual(cfg.protocol, "rdma") + + def test_from_file_uses_protocol_env_when_omitted(self): + with temp_set_env(MOONCAKE_PROTOCOL="efa"): + cfg = self._load_from_file() + self.assertEqual(cfg.protocol, "efa") + + def test_from_file_prefers_explicit_protocol_over_env(self): + with temp_set_env(MOONCAKE_PROTOCOL="efa"): + cfg = self._load_from_file({"protocol": "rdma"}) + self.assertEqual(cfg.protocol, "rdma") + + if __name__ == "__main__": unittest.main()