[CI] Add GB200 nightly perf regression pipeline (#22461)

.github/workflows/nightly-perf-regression-gb200.yml

@@ -0,0 +1,178 @@
+name: Nightly Perf Regression (GB200)
+
+# NOTE: This workflow is intentionally cron-only.
+# It must NOT be triggered manually (no workflow_dispatch) to prevent
+# individuals from queuing arbitrary jobs on the shared GB200 cluster.
+on:
+  schedule:
+    - cron: '0 2 * * *'  # 2 AM UTC daily (offset from other nightly runs)
+
+concurrency:
+  group: nightly-test-gb200
+  cancel-in-progress: false
+
+env:
+  SGLANG_IS_IN_CI: true
+  SRT_SLURM_BRANCH: sglang-nightly-regression
+  SLURM_PARTITION: batch
+  SLURM_ACCOUNT: sglang
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Reads scripts/ci/slurm/nightly-configs.yaml and generates one matrix entry
+  # per recipe YAML. Each job runs the full concurrency sweep defined in the
+  # recipe as a single Slurm job.
+  # To add/remove configs, edit nightly-configs.yaml only.
+  # ---------------------------------------------------------------------------
+  setup:
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate.outputs.matrix }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Generate benchmark matrix
+        id: generate
+        run: |
+          pip install pyyaml -q
+          MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py scripts/ci/slurm/nightly-configs.yaml --runner gb200)
+          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+
+  # ---------------------------------------------------------------------------
+  # Import Docker images to Lustre squash files once before all benchmark jobs.
+  # This avoids parallel jobs racing to enroot import the same image.
+  # ---------------------------------------------------------------------------
+  prepare-image:
+    needs: setup
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: gb200
+    outputs:
+      squash_file: ${{ steps.import.outputs.squash_file }}
+      nginx_squash_file: ${{ steps.import.outputs.nginx_squash_file }}
+    env:
+      IMAGE: lmsysorg/sglang:dev-cu13
+      NGINX_IMAGE: nginx:1.27.4
+    steps:
+      - name: Import Docker images to Lustre
+        id: import
+        run: |
+          SQUASH_FILE="/mnt/lustre01/users-public/sglang-ci/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g')_$(date +%Y%m%d).sqsh"
+          NGINX_SQUASH_FILE="/mnt/lustre01/users-public/sglang-ci/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
+
+          if [ -f "$SQUASH_FILE" ]; then
+            echo "Squash file already exists, skipping import: $SQUASH_FILE"
+          else
+            enroot import -o "$SQUASH_FILE" "docker://$IMAGE"
+          fi
+
+          if [ -f "$NGINX_SQUASH_FILE" ]; then
+            echo "Nginx squash file already exists, skipping import: $NGINX_SQUASH_FILE"
+          else
+            enroot import -o "$NGINX_SQUASH_FILE" "docker://$NGINX_IMAGE"
+          fi
+
+          echo "squash_file=$SQUASH_FILE" >> $GITHUB_OUTPUT
+          echo "nginx_squash_file=$NGINX_SQUASH_FILE" >> $GITHUB_OUTPUT
+
+  nightly-gb200-benchmark:
+    needs: [setup, prepare-image]
+    if: github.repository == 'sgl-project/sglang'
+    runs-on: gb200
+    strategy:
+      fail-fast: false
+      matrix:
+        config: ${{ fromJson(needs.setup.outputs.matrix) }}
+    env:
+      FRAMEWORK: dynamo-sglang
+      MODEL: ${{ matrix.config.model }}
+      MODEL_PREFIX: ${{ matrix.config.model_prefix }}
+      PRECISION: ${{ matrix.config.precision }}
+      ISL: ${{ matrix.config.isl }}
+      OSL: ${{ matrix.config.osl }}
+      CONFIG_FILE: ${{ matrix.config.config_file }}
+      RESULT_FILENAME: gb200-${{ matrix.config.name }}
+      SQUASH_FILE: ${{ needs.prepare-image.outputs.squash_file }}
+      NGINX_SQUASH_FILE: ${{ needs.prepare-image.outputs.nginx_squash_file }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Clean up prior Slurm jobs from this runner
+        continue-on-error: true
+        env:
+          RUNNER_NAME: ${{ runner.name }}
+        run: |
+          STALE_JOBS=$(squeue --noheader --format="%i %j" | grep "${RUNNER_NAME}" | awk '{print $1}')
+          if [ -n "$STALE_JOBS" ]; then
+            echo "Cancelling stale jobs: $STALE_JOBS"
+            scancel $STALE_JOBS
+          fi
+
+      - name: Launch GB200 benchmark via srt-slurm
+        timeout-minutes: 360
+        env:
+          RUNNER_NAME: ${{ runner.name }}
+        run: bash scripts/ci/slurm/launch_gb200.sh
+
+      - name: Process results
+        if: always()
+        env:
+          RUNNER_NAME: ${{ runner.name }}
+        run: |
+          pip install tabulate pyyaml -q
+          SRT_REPO_DIR="/mnt/lustre01/users-public/sglang-ci/workspace/${RUNNER_NAME}/srt-slurm"
+          for result_file in ${{ github.workspace }}/${RESULT_FILENAME}_*.json; do
+            [ -f "$result_file" ] || continue
+            basename_file=$(basename "$result_file")
+            ctx=$(echo "$basename_file" | sed -n 's/.*_ctx_\([0-9]*\)_gen.*/\1/p')
+            gen=$(echo "$basename_file" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p')
+            [ -n "$ctx" ] && [ -n "$gen" ] || continue
+            RESULT_FILENAME="${result_file%.json}" PREFILL_GPUS="$ctx" DECODE_GPUS="$gen" \
+              RECIPE_FILE="$SRT_REPO_DIR/$CONFIG_FILE" \
+              python3 scripts/ci/slurm/process_result.py
+          done
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gb200-${{ matrix.config.name }}-${{ github.run_id }}
+          path: |
+            ${{ github.workspace }}/*.json
+            ${{ github.workspace }}/multinode_server_logs.tar.gz
+          retention-days: 30
+          if-no-files-found: warn
+
+      - name: Clean up Slurm jobs on failure/cancel
+        if: failure() || cancelled()
+        continue-on-error: true
+        env:
+          RUNNER_NAME: ${{ runner.name }}
+        run: |
+          ACTIVE_JOBS=$(squeue --noheader --format="%i %j" | grep "${RUNNER_NAME}" | awk '{print $1}')
+          if [ -n "$ACTIVE_JOBS" ]; then
+            echo "Cancelling jobs: $ACTIVE_JOBS"
+            scancel $ACTIVE_JOBS
+          fi
+
+  collect-results:
+    needs: nightly-gb200-benchmark
+    if: github.repository == 'sgl-project/sglang' && always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: results/
+          pattern: gb200-*
+
+      - name: Print summary
+        run: |
+          pip install tabulate -q
+          python3 scripts/ci/slurm/summarize.py results/ >> $GITHUB_STEP_SUMMARY

scripts/ci/slurm/generate_matrix.py

@@ -0,0 +1,73 @@
+"""
+Reads nightly-configs.yaml and generates one matrix entry per recipe YAML,
+where each srt-slurm recipe runs its full concurrency sweep as a single Slurm job.
+
+conc-list in the config is documentation only and is not used to split jobs.
+
+Output: JSON array written to stdout, consumed by the workflow setup job as
+a dynamic matrix via fromJson(needs.setup.outputs.matrix).
+
+Usage:
+    python3 generate_matrix.py <path-to-nightly-configs.yaml> --runner <label>
+
+Example:
+    python3 generate_matrix.py scripts/ci/slurm/nightly-configs.yaml --runner gb200
+"""
+
+import argparse
+import json
+
+import yaml
+
+
+def seq_len_str(isl, osl):
+    def fmt(n):
+        return f"{n // 1024}k" if n % 1024 == 0 else str(n)
+
+    return f"{fmt(isl)}{fmt(osl)}"
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("config_file", help="Path to nightly-configs.yaml")
+    parser.add_argument(
+        "--runner",
+        required=True,
+        help="Filter configs by runner label (e.g. gb200, b200)",
+    )
+    args = parser.parse_args()
+
+    with open(args.config_file) as f:
+        data = yaml.safe_load(f)
+
+    matrix = []
+    for exp_name, exp in data.items():
+        if exp["runner"] != args.runner:
+            continue
+
+        for seq_cfg in exp["seq-len-configs"]:
+            isl, osl = seq_cfg["isl"], seq_cfg["osl"]
+            sl = seq_len_str(isl, osl)
+
+            for entry in seq_cfg["search-space"]:
+                config_file = entry["config_file"]
+                topology = config_file.rsplit("/", 1)[-1].replace(".yaml", "")
+
+                matrix.append(
+                    {
+                        "name": f"{exp['model-prefix']}-{exp['precision']}-{sl}-{topology}",
+                        "exp_name": exp_name,
+                        "model": exp["model"],
+                        "model_prefix": exp["model-prefix"],
+                        "precision": exp["precision"],
+                        "isl": str(isl),
+                        "osl": str(osl),
+                        "config_file": config_file,
+                    }
+                )
+
+    print(json.dumps(matrix))
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/slurm/launch_gb200.sh

@@ -0,0 +1,213 @@
+#!/usr/bin/env bash
+# Launch a dynamo-sglang benchmark job on the GB200 cluster via srt-slurm.
+#
+# Required environment variables (set by the GitHub Actions workflow):
+#   FRAMEWORK         - must be "dynamo-sglang"
+#   MODEL             - HuggingFace model ID (used as fallback if no local path)
+#   MODEL_PREFIX      - short prefix: "dsr1"
+#   PRECISION         - "fp8" or "fp4"
+#   ISL               - input sequence length (e.g. "1024")
+#   OSL               - output sequence length (e.g. "1024")
+#   CONFIG_FILE       - path relative to srt-slurm repo root (e.g. recipes/gb200-fp8/1k1k/low-latency.yaml)
+#   RESULT_FILENAME   - prefix for output JSON filenames
+#   RUNNER_NAME       - GitHub Actions runner name (used to tag the Slurm job)
+#   SQUASH_FILE       - path to pre-imported sglang enroot squash file on Lustre
+#   NGINX_SQUASH_FILE - path to pre-imported nginx enroot squash file on Lustre
+#   SLURM_PARTITION   - Slurm partition (default: batch)
+#   SLURM_ACCOUNT     - Slurm account  (default: sglang)
+#   SRT_SLURM_BRANCH  - branch of srt-slurm repo to check out
+#   GITHUB_WORKSPACE  - set automatically by GitHub Actions
+
+set -euo pipefail
+set -x
+
+# ---------------------------------------------------------------------------
+# Validate required vars
+# ---------------------------------------------------------------------------
+: "${FRAMEWORK:?}"
+: "${MODEL_PREFIX:?}"
+: "${PRECISION:?}"
+: "${ISL:?}"
+: "${OSL:?}"
+: "${CONFIG_FILE:?}"
+: "${RESULT_FILENAME:?}"
+: "${RUNNER_NAME:?}"
+: "${SQUASH_FILE:?}"
+: "${NGINX_SQUASH_FILE:?}"
+: "${GITHUB_WORKSPACE:?}"
+
+SLURM_PARTITION="${SLURM_PARTITION:-batch}"
+SLURM_ACCOUNT="${SLURM_ACCOUNT:-sglang}"
+SRT_SLURM_BRANCH="${SRT_SLURM_BRANCH:-sglang-nightly-regression}"
+
+# ---------------------------------------------------------------------------
+# Resolve local model paths on Lustre (avoids re-downloading on each run)
+# ---------------------------------------------------------------------------
+if [[ "$MODEL_PREFIX" == "dsr1" && "$PRECISION" == "fp8" ]]; then
+    MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528"
+    SRT_SLURM_MODEL_PREFIX="dsr1-fp8"
+elif [[ "$MODEL_PREFIX" == "dsr1" && "$PRECISION" == "fp4" ]]; then
+    MODEL_PATH="/mnt/lustre01/models/deepseek-r1-0528-fp4-v2/"
+    SRT_SLURM_MODEL_PREFIX="dsr1-fp4"
+else
+    MODEL_PATH="$MODEL"
+    SRT_SLURM_MODEL_PREFIX="$MODEL_PREFIX"
+fi
+
+# ---------------------------------------------------------------------------
+# Set up per-runner Lustre workspace (cleaned before each run, accessible
+# to both the runner and compute nodes)
+# ---------------------------------------------------------------------------
+LUSTRE_WORKSPACE="/mnt/lustre01/users-public/sglang-ci/workspace/${RUNNER_NAME}"
+rm -rf "$LUSTRE_WORKSPACE"
+mkdir -p "$LUSTRE_WORKSPACE"
+
+# ---------------------------------------------------------------------------
+# Clone and set up srt-slurm
+# ---------------------------------------------------------------------------
+SRT_REPO_DIR="$LUSTRE_WORKSPACE/srt-slurm"
+
+git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
+cd "$SRT_REPO_DIR"
+git checkout "$SRT_SLURM_BRANCH"
+echo "--- srt-slurm last commit ---"
+git log -1 --format="commit %H%nauthor %an%ndate   %ad%nsubject %s" --date=iso
+
+curl -LsSf https://astral.sh/uv/install.sh | sh
+source "$HOME/.local/bin/env"
+
+uv venv
+source .venv/bin/activate
+uv pip install -e .
+
+if ! command -v srtctl &>/dev/null; then
+    echo "ERROR: srtctl installation failed"
+    exit 1
+fi
+
+# ---------------------------------------------------------------------------
+# Generate srtslurm.yaml
+# ---------------------------------------------------------------------------
+SRTCTL_ROOT="$SRT_REPO_DIR"
+
+cat > srtslurm.yaml <<EOF
+# SRT SLURM configuration for SGLang GB200 nightly CI
+default_account: "${SLURM_ACCOUNT}"
+default_partition: "${SLURM_PARTITION}"
+default_time_limit: "6:00:00"
+
+gpus_per_node: 4
+network_interface: ""
+
+srtctl_root: "${SRTCTL_ROOT}"
+
+model_paths:
+  "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}"
+
+containers:
+  dynamo-sglang: ${SQUASH_FILE}
+  nginx: ${NGINX_SQUASH_FILE}
+  nginx-sqsh: ${NGINX_SQUASH_FILE}
+EOF
+
+echo "--- srtslurm.yaml ---"
+cat srtslurm.yaml
+
+make setup ARCH=aarch64
+
+# ---------------------------------------------------------------------------
+# Patch job name and submit via srtctl
+# ---------------------------------------------------------------------------
+sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE"
+
+SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" \
+    --tags "gb200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},sglang-nightly-$(date +%Y%m%d)" \
+    --setup-script install-torchao.sh 2>&1)
+echo "$SRTCTL_OUTPUT"
+
+JOB_ID=$(echo "$SRTCTL_OUTPUT" | grep -oP '✅ Job \K[0-9]+' || echo "$SRTCTL_OUTPUT" | grep -oP 'Job \K[0-9]+' || true)
+
+if [ -z "$JOB_ID" ]; then
+    echo "ERROR: Could not extract JOB_ID from srtctl output"
+    exit 1
+fi
+
+echo "Submitted Slurm job: $JOB_ID"
+
+set +x
+
+# ---------------------------------------------------------------------------
+# Wait for job and stream logs
+# ---------------------------------------------------------------------------
+LOGS_DIR="outputs/$JOB_ID/logs"
+LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log"
+
+mkdir -p "$LOGS_DIR"
+
+while ! ls "$LOG_FILE" &>/dev/null; do
+    if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then
+        echo "ERROR: Job $JOB_ID failed before creating log file"
+        scontrol show job "$JOB_ID" || true
+        exit 1
+    fi
+    echo "Waiting for job $JOB_ID to start and $LOG_FILE to appear..."
+    sleep 5
+done
+
+(
+    while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do
+        sleep 10
+    done
+) &
+POLL_PID=$!
+
+tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
+
+wait $POLL_PID
+
+set -x
+
+echo "Job $JOB_ID completed. Collecting results..."
+
+# ---------------------------------------------------------------------------
+# Collect results
+# ---------------------------------------------------------------------------
+if [ ! -d "$LOGS_DIR" ]; then
+    echo "WARNING: Logs directory not found at $LOGS_DIR"
+    exit 1
+fi
+
+cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS"
+tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" .
+
+RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null || true)
+
+if [ -z "$RESULT_SUBDIRS" ]; then
+    echo "ERROR: No result subdirectories found in $LOGS_DIR — benchmark did not produce any output"
+    exit 1
+else
+    RESULT_COUNT=0
+    for result_subdir in $RESULT_SUBDIRS; do
+        CONFIG_NAME=$(basename "$result_subdir")
+        RESULT_FILES=$(find "$result_subdir" -name "results_concurrency_*.json" 2>/dev/null || true)
+        for result_file in $RESULT_FILES; do
+            if [ -f "$result_file" ]; then
+                filename=$(basename "$result_file")
+                concurrency=$(echo "$filename" | sed -n 's/results_concurrency_\([0-9]*\)_gpus_.*/\1/p')
+                gpus=$(echo "$filename" | sed -n 's/results_concurrency_[0-9]*_gpus_\([0-9]*\)_ctx_.*/\1/p')
+                ctx=$(echo "$filename" | sed -n 's/.*_ctx_\([0-9]*\)_gen_.*/\1/p')
+                gen=$(echo "$filename" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p')
+                DEST="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${CONFIG_NAME}_conc${concurrency}_gpus_${gpus}_ctx_${ctx}_gen_${gen}.json"
+                cp "$result_file" "$DEST"
+                echo "Saved: $DEST"
+                RESULT_COUNT=$((RESULT_COUNT + 1))
+            fi
+        done
+    done
+    if [ "$RESULT_COUNT" -eq 0 ]; then
+        echo "ERROR: Result subdirectories found but no result JSON files produced — benchmark failed"
+        exit 1
+    fi
+fi
+
+echo "Done."

scripts/ci/slurm/nightly-configs.yaml

@@ -0,0 +1,46 @@
+# Nightly benchmark configurations for srt-slurm powered runners.
+#
+# Structure mirrors InferenceX nvidia-master.yaml but only includes fields
+# actually needed by the runner — prefill/decode topology details are already
+# encoded in each srt-slurm recipe YAML and are not duplicated here.
+#
+# To add/remove concurrencies: edit conc-list for the relevant search-space entry.
+# To add a new runner:         add a new top-level block and create a corresponding
+#                              nightly-test-<runner>.yml workflow.
+# Never edit workflow YAML files directly for these changes.
+
+dsr1-fp8-gb200-dynamo-sglang:
+  model: deepseek-ai/DeepSeek-R1-0528
+  model-prefix: dsr1
+  runner: gb200
+  precision: fp8
+  framework: dynamo-sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1024, 2048, 4096, 6144]
+          # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp8/1k1k/max-tpt.yaml
+          config_file: recipes/gb200-fp8/1k1k/max-tpt.yaml
+
+        - conc-list: [4096]
+          # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp8/1k1k/ultra-tpt.yaml
+          config_file: recipes/gb200-fp8/1k1k/ultra-tpt.yaml
+
+dsr1-fp4-gb200-dynamo-sglang:
+  model: nvidia/DeepSeek-R1-0528-NVFP4-v2
+  model-prefix: dsr1
+  runner: gb200
+  precision: fp4
+  framework: dynamo-sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [512, 2048, 4096, 8192]
+          # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp4/1k1k/mid-curve.yaml
+          config_file: recipes/gb200-fp4/1k1k/mid-curve.yaml

scripts/ci/slurm/process_result.py

@@ -0,0 +1,117 @@
+"""Process a raw srt-slurm benchmark result JSON into an aggregated format.
+
+Usage (called once per result file):
+    RESULT_FILENAME=<path_without_.json> PREFILL_GPUS=<n> DECODE_GPUS=<n> \\
+        RECIPE_FILE=<path_to_recipe.yaml> python3 process_result.py
+
+Required env vars:
+    RESULT_FILENAME   - path to the result file without the .json extension
+    FRAMEWORK         - e.g. dynamo-sglang
+    PRECISION         - e.g. fp8, fp4
+    MODEL_PREFIX      - short model label, e.g. dsr1
+    ISL               - input sequence length
+    OSL               - output sequence length
+    PREFILL_GPUS      - number of prefill GPUs (extracted from result filename)
+    DECODE_GPUS       - number of decode GPUs (extracted from result filename)
+
+Optional env vars:
+    RECIPE_FILE       - path to the srt-slurm recipe YAML; if set, topology
+                        fields (TP, EP, DP, workers) are parsed from it
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def require(var):
+    val = os.environ.get(var)
+    if val is None:
+        print(f"ERROR: Missing required env var: {var}", file=sys.stderr)
+        sys.exit(1)
+    return val
+
+
+result_filename = require("RESULT_FILENAME")
+framework = require("FRAMEWORK")
+precision = require("PRECISION")
+model_prefix = require("MODEL_PREFIX")
+isl = int(require("ISL"))
+osl = int(require("OSL"))
+prefill_gpus = int(require("PREFILL_GPUS"))
+decode_gpus = int(require("DECODE_GPUS"))
+
+with open(f"{result_filename}.json") as f:
+    raw = json.load(f)
+
+# ---------------------------------------------------------------------------
+# Topology — parse from recipe YAML if available, otherwise default to 0/"-"
+# ---------------------------------------------------------------------------
+prefill_tp = prefill_ep = prefill_dp_attn = 0
+prefill_num_workers = decode_tp = decode_ep = decode_dp_attn = decode_num_workers = 0
+
+recipe_file = os.environ.get("RECIPE_FILE")
+if recipe_file and Path(recipe_file).exists():
+    import yaml
+
+    with open(recipe_file) as f:
+        recipe = yaml.safe_load(f)
+
+    res = recipe.get("resources", {})
+    prefill_num_workers = res.get("prefill_workers", 0)
+    decode_num_workers = res.get("decode_workers", 0)
+
+    sgl = recipe.get("backend", {}).get("sglang_config", {})
+    p = sgl.get("prefill", {})
+    d = sgl.get("decode", {})
+
+    prefill_tp = p.get("tensor-parallel-size", 0)
+    prefill_ep = p.get("expert-parallel-size", 0)
+    prefill_dp_attn = p.get("data-parallel-size", "-")
+    decode_tp = d.get("tensor-parallel-size", 0)
+    decode_ep = d.get("expert-parallel-size", 0)
+    decode_dp_attn = d.get("data-parallel-size", "-")
+
+total_gpus = prefill_gpus + decode_gpus
+
+data = {
+    "hw": "gb200",
+    "conc": int(raw["max_concurrency"]),
+    "model": raw["model_id"],
+    "infmax_model_prefix": model_prefix,
+    "framework": framework,
+    "precision": precision,
+    "isl": isl,
+    "osl": osl,
+    "is_multinode": True,
+    "disagg": True,
+    "num_prefill_gpu": prefill_gpus,
+    "num_decode_gpu": decode_gpus,
+    "prefill_num_workers": prefill_num_workers,
+    "prefill_tp": prefill_tp,
+    "prefill_ep": prefill_ep,
+    "prefill_dp_attention": prefill_dp_attn,
+    "decode_num_workers": decode_num_workers,
+    "decode_tp": decode_tp,
+    "decode_ep": decode_ep,
+    "decode_dp_attention": decode_dp_attn,
+    "tput_per_gpu": float(raw["total_token_throughput"]) / total_gpus,
+    "output_tput_per_gpu": float(raw["output_throughput"]) / decode_gpus,
+    "input_tput_per_gpu": (
+        float(raw["total_token_throughput"]) - float(raw["output_throughput"])
+    )
+    / prefill_gpus,
+}
+
+for key, value in raw.items():
+    if key.endswith("_ms"):
+        data[key.replace("_ms", "")] = float(value) / 1000.0
+    if "tpot" in key:
+        data[key.replace("_ms", "").replace("tpot", "intvty")] = 1000.0 / float(value)
+
+out_path = Path(result_filename).parent / f"agg_{Path(result_filename).name}.json"
+with open(out_path, "w") as f:
+    json.dump(data, f, indent=2)
+
+print(f"Written: {out_path}")

scripts/ci/slurm/summarize.py

@@ -0,0 +1,122 @@
+"""Print a markdown summary table from processed benchmark results.
+
+Usage:
+    python3 summarize.py <results_dir>
+
+Reads all agg_*.json files recursively from <results_dir> and prints a
+markdown table to stdout (redirect to $GITHUB_STEP_SUMMARY to publish).
+"""
+
+import json
+import sys
+from pathlib import Path
+
+from tabulate import tabulate
+
+HEADERS = [
+    "Model",
+    "Served Model",
+    "Hardware",
+    "Framework",
+    "Precision",
+    "ISL",
+    "OSL",
+    "Prefill TP",
+    "Prefill EP",
+    "Prefill DP Attn",
+    "Prefill Workers",
+    "Prefill GPUs",
+    "Decode TP",
+    "Decode EP",
+    "Decode DP Attn",
+    "Decode Workers",
+    "Decode GPUs",
+    "Conc",
+    "TTFT (ms)",
+    "TPOT (ms)",
+    "Interactivity (tok/s/user)",
+    "E2EL (s)",
+    "TPUT per GPU",
+    "Output TPUT per GPU",
+    "Input TPUT per GPU",
+]
+
+
+def load_json(path):
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except Exception:
+        return None
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python3 summarize.py <results_dir>")
+        sys.exit(1)
+
+    results_dir = Path(sys.argv[1])
+    results = [
+        r
+        for path in results_dir.rglob("agg_*.json")
+        if (r := load_json(path)) and "is_multinode" in r
+    ]
+
+    if not results:
+        print("No processed result files found.")
+        return
+
+    results.sort(
+        key=lambda r: (
+            r["infmax_model_prefix"],
+            r["hw"],
+            r["framework"],
+            r["precision"],
+            r["isl"],
+            r["osl"],
+            r["prefill_tp"],
+            r["prefill_ep"],
+            r["decode_tp"],
+            r["decode_ep"],
+            r["conc"],
+        )
+    )
+
+    rows = [
+        [
+            r["infmax_model_prefix"],
+            r["model"],
+            r["hw"].upper(),
+            r["framework"].upper(),
+            r["precision"].upper(),
+            r["isl"],
+            r["osl"],
+            r["prefill_tp"],
+            r["prefill_ep"],
+            r["prefill_dp_attention"],
+            r["prefill_num_workers"],
+            r["num_prefill_gpu"],
+            r["decode_tp"],
+            r["decode_ep"],
+            r["decode_dp_attention"],
+            r["decode_num_workers"],
+            r["num_decode_gpu"],
+            r["conc"],
+            f"{r['median_ttft'] * 1000:.4f}",
+            f"{r['median_tpot'] * 1000:.4f}",
+            f"{r['median_intvty']:.4f}",
+            f"{r['median_e2el']:.4f}",
+            f"{r['tput_per_gpu']:.4f}",
+            f"{r['output_tput_per_gpu']:.4f}",
+            f"{r['input_tput_per_gpu']:.4f}",
+        ]
+        for r in results
+    ]
+
+    print("## GB200 Nightly Benchmark Results\n")
+    print(tabulate(rows, headers=HEADERS, tablefmt="github"))
+    print()
+
+
+if __name__ == "__main__":
+    main()