Upload nightly test metrics to GH artifacts (#17696)

.github/workflows/nightly-test-nvidia.yml

@@ -134,6 +134,26 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error
 
+      - name: Collect performance metrics
+        if: always()
+        run: |
+          python3 scripts/ci/save_metrics.py \
+            --gpu-config 8-gpu-h200 \
+            --partition ${{ matrix.partition }} \
+            --run-id ${{ github.run_id }} \
+            --output test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json \
+            --search-dir test/performance_profiles_8_gpu \
+            --search-dir test
+
+      - name: Upload partition metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-8gpu-h200-partition-${{ matrix.partition }}
+          path: test/metrics-8gpu-h200-partition-${{ matrix.partition }}.json
+          retention-days: 5
+          if-no-files-found: ignore
+
   # General tests - 8 GPU H20
   nightly-test-general-8-gpu-h20:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
@@ -187,6 +207,26 @@ jobs:
           cd test
           IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error --auto-partition-id=${{ matrix.partition }} --auto-partition-size=3
 
+      - name: Collect performance metrics
+        if: always()
+        run: |
+          python3 scripts/ci/save_metrics.py \
+            --gpu-config 8-gpu-b200 \
+            --partition ${{ matrix.partition }} \
+            --run-id ${{ github.run_id }} \
+            --output test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json \
+            --search-dir test/performance_profiles_8_gpu \
+            --search-dir test
+
+      - name: Upload partition metrics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: metrics-8gpu-b200-partition-${{ matrix.partition }}
+          path: test/metrics-8gpu-b200-partition-${{ matrix.partition }}.json
+          retention-days: 5
+          if-no-files-found: ignore
+
   # Text model accuracy tests
   nightly-test-text-accuracy-2-gpu-runner:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-runner')
@@ -403,6 +443,48 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-8-gpu-b200 --nightly --continue-on-error --timeout-per-file 2400
 
+  # Consolidate performance metrics from all 8-GPU jobs
+  consolidate-metrics:
+    if: github.repository == 'sgl-project/sglang' && always()
+    needs:
+      - nightly-test-general-8-gpu-h200
+      - nightly-test-general-8-gpu-b200
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Download all partition metrics
+        uses: actions/download-artifact@v4
+        with:
+          pattern: metrics-*
+          path: metrics/
+          merge-multiple: true
+
+      - name: List downloaded metrics
+        run: |
+          echo "Downloaded metrics files:"
+          find metrics/ -name "*.json" -type f 2>/dev/null || echo "No metrics files found"
+
+      - name: Merge metrics
+        run: |
+          python3 scripts/ci/merge_metrics.py \
+            --input-dir metrics/ \
+            --output consolidated-metrics-${{ github.run_id }}.json \
+            --run-id ${{ github.run_id }} \
+            --commit-sha ${{ github.sha }} \
+            --branch ${{ github.ref_name }}
+
+      - name: Upload consolidated metrics
+        uses: actions/upload-artifact@v4
+        with:
+          name: consolidated-metrics-${{ github.run_id }}
+          path: consolidated-metrics-${{ github.run_id }}.json
+          retention-days: 90
+          if-no-files-found: warn
+
   # Final check job
   check-all-jobs:
     if: github.repository == 'sgl-project/sglang' && always()
@@ -420,6 +502,7 @@ jobs:
       - nightly-test-multimodal-server-2-gpu
       - nightly-test-perf-4-gpu-b200
       - nightly-test-specialized-8-gpu-b200
+      - consolidate-metrics
     runs-on: ubuntu-latest
     steps:
       - name: Check if any job failed

python/sglang/bench_one_batch_server.py

@@ -31,6 +31,7 @@ def run_benchmark(server_args: ServerArgs, bench_args: BenchArgs):
             results,
             pydantic_result_filename=bench_args.pydantic_result_filename,
             model_path=server_args.model_path,
+            server_args=bench_args.server_args_for_metrics,
         )
 
     return results, server_info

python/sglang/test/bench_one_batch_server_internal.py

@@ -109,6 +109,7 @@ class BenchArgs:
     append_to_github_summary: bool = True
     seed: int = 42
     cache_hit_rate: float = 0.0
+    server_args_for_metrics: Optional[List[str]] = None
 
     @staticmethod
     def add_cli_args(parser: argparse.ArgumentParser):
@@ -192,6 +193,13 @@ class BenchArgs:
             help="Cache hit rate for benchmarking (0.0-1.0). "
             "0.0 means no cache hits (flush all), 0.4 means 40%% of input tokens are cached.",
         )
+        parser.add_argument(
+            "--server-args-for-metrics",
+            type=str,
+            nargs="*",
+            default=None,
+            help="Server launch arguments to record in metrics output (for tracking configurations).",
+        )
 
     @classmethod
     def from_cli_args(cls, args: argparse.Namespace):

python/sglang/test/nightly_bench_utils.py

@@ -31,6 +31,7 @@ class BenchmarkResult(BaseModel):
     acc_length: Optional[float] = None
     profile_link_extend: Optional[str] = None
     profile_link_decode: Optional[str] = None
+    server_args: Optional[List[str]] = None
 
     @staticmethod
     def help_str() -> str:
@@ -122,7 +123,10 @@ def generate_markdown_report(
 
 
 def save_results_as_pydantic_models(
-    results: List, pydantic_result_filename: str, model_path: str
+    results: List,
+    pydantic_result_filename: str,
+    model_path: str,
+    server_args: Optional[List[str]] = None,
 ):
     """Save benchmark results as JSON using Pydantic models."""
     json_results = []
@@ -167,6 +171,7 @@ def save_results_as_pydantic_models(
             acc_length=res.acc_length,
             profile_link_extend=profile_link_extend,
             profile_link_decode=profile_link_decode,
+            server_args=server_args,
         )
         json_results.append(benchmark_result.model_dump())
 

python/sglang/test/nightly_utils.py

@@ -93,6 +93,7 @@ class NightlyBenchmarkRunner:
         profile_path_prefix: str,
         json_output_file: str,
         extra_args: Optional[List[str]] = None,
+        server_args: Optional[List[str]] = None,
     ) -> List[str]:
         """Build the benchmark command with all required arguments.
 
@@ -104,6 +105,7 @@ class NightlyBenchmarkRunner:
             profile_path_prefix: Prefix for profile output files
             json_output_file: Path to JSON output file
             extra_args: Optional extra arguments to append to command
+            server_args: Optional server launch arguments to record in metrics
 
         Returns:
             List of command arguments ready for subprocess.run()
@@ -135,6 +137,11 @@ class NightlyBenchmarkRunner:
         if extra_args:
             command.extend(extra_args)
 
+        # Record server launch arguments in metrics for tracking configurations
+        if server_args:
+            command.append("--server-args-for-metrics")
+            command.extend(server_args)
+
         return command
 
     def run_benchmark_command(
@@ -192,17 +199,14 @@ class NightlyBenchmarkRunner:
                 f"Loaded {len(benchmark_results)} benchmark results from {json_output_file}"
             )
 
-            # Clean up JSON file
+            # Note: JSON files are preserved for metrics collection by CI scripts
-            os.remove(json_output_file)
+            # They will be collected by scripts/ci/save_metrics.py
 
             return benchmark_results, True
 
         except Exception as e:
             desc = model_description or "model"
             print(f"Error loading benchmark results for {desc}: {e}")
-            # Try to clean up the file anyway
-            if os.path.exists(json_output_file):
-                os.remove(json_output_file)
             return benchmark_results, False
 
     def run_benchmark_for_model(
@@ -268,6 +272,7 @@ class NightlyBenchmarkRunner:
                 profile_path_prefix,
                 json_output_file,
                 extra_args=bench_args,
+                server_args=other_args,
             )
 
             result, cmd_success = self.run_benchmark_command(command, model_description)

python/sglang/test/performance_test_runner.py

@@ -77,6 +77,7 @@ def run_performance_test(
             input_lens=input_lens,
             output_lens=output_lens,
             other_args=model.extra_args,
+            variant=model.variant or "",
             extra_bench_args=extra_bench_args,
         )
 

scripts/ci/merge_metrics.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Merge per-partition metrics into a consolidated metrics file.
+
+This script reads all per-partition metric JSON files and consolidates them
+into a single JSON file with run-level metadata.
+
+Usage:
+    python3 scripts/ci/merge_metrics.py \
+        --input-dir metrics/ \
+        --output consolidated-metrics-12345678.json \
+        --run-id 12345678 \
+        --commit-sha abc123def456
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_partition_files(input_dir: str) -> list[str]:
+    """Find all partition metric files in the input directory."""
+    pattern = os.path.join(input_dir, "**/metrics-*.json")
+    return glob.glob(pattern, recursive=True)
+
+
+def load_partition_metrics(filepath: str) -> dict | None:
+    """Load a partition metrics file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to load {filepath}: {e}")
+        return None
+
+
+def merge_metrics(
+    input_dir: str,
+    output_file: str,
+    run_id: str,
+    commit_sha: str,
+    branch: str | None = None,
+) -> bool:
+    """Merge all partition metrics into a consolidated file."""
+    run_date = datetime.now(timezone.utc).isoformat()
+
+    # Find all partition files
+    partition_files = find_partition_files(input_dir)
+    print(f"Found {len(partition_files)} partition file(s)")
+
+    all_results = []
+    if not partition_files:
+        print("No partition metrics files found")
+    else:
+        # Load all partition files
+        for filepath in sorted(partition_files):
+            print(f"  Reading: {filepath}")
+            metrics = load_partition_metrics(filepath)
+            if metrics and "results" in metrics:
+                all_results.extend(metrics["results"])
+        print(f"Total results collected: {len(all_results)}")
+
+    # Create consolidated structure
+    consolidated = {
+        "run_id": run_id,
+        "run_date": run_date,
+        "commit_sha": commit_sha,
+        "branch": branch,
+        "results": all_results,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(consolidated, f, indent=2)
+
+        if not partition_files:
+            print(f"Created empty consolidated file: {output_file}")
+        else:
+            print(f"Saved consolidated metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing consolidated file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Merge per-partition metrics into consolidated file"
+    )
+    parser.add_argument(
+        "--input-dir",
+        required=True,
+        help="Directory containing partition metric files",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for consolidated metrics JSON",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--commit-sha",
+        required=True,
+        help="Git commit SHA",
+    )
+    parser.add_argument(
+        "--branch",
+        default=None,
+        help="Git branch name (optional)",
+    )
+
+    args = parser.parse_args()
+
+    success = merge_metrics(
+        input_dir=args.input_dir,
+        output_file=args.output,
+        run_id=args.run_id,
+        commit_sha=args.commit_sha,
+        branch=args.branch,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/save_metrics.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Collect and save performance metrics from nightly benchmark results.
+
+This script reads benchmark result JSON files from performance profile directories
+and saves them with metadata for artifact collection in CI.
+
+Usage:
+    python3 scripts/ci/save_metrics.py \
+        --gpu-config 8-gpu-h200 \
+        --partition 0 \
+        --run-id 12345678 \
+        --output test/metrics-8gpu-h200-partition-0.json
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def find_result_files(search_dirs: list[str]) -> list[str]:
+    """Find all results_*.json files in the given directories."""
+    result_files = set()
+    for search_dir in search_dirs:
+        if os.path.exists(search_dir):
+            pattern = os.path.join(search_dir, "**/results_*.json")
+            result_files.update(glob.glob(pattern, recursive=True))
+    return list(result_files)
+
+
+def parse_result_file(filepath: str) -> list[dict]:
+    """Parse a benchmark result JSON file."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        if isinstance(data, list):
+            return data
+        return [data]
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Warning: Failed to parse {filepath}: {e}")
+        return []
+
+
+def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> dict:
+    """Transform a benchmark result to the metrics schema."""
+    # Handle None values safely for numeric conversions
+    latency = result.get("latency")
+    last_ttft = result.get("last_ttft")
+
+    return {
+        "batch_size": result.get("batch_size"),
+        "input_len": result.get("input_len"),
+        "output_len": result.get("output_len"),
+        "latency_ms": latency * 1000 if latency is not None else None,
+        "input_throughput": result.get("input_throughput"),
+        "output_throughput": result.get("output_throughput"),
+        "overall_throughput": result.get("overall_throughput"),
+        "ttft_ms": last_ttft * 1000 if last_ttft is not None else None,
+        "acc_length": result.get("acc_length"),
+    }
+
+
+def group_results_by_model(
+    results: list[dict], gpu_config: str, partition: int
+) -> list[dict]:
+    """Group benchmark results by model, variant, and server_args."""
+    groups = {}
+
+    for result in results:
+        model_path = result.get("model_path", "unknown")
+        run_name = result.get("run_name", "default")
+        variant = run_name if run_name != "default" else None
+        server_args = result.get("server_args")
+        # Convert server_args list to tuple for use as dict key (lists are not hashable)
+        server_args_key = tuple(server_args) if server_args else None
+
+        key = (model_path, variant, server_args_key)
+        if key not in groups:
+            groups[key] = {
+                "gpu_config": gpu_config,
+                "partition": partition,
+                "model": model_path,
+                "variant": variant,
+                "server_args": server_args,
+                "benchmarks": [],
+            }
+
+        groups[key]["benchmarks"].append(
+            transform_benchmark_result(result, gpu_config, partition)
+        )
+
+    return list(groups.values())
+
+
+def save_metrics(
+    gpu_config: str,
+    partition: int,
+    run_id: str,
+    output_file: str,
+    search_dirs: list[str],
+) -> bool:
+    """Collect metrics and save to output file."""
+    timestamp = datetime.now(timezone.utc).isoformat()
+
+    # Find all result files
+    result_files = find_result_files(search_dirs)
+    print(f"Found {len(result_files)} result file(s)")
+
+    grouped = []
+    if not result_files:
+        print("No benchmark result files found")
+    else:
+        # Parse all result files
+        all_results = []
+        for filepath in sorted(result_files):
+            print(f"  Reading: {filepath}")
+            results = parse_result_file(filepath)
+            all_results.extend(results)
+        print(f"Total benchmark results: {len(all_results)}")
+
+        # Group by model/variant
+        grouped = group_results_by_model(all_results, gpu_config, partition)
+
+    # Create metrics structure
+    metrics = {
+        "run_id": run_id,
+        "timestamp": timestamp,
+        "gpu_config": gpu_config,
+        "partition": partition,
+        "results": grouped,
+    }
+
+    # Ensure output directory exists and write output
+    try:
+        os.makedirs(os.path.dirname(output_file) or ".", exist_ok=True)
+        with open(output_file, "w", encoding="utf-8") as f:
+            json.dump(metrics, f, indent=2)
+
+        if not result_files:
+            print(f"Created empty metrics file: {output_file}")
+        else:
+            print(f"Saved metrics to: {output_file}")
+        return True
+    except OSError as e:
+        print(f"Error writing metrics file: {e}")
+        return False
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Collect performance metrics from benchmark results"
+    )
+    parser.add_argument(
+        "--gpu-config",
+        required=True,
+        help="GPU configuration (e.g., 8-gpu-h200, 8-gpu-b200)",
+    )
+    parser.add_argument(
+        "--partition",
+        type=int,
+        required=True,
+        help="Partition number (0, 1, 2, etc.)",
+    )
+    parser.add_argument(
+        "--run-id",
+        required=True,
+        help="GitHub Actions run ID",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output file path for metrics JSON",
+    )
+    parser.add_argument(
+        "--search-dir",
+        action="append",
+        default=[],
+        dest="search_dirs",
+        help="Directory to search for result files (can be specified multiple times)",
+    )
+
+    args = parser.parse_args()
+
+    # Default search directories if none specified
+    search_dirs = args.search_dirs or [
+        "test/performance_profiles_8_gpu",
+        "test/performance_profiles_text_models",
+        "test/performance_profiles_vlms",
+        "test",
+        ".",
+    ]
+
+    success = save_metrics(
+        gpu_config=args.gpu_config,
+        partition=args.partition,
+        run_id=args.run_id,
+        output_file=args.output,
+        search_dirs=search_dirs,
+    )
+
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()