Add a performance dashboard server and frontend for nightly CUDA tests (#17725)

docs/performance_dashboard/README.md

@@ -0,0 +1,147 @@
+# SGLang Performance Dashboard
+
+A web-based dashboard for visualizing SGLang nightly test performance metrics.
+
+## Features
+
+- **Performance Trends**: View throughput, latency, and TTFT trends over time
+- **Model Comparison**: Compare performance across different models and configurations
+- **Filtering**: Filter by GPU configuration, model, variant, and batch size
+- **Interactive Charts**: Zoom, pan, and hover for detailed metrics
+- **Run History**: View recent benchmark runs with links to GitHub Actions
+
+## Quick Start
+
+### Option 1: Run with Local Server (Recommended)
+
+For live data from GitHub Actions artifacts:
+
+```bash
+# Install requirements
+pip install requests
+
+# Run the server
+python server.py --fetch-on-start
+
+# Visit http://localhost:8000
+```
+
+The server provides:
+- Automatic fetching of metrics from GitHub
+- Caching to reduce API calls
+- `/api/metrics` endpoint for the frontend
+
+### Option 2: Fetch Data Manually
+
+Use the fetch script to download metrics data:
+
+```bash
+# Fetch last 30 days of metrics
+python fetch_metrics.py --output metrics_data.json
+
+# Fetch a specific run
+python fetch_metrics.py --run-id 21338741812 --output single_run.json
+
+# Fetch only scheduled (nightly) runs
+python fetch_metrics.py --scheduled-only --days 7
+```
+
+## GitHub Token
+
+To download artifacts from GitHub, you need authentication:
+
+1. **Using `gh` CLI** (recommended):
+   ```bash
+   gh auth login
+   ```
+
+2. **Using environment variable**:
+   ```bash
+   export GITHUB_TOKEN=your_token_here
+   ```
+
+Without a token, the dashboard will show run metadata but not detailed benchmark results.
+
+## Data Structure
+
+The metrics JSON has this structure:
+
+```json
+{
+  "run_id": "21338741812",
+  "run_date": "2026-01-25T22:24:02.090218+00:00",
+  "commit_sha": "5cdb391...",
+  "branch": "main",
+  "results": [
+    {
+      "gpu_config": "8-gpu-h200",
+      "partition": 0,
+      "model": "deepseek-ai/DeepSeek-V3.1",
+      "variant": "TP8+MTP",
+      "benchmarks": [
+        {
+          "batch_size": 1,
+          "input_len": 4096,
+          "output_len": 512,
+          "latency_ms": 2400.72,
+          "input_throughput": 21408.64,
+          "output_throughput": 231.74,
+          "overall_throughput": 1919.43,
+          "ttft_ms": 191.32,
+          "acc_length": 3.19
+        }
+      ]
+    }
+  ]
+}
+```
+
+## Deployment
+
+### GitHub Pages
+
+The dashboard can be deployed to GitHub Pages for public access:
+
+1. Copy the dashboard files to `docs/performance_dashboard/`
+2. Enable GitHub Pages in repository settings
+3. Set up a GitHub Action to periodically update metrics data
+
+### Self-Hosted
+
+For a self-hosted deployment with live data:
+
+1. Set up a server running `server.py`
+2. Configure a cron job or systemd timer to refresh data
+3. Optionally put behind nginx/caddy for SSL
+
+## Metrics Explained
+
+- **Overall Throughput**: Total tokens (input + output) processed per second
+- **Input Throughput**: Input tokens processed per second (prefill speed)
+- **Output Throughput**: Output tokens generated per second (decode speed)
+- **Latency**: End-to-end time to complete the request
+- **TTFT**: Time to First Token - time until the first output token
+- **Acc Length**: Acceptance length for speculative decoding (MTP variants)
+
+## Contributing
+
+To add support for new metrics or visualizations:
+
+1. Update `fetch_metrics.py` if data collection needs changes
+2. Modify `app.js` to add new chart types or filters
+3. Update `index.html` for UI changes
+
+## Troubleshooting
+
+**No data displayed**
+- Check browser console for errors
+- Verify GitHub API is accessible
+- Try running with `server.py --fetch-on-start`
+
+**API rate limits**
+- Use a GitHub token for higher limits
+- The server caches data for 5 minutes
+
+**Charts not rendering**
+- Ensure Chart.js is loading from CDN
+- Check for JavaScript errors in console

docs/performance_dashboard/app.js

@@ -0,0 +1,836 @@
+// SGLang Performance Dashboard Application
+
+const GITHUB_REPO = 'sgl-project/sglang';
+const WORKFLOW_NAME = 'nightly-test-nvidia.yml';
+const ARTIFACT_PREFIX = 'consolidated-metrics-';
+
+// Chart instances (array for batch-separated charts)
+let activeCharts = [];
+
+// Data storage
+let allMetricsData = [];
+let currentModel = null;
+let currentMetricType = 'throughput'; // throughput, latency, ttft, inputThroughput
+
+// Metric type definitions
+const metricTypes = {
+    throughput: { label: 'Overall Throughput', unit: 'tokens/sec', field: 'throughput' },
+    outputThroughput: { label: 'Output Throughput', unit: 'tokens/sec', field: 'outputThroughput' },
+    inputThroughput: { label: 'Input Throughput', unit: 'tokens/sec', field: 'inputThroughput' },
+    latency: { label: 'Latency', unit: 'ms', field: 'latency' },
+    ttft: { label: 'Time to First Token', unit: 'ms', field: 'ttft' },
+    accLength: { label: 'Accept Length', unit: 'tokens', field: 'accLength', filterInvalid: true }
+};
+
+// Chart.js default configuration for dark theme
+Chart.defaults.color = '#8b949e';
+Chart.defaults.borderColor = '#30363d';
+
+const chartColors = [
+    '#58a6ff', '#3fb950', '#d29922', '#f85149', '#a371f7',
+    '#79c0ff', '#56d364', '#e3b341', '#ff7b72', '#bc8cff'
+];
+
+// Initialize the dashboard
+async function init() {
+    try {
+        await loadData();
+        document.getElementById('loading').style.display = 'none';
+        document.getElementById('content').style.display = 'block';
+        populateFilters();
+        updateStats();
+        updateCharts();
+        updateRunsTable();
+    } catch (error) {
+        console.error('Failed to initialize dashboard:', error);
+        document.getElementById('loading').style.display = 'none';
+        document.getElementById('error').style.display = 'block';
+        document.getElementById('error-message').textContent = error.message;
+    }
+}
+
+// Load data from local server API or GitHub
+async function loadData() {
+    // Try local server API first (if running server.py)
+    try {
+        const response = await fetch('/api/metrics');
+        if (response.ok) {
+            const data = await response.json();
+            if (data.length > 0 && data[0].results && data[0].results.length > 0) {
+                allMetricsData = data;
+                console.log(`Loaded ${data.length} records from local API`);
+                allMetricsData.sort((a, b) => new Date(b.run_date) - new Date(a.run_date));
+                return;
+            }
+        }
+    } catch (error) {
+        console.log('Local API not available, trying GitHub API');
+    }
+
+    // Try to load from GitHub API
+    const runs = await fetchWorkflowRuns();
+    const metricsPromises = runs.map(run => fetchMetricsForRun(run));
+    const results = await Promise.allSettled(metricsPromises);
+
+    allMetricsData = results
+        .filter(r => r.status === 'fulfilled' && r.value !== null)
+        .map(r => r.value);
+
+    if (allMetricsData.length === 0) {
+        throw new Error('No metrics data available. Please run the server.py with --fetch-on-start to fetch data from GitHub.');
+    }
+
+    // Sort by date descending
+    allMetricsData.sort((a, b) => new Date(b.run_date) - new Date(a.run_date));
+}
+
+// Fetch workflow runs from GitHub API
+async function fetchWorkflowRuns() {
+    const response = await fetch(
+        `https://api.github.com/repos/${GITHUB_REPO}/actions/workflows/${WORKFLOW_NAME}/runs?status=completed&per_page=30`,
+        {
+            headers: {
+                'Accept': 'application/vnd.github.v3+json'
+            }
+        }
+    );
+
+    if (!response.ok) {
+        throw new Error(`GitHub API error: ${response.status}`);
+    }
+
+    const data = await response.json();
+    return data.workflow_runs || [];
+}
+
+// Fetch metrics artifact for a specific run
+async function fetchMetricsForRun(run) {
+    try {
+        // Get artifacts for this run
+        const artifactsResponse = await fetch(
+            `https://api.github.com/repos/${GITHUB_REPO}/actions/runs/${run.id}/artifacts`,
+            {
+                headers: {
+                    'Accept': 'application/vnd.github.v3+json'
+                }
+            }
+        );
+
+        if (!artifactsResponse.ok) return null;
+
+        const artifactsData = await artifactsResponse.json();
+        const metricsArtifact = artifactsData.artifacts.find(
+            a => a.name.startsWith(ARTIFACT_PREFIX)
+        );
+
+        if (!metricsArtifact) return null;
+
+        // Note: GitHub API doesn't allow direct artifact download without authentication
+        // For public access, we would need to use a proxy or pre-process the data
+        // For now, return run metadata - in production, use a backend to fetch artifacts
+        return {
+            run_id: run.id.toString(),
+            run_date: run.created_at,
+            commit_sha: run.head_sha,
+            branch: run.head_branch,
+            artifact_id: metricsArtifact.id,
+            results: [] // Would be populated from artifact content
+        };
+    } catch (error) {
+        console.warn(`Failed to fetch metrics for run ${run.id}:`, error);
+        return null;
+    }
+}
+
+// Populate filter dropdowns
+function populateFilters() {
+    const gpuConfigs = new Set();
+    const models = new Set();
+    const batchSizes = new Set();
+    const ioLengths = new Set();
+
+    allMetricsData.forEach(run => {
+        run.results.forEach(result => {
+            gpuConfigs.add(result.gpu_config);
+            models.add(result.model);
+            // Try new structure first (benchmarks_by_io_len), fall back to flat benchmarks
+            if (result.benchmarks_by_io_len) {
+                Object.entries(result.benchmarks_by_io_len).forEach(([ioKey, ioData]) => {
+                    ioLengths.add(ioKey);
+                    ioData.benchmarks.forEach(bench => {
+                        batchSizes.add(bench.batch_size);
+                    });
+                });
+            } else if (result.benchmarks) {
+                result.benchmarks.forEach(bench => {
+                    batchSizes.add(bench.batch_size);
+                    if (bench.input_len && bench.output_len) {
+                        ioLengths.add(`${bench.input_len}_${bench.output_len}`);
+                    }
+                });
+            }
+        });
+    });
+
+    // No "all" option for GPU and Model - populate with first value selected
+    const gpuArray = Array.from(gpuConfigs).sort();
+    const modelArray = Array.from(models).sort();
+
+    populateSelectNoAll('gpu-filter', gpuArray);
+    populateSelectNoAll('model-filter', modelArray);
+    populateSelect('batch-filter', Array.from(batchSizes).sort((a, b) => a - b));
+    populateSelectWithLabels('io-len-filter', sortIoLengths(Array.from(ioLengths)), formatIoLenLabel);
+
+    // Set initial values (first option)
+    if (gpuArray.length > 0) {
+        document.getElementById('gpu-filter').value = gpuArray[0];
+    }
+    if (modelArray.length > 0) {
+        document.getElementById('model-filter').value = modelArray[0];
+        currentModel = modelArray[0];
+    }
+
+    // Update variants based on selected model
+    updateVariantFilter();
+    // Update IO length filter based on selected GPU/model
+    updateIoLenFilter();
+
+    // Create metric type tabs
+    createMetricTabs();
+}
+
+// Format input/output length key for display
+function formatIoLenLabel(ioKey) {
+    if (!ioKey) return 'Unknown';
+    const parts = ioKey.split('_');
+    if (parts.length === 2) {
+        return `In: ${parts[0]}, Out: ${parts[1]}`;
+    }
+    return ioKey;
+}
+
+// Sort IO length keys numerically (by input length, then output length)
+function sortIoLengths(ioLengths) {
+    return ioLengths.filter(key => key && key.includes('_')).sort((a, b) => {
+        const [aIn, aOut] = a.split('_').map(Number);
+        const [bIn, bOut] = b.split('_').map(Number);
+        if (isNaN(aIn) || isNaN(bIn)) return 0;
+        return (aIn - bIn) || (aOut - bOut);
+    });
+}
+
+// Populate select with custom label formatting
+function populateSelectWithLabels(selectId, options, labelFormatter) {
+    const select = document.getElementById(selectId);
+    options.forEach(option => {
+        const opt = document.createElement('option');
+        opt.value = option;
+        opt.textContent = labelFormatter ? labelFormatter(option) : option;
+        select.appendChild(opt);
+    });
+}
+
+// Update IO length filter based on selected GPU and model
+function updateIoLenFilter() {
+    const gpuFilterEl = document.getElementById('gpu-filter');
+    const modelFilterEl = document.getElementById('model-filter');
+    const ioLenSelect = document.getElementById('io-len-filter');
+    if (!gpuFilterEl || !modelFilterEl || !ioLenSelect) return;
+
+    const gpuFilter = gpuFilterEl.value;
+    const modelFilter = modelFilterEl.value;
+
+    const ioLengths = new Set();
+
+    allMetricsData.forEach(run => {
+        run.results.forEach(result => {
+            if (result.gpu_config === gpuFilter && result.model === modelFilter) {
+                if (result.benchmarks_by_io_len) {
+                    Object.keys(result.benchmarks_by_io_len).forEach(ioKey => {
+                        ioLengths.add(ioKey);
+                    });
+                } else if (result.benchmarks) {
+                    result.benchmarks.forEach(bench => {
+                        if (bench.input_len && bench.output_len) {
+                            ioLengths.add(`${bench.input_len}_${bench.output_len}`);
+                        }
+                    });
+                }
+            }
+        });
+    });
+
+    const ioLenArray = sortIoLengths(Array.from(ioLengths));
+    const currentIoLen = ioLenSelect.value;
+
+    // Clear and repopulate
+    ioLenSelect.innerHTML = '<option value="all">All Lengths</option>';
+    ioLenArray.forEach(ioLen => {
+        const opt = document.createElement('option');
+        opt.value = ioLen;
+        opt.textContent = formatIoLenLabel(ioLen);
+        ioLenSelect.appendChild(opt);
+    });
+
+    // Try to restore previous selection if still valid
+    if (ioLenArray.includes(currentIoLen)) {
+        ioLenSelect.value = currentIoLen;
+    } else {
+        ioLenSelect.value = 'all';
+    }
+}
+
+// Update variant filter based on selected GPU and model
+function updateVariantFilter() {
+    const gpuFilter = document.getElementById('gpu-filter').value;
+    const modelFilter = document.getElementById('model-filter').value;
+
+    const variants = new Set();
+
+    allMetricsData.forEach(run => {
+        run.results.forEach(result => {
+            if (result.gpu_config === gpuFilter && result.model === modelFilter) {
+                // Use 'default' for null/undefined variants
+                variants.add(result.variant || 'default');
+            }
+        });
+    });
+
+    const variantArray = Array.from(variants).sort();
+    const variantSelect = document.getElementById('variant-filter');
+    const currentVariant = variantSelect.value;
+
+    // Clear and repopulate
+    variantSelect.innerHTML = '<option value="all">All Variants</option>';
+    variantArray.forEach(variant => {
+        const opt = document.createElement('option');
+        opt.value = variant;
+        opt.textContent = variant;
+        variantSelect.appendChild(opt);
+    });
+
+    // Try to restore previous selection if still valid
+    if (variantArray.includes(currentVariant)) {
+        variantSelect.value = currentVariant;
+    } else {
+        variantSelect.value = 'all';
+    }
+}
+
+function populateSelect(selectId, options) {
+    const select = document.getElementById(selectId);
+    options.forEach(option => {
+        const opt = document.createElement('option');
+        opt.value = option;
+        opt.textContent = option;
+        select.appendChild(opt);
+    });
+}
+
+function populateSelectNoAll(selectId, options) {
+    const select = document.getElementById(selectId);
+    // Remove the "all" option if present
+    while (select.options.length > 0) {
+        select.remove(0);
+    }
+    options.forEach(option => {
+        const opt = document.createElement('option');
+        opt.value = option;
+        opt.textContent = option;
+        select.appendChild(opt);
+    });
+}
+
+function createMetricTabs() {
+    const tabsContainer = document.getElementById('metric-tabs');
+    tabsContainer.innerHTML = '';
+
+    Object.entries(metricTypes).forEach(([key, metric], index) => {
+        const tab = document.createElement('div');
+        tab.className = index === 0 ? 'tab active' : 'tab';
+        tab.textContent = metric.label;
+        tab.dataset.metric = key;
+        tab.onclick = () => selectMetricTab(key, tab);
+        tabsContainer.appendChild(tab);
+    });
+}
+
+function selectMetricTab(metricKey, tabElement) {
+    document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
+    tabElement.classList.add('active');
+    currentMetricType = metricKey;
+
+    // Update chart title
+    const metric = metricTypes[metricKey];
+    document.getElementById('metric-title').textContent = `${metric.label} (${metric.unit})`;
+
+    updateCharts();
+}
+
+// Handle model filter dropdown change
+function handleModelFilterChange(model) {
+    currentModel = model;
+    // Update variant filter based on new model selection
+    updateVariantFilter();
+    // Update IO length filter based on new model selection
+    updateIoLenFilter();
+    updateCharts();
+}
+
+// Handle GPU filter change
+function handleGpuFilterChange() {
+    // Update variant filter based on new GPU selection
+    updateVariantFilter();
+    // Update IO length filter based on new GPU selection
+    updateIoLenFilter();
+    updateCharts();
+}
+
+// Update summary stats
+function updateStats() {
+    const statsRow = document.getElementById('stats-row');
+    const latestRun = allMetricsData[0];
+
+    if (!latestRun) {
+        statsRow.innerHTML = '';
+        const noDataDiv = document.createElement('div');
+        noDataDiv.className = 'no-data';
+        noDataDiv.textContent = 'No data available';
+        statsRow.appendChild(noDataDiv);
+        return;
+    }
+
+    const totalModels = new Set(latestRun.results.map(r => r.model)).size;
+    const totalBenchmarks = latestRun.results.reduce((sum, r) => {
+        // Count benchmarks from either structure
+        if (r.benchmarks_by_io_len) {
+            return sum + Object.values(r.benchmarks_by_io_len).reduce(
+                (ioSum, ioData) => ioSum + ioData.benchmarks.length, 0
+            );
+        }
+        return sum + (r.benchmarks ? r.benchmarks.length : 0);
+    }, 0);
+
+    statsRow.innerHTML = ''; // Clear previous stats
+
+    const addStat = (label, value) => {
+        const card = document.createElement('div');
+        card.className = 'stat-card';
+        const labelEl = document.createElement('div');
+        labelEl.className = 'label';
+        labelEl.textContent = label;
+        const valueEl = document.createElement('div');
+        valueEl.className = 'value';
+        valueEl.textContent = value;
+        card.appendChild(labelEl);
+        card.appendChild(valueEl);
+        statsRow.appendChild(card);
+    };
+
+    addStat('Total Runs', allMetricsData.length);
+    addStat('Models Tested', totalModels);
+    addStat('Benchmarks', totalBenchmarks);
+}
+
+// Update charts based on current filters and selected metric type
+function updateCharts() {
+    const gpuFilter = document.getElementById('gpu-filter').value;
+    const modelFilter = currentModel;
+    const variantFilter = document.getElementById('variant-filter').value;
+    const ioLenFilter = document.getElementById('io-len-filter').value;
+    const batchFilter = document.getElementById('batch-filter').value;
+
+    // Prepare data for charts - grouped by batch size
+    const chartDataByBatch = prepareChartDataByBatch(gpuFilter, modelFilter, variantFilter, ioLenFilter, batchFilter);
+
+    // Update chart for the selected metric type
+    updateMetricChart(chartDataByBatch, currentMetricType);
+}
+
+function prepareChartData(gpuFilter, modelFilter, variantFilter, ioLenFilter, batchFilter) {
+    const seriesMap = new Map();
+
+    allMetricsData.forEach(run => {
+        const runDate = new Date(run.run_date);
+
+        run.results.forEach(result => {
+            // Apply filters
+            if (result.gpu_config !== gpuFilter) return;
+            if (result.model !== modelFilter) return;
+            if (variantFilter !== 'all' && result.variant !== variantFilter) return;
+
+            // Helper function to process a benchmark entry
+            const processBenchmark = (bench, ioKey) => {
+                if (batchFilter !== 'all' && bench.batch_size !== parseInt(batchFilter)) return;
+
+                const ioLabel = ioKey ? `, ${formatIoLenLabel(ioKey)}` : '';
+                const seriesKey = `${result.model.split('/').pop()} (${result.variant}, BS=${bench.batch_size}${ioLabel})`;
+
+                if (!seriesMap.has(seriesKey)) {
+                    seriesMap.set(seriesKey, {
+                        label: seriesKey,
+                        data: [],
+                        model: result.model,
+                        variant: result.variant,
+                        batchSize: bench.batch_size,
+                        ioKey: ioKey
+                    });
+                }
+
+                seriesMap.get(seriesKey).data.push({
+                    x: runDate,
+                    throughput: bench.overall_throughput,
+                    outputThroughput: bench.output_throughput,
+                    latency: bench.latency_ms,
+                    ttft: bench.ttft_ms,
+                    inputThroughput: bench.input_throughput,
+                    accLength: bench.acc_length,
+                    runId: run.run_id
+                });
+            };
+
+            // Use benchmarks_by_io_len if available
+            if (result.benchmarks_by_io_len) {
+                Object.entries(result.benchmarks_by_io_len).forEach(([ioKey, ioData]) => {
+                    if (ioLenFilter !== 'all' && ioKey !== ioLenFilter) return;
+                    ioData.benchmarks.forEach(bench => processBenchmark(bench, ioKey));
+                });
+            } else if (result.benchmarks) {
+                result.benchmarks.forEach(bench => {
+                    const benchIoKey = bench.input_len && bench.output_len
+                        ? `${bench.input_len}_${bench.output_len}`
+                        : null;
+                    if (ioLenFilter !== 'all' && benchIoKey !== ioLenFilter) return;
+                    processBenchmark(bench, benchIoKey);
+                });
+            }
+        });
+    });
+
+    // Sort data points by date
+    seriesMap.forEach(series => {
+        series.data.sort((a, b) => a.x - b.x);
+    });
+
+    return Array.from(seriesMap.values());
+}
+
+// Prepare chart data grouped by batch size - each batch size is a separate series
+function prepareChartDataByBatch(gpuFilter, modelFilter, variantFilter, ioLenFilter, batchFilter) {
+    const batchDataMap = new Map(); // batch_size -> Map of variant -> data
+
+    allMetricsData.forEach(run => {
+        const runDate = new Date(run.run_date);
+
+        run.results.forEach(result => {
+            // Apply filters - GPU and Model are required (no "all" option)
+            if (result.gpu_config !== gpuFilter) return;
+            if (result.model !== modelFilter) return;
+            if (variantFilter !== 'all' && result.variant !== variantFilter) return;
+
+            // Use benchmarks_by_io_len if available, otherwise fall back to flat benchmarks
+            if (result.benchmarks_by_io_len) {
+                Object.entries(result.benchmarks_by_io_len).forEach(([ioKey, ioData]) => {
+                    // Apply IO length filter
+                    if (ioLenFilter !== 'all' && ioKey !== ioLenFilter) return;
+
+                    ioData.benchmarks.forEach(bench => {
+                        if (batchFilter !== 'all' && bench.batch_size !== parseInt(batchFilter)) return;
+
+                        const batchSize = bench.batch_size;
+                        const variantLabel = result.variant || 'default';
+                        // Include IO length in series key when showing all lengths
+                        const seriesKey = ioLenFilter === 'all'
+                            ? `${variantLabel} (${formatIoLenLabel(ioKey)})`
+                            : variantLabel;
+
+                        if (!batchDataMap.has(batchSize)) {
+                            batchDataMap.set(batchSize, new Map());
+                        }
+
+                        const variantMap = batchDataMap.get(batchSize);
+                        if (!variantMap.has(seriesKey)) {
+                            variantMap.set(seriesKey, {
+                                label: seriesKey,
+                                data: [],
+                                model: result.model,
+                                variant: result.variant,
+                                batchSize: batchSize,
+                                ioKey: ioKey
+                            });
+                        }
+
+                        variantMap.get(seriesKey).data.push({
+                            x: runDate,
+                            throughput: bench.overall_throughput,
+                            outputThroughput: bench.output_throughput,
+                            latency: bench.latency_ms,
+                            ttft: bench.ttft_ms,
+                            inputThroughput: bench.input_throughput,
+                            accLength: bench.acc_length,
+                            runId: run.run_id
+                        });
+                    });
+                });
+            } else if (result.benchmarks) {
+                // Fall back to flat benchmarks for backward compatibility
+                result.benchmarks.forEach(bench => {
+                    // Apply IO length filter using flat structure
+                    const benchIoKey = bench.input_len && bench.output_len
+                        ? `${bench.input_len}_${bench.output_len}`
+                        : null;
+                    if (ioLenFilter !== 'all' && benchIoKey !== ioLenFilter) return;
+                    if (batchFilter !== 'all' && bench.batch_size !== parseInt(batchFilter)) return;
+
+                    const batchSize = bench.batch_size;
+                    const variantLabel = result.variant || 'default';
+                    // Include IO length in series key when showing all lengths
+                    const seriesKey = ioLenFilter === 'all' && benchIoKey
+                        ? `${variantLabel} (${formatIoLenLabel(benchIoKey)})`
+                        : variantLabel;
+
+                    if (!batchDataMap.has(batchSize)) {
+                        batchDataMap.set(batchSize, new Map());
+                    }
+
+                    const variantMap = batchDataMap.get(batchSize);
+                    if (!variantMap.has(seriesKey)) {
+                        variantMap.set(seriesKey, {
+                            label: seriesKey,
+                            data: [],
+                            model: result.model,
+                            variant: result.variant,
+                            batchSize: batchSize,
+                            ioKey: benchIoKey
+                        });
+                    }
+
+                    variantMap.get(seriesKey).data.push({
+                        x: runDate,
+                        throughput: bench.overall_throughput,
+                        outputThroughput: bench.output_throughput,
+                        latency: bench.latency_ms,
+                        ttft: bench.ttft_ms,
+                        inputThroughput: bench.input_throughput,
+                        accLength: bench.acc_length,
+                        runId: run.run_id
+                    });
+                });
+            }
+        });
+    });
+
+    // Sort data points by date and convert to array format
+    const result = {};
+    batchDataMap.forEach((variantMap, batchSize) => {
+        variantMap.forEach(series => {
+            series.data.sort((a, b) => a.x - b.x);
+        });
+        result[batchSize] = Array.from(variantMap.values());
+    });
+
+    return result;
+}
+
+// Unified chart update function for any metric type
+function updateMetricChart(chartDataByBatch, metricType) {
+    const container = document.getElementById('charts-container');
+    container.innerHTML = '';
+
+    // Destroy existing charts
+    activeCharts.forEach(chart => chart.destroy());
+    activeCharts = [];
+
+    const metric = metricTypes[metricType];
+    const batchSizes = Object.keys(chartDataByBatch).sort((a, b) => parseInt(a) - parseInt(b));
+
+    if (batchSizes.length === 0) {
+        container.innerHTML = '<div class="no-data">No data available for the selected filters</div>';
+        return;
+    }
+
+    let hasAnyData = false;
+
+    batchSizes.forEach(batchSize => {
+        const chartData = chartDataByBatch[batchSize];
+
+        const ctx_datasets = chartData.map((series, index) => {
+            // Filter data points - for metrics like accLength, exclude invalid values (-1 or null)
+            let dataPoints = series.data.map(d => ({ x: d.x, y: d[metric.field] }));
+            if (metric.filterInvalid) {
+                dataPoints = dataPoints.filter(d => d.y != null && d.y !== -1 && d.y > 0);
+            }
+            return {
+                label: series.label,
+                data: dataPoints,
+                borderColor: chartColors[index % chartColors.length],
+                backgroundColor: chartColors[index % chartColors.length] + '20',
+                tension: 0.1,
+                fill: false
+            };
+        }).filter(dataset => dataset.data.length > 0); // Remove empty datasets
+
+        // Skip this batch size if no valid data
+        if (ctx_datasets.length === 0) {
+            return;
+        }
+
+        hasAnyData = true;
+
+        const chartWrapper = document.createElement('div');
+        chartWrapper.className = 'batch-chart-wrapper';
+
+        const title = document.createElement('div');
+        title.className = 'batch-chart-title';
+        title.textContent = `Batch Size: ${batchSize}`;
+        chartWrapper.appendChild(title);
+
+        const chartContainer = document.createElement('div');
+        chartContainer.className = 'chart-container';
+        const canvas = document.createElement('canvas');
+        chartContainer.appendChild(canvas);
+        chartWrapper.appendChild(chartContainer);
+        container.appendChild(chartWrapper);
+
+        const ctx = canvas.getContext('2d');
+
+        const chart = new Chart(ctx, {
+            type: 'line',
+            data: { datasets: ctx_datasets },
+            options: getChartOptions(metric.unit)
+        });
+        activeCharts.push(chart);
+    });
+
+    // Show message if no valid data for this metric
+    if (!hasAnyData) {
+        container.innerHTML = `<div class="no-data">No valid ${metric.label.toLowerCase()} data available for the selected filters</div>`;
+    }
+}
+
+function getChartOptions(yAxisLabel) {
+    return {
+        responsive: true,
+        maintainAspectRatio: false,
+        interaction: {
+            mode: 'index',
+            intersect: false
+        },
+        plugins: {
+            legend: {
+                position: 'bottom',
+                labels: {
+                    boxWidth: 12,
+                    padding: 10,
+                    font: { size: 11 }
+                }
+            },
+            tooltip: {
+                backgroundColor: '#21262d',
+                borderColor: '#30363d',
+                borderWidth: 1,
+                titleFont: { size: 13 },
+                bodyFont: { size: 12 },
+                padding: 12
+            }
+        },
+        scales: {
+            x: {
+                type: 'time',
+                time: {
+                    unit: 'day',
+                    displayFormats: {
+                        day: 'MMM d'
+                    }
+                },
+                grid: {
+                    color: '#21262d'
+                }
+            },
+            y: {
+                title: {
+                    display: true,
+                    text: yAxisLabel
+                },
+                grid: {
+                    color: '#21262d'
+                }
+            }
+        }
+    };
+}
+
+// Escape HTML to prevent XSS
+function escapeHtml(text) {
+    const div = document.createElement('div');
+    div.textContent = text;
+    return div.innerHTML;
+}
+
+// Update runs table
+function updateRunsTable() {
+    const tbody = document.getElementById('runs-table-body');
+    tbody.innerHTML = '';
+
+    allMetricsData.slice(0, 10).forEach(run => {
+        const models = new Set(run.results.map(r => r.model.split('/').pop()));
+        const date = new Date(run.run_date);
+
+        const row = document.createElement('tr');
+
+        // Create cells safely to prevent XSS
+        const dateCell = document.createElement('td');
+        dateCell.textContent = `${date.toLocaleDateString()} ${date.toLocaleTimeString()}`;
+
+        const runIdCell = document.createElement('td');
+        const runLink = document.createElement('a');
+        runLink.href = `https://github.com/${GITHUB_REPO}/actions/runs/${encodeURIComponent(run.run_id)}`;
+        runLink.target = '_blank';
+        runLink.className = 'run-link';
+        runLink.textContent = run.run_id;
+        runIdCell.appendChild(runLink);
+
+        const commitCell = document.createElement('td');
+        const commitCode = document.createElement('code');
+        commitCode.textContent = run.commit_sha.substring(0, 7);
+        commitCell.appendChild(commitCode);
+
+        const branchCell = document.createElement('td');
+        branchCell.textContent = run.branch;
+
+        const modelsCell = document.createElement('td');
+        Array.from(models).forEach((model, index) => {
+            if (index > 0) modelsCell.appendChild(document.createTextNode(' '));
+            const badge = document.createElement('span');
+            badge.className = 'model-badge';
+            badge.textContent = model;
+            modelsCell.appendChild(badge);
+        });
+
+        row.appendChild(dateCell);
+        row.appendChild(runIdCell);
+        row.appendChild(commitCell);
+        row.appendChild(branchCell);
+        row.appendChild(modelsCell);
+
+        tbody.appendChild(row);
+    });
+}
+
+// Refresh data
+async function refreshData() {
+    document.getElementById('content').style.display = 'none';
+    document.getElementById('loading').style.display = 'flex';
+    await init();
+}
+
+// Format numbers for display
+function formatNumber(num) {
+    if (num >= 1000) {
+        return (num / 1000).toFixed(1) + 'k';
+    }
+    return num.toFixed(1);
+}
+
+// Initialize on page load
+document.addEventListener('DOMContentLoaded', init);

docs/performance_dashboard/fetch_metrics.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+"""
+Fetch and process SGLang nightly test metrics from GitHub Actions artifacts.
+
+This script fetches consolidated metrics from GitHub Actions workflow runs
+and outputs them as JSON for the performance dashboard.
+
+Usage:
+    python fetch_metrics.py --output metrics_data.json
+    python fetch_metrics.py --output metrics_data.json --days 30
+    python fetch_metrics.py --output metrics_data.json --run-id 21338741812
+"""
+
+import argparse
+import io
+import json
+import os
+import sys
+import zipfile
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Optional
+
+import requests
+
+GITHUB_REPO = "sgl-project/sglang"
+WORKFLOW_NAME = "nightly-test-nvidia.yml"
+ARTIFACT_PREFIX = "consolidated-metrics-"
+
+
+def get_github_token() -> Optional[str]:
+    """Get GitHub token from environment or gh CLI."""
+    # Check environment variable first
+    token = os.environ.get("GITHUB_TOKEN")
+    if token:
+        return token
+
+    # Try gh CLI
+    try:
+        import subprocess
+
+        result = subprocess.run(
+            ["gh", "auth", "token"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        return result.stdout.strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        pass
+
+    return None
+
+
+def get_headers(token: Optional[str]) -> dict:
+    """Get request headers with optional authentication."""
+    headers = {
+        "Accept": "application/vnd.github.v3+json",
+    }
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    return headers
+
+
+def fetch_workflow_runs(
+    token: Optional[str],
+    days: int = 30,
+    event: Optional[str] = None,
+) -> list:
+    """Fetch completed workflow runs from GitHub Actions."""
+    url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/workflows/{WORKFLOW_NAME}/runs"
+
+    params = {
+        "status": "completed",
+        "per_page": 100,
+    }
+
+    if event:
+        params["event"] = event
+
+    response = requests.get(url, headers=get_headers(token), params=params, timeout=30)
+    response.raise_for_status()
+
+    runs = response.json().get("workflow_runs", [])
+
+    # Filter by date
+    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+    runs = [
+        run
+        for run in runs
+        if datetime.fromisoformat(run["created_at"].replace("Z", "+00:00")) > cutoff
+    ]
+
+    return runs
+
+
+def fetch_run_artifacts(token: Optional[str], run_id: int) -> list:
+    """Fetch artifacts for a specific workflow run."""
+    url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}/artifacts"
+
+    response = requests.get(url, headers=get_headers(token), timeout=30)
+    response.raise_for_status()
+
+    return response.json().get("artifacts", [])
+
+
+def download_artifact(token: Optional[str], artifact_id: int) -> Optional[bytes]:
+    """Download an artifact by ID."""
+    if not token:
+        print(f"Warning: GitHub token required to download artifacts", file=sys.stderr)
+        return None
+
+    url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/artifacts/{artifact_id}/zip"
+
+    headers = get_headers(token)
+    response = requests.get(url, headers=headers, allow_redirects=True, timeout=60)
+
+    if response.status_code == 200:
+        return response.content
+
+    print(
+        f"Failed to download artifact {artifact_id}: {response.status_code}",
+        file=sys.stderr,
+    )
+    return None
+
+
+def extract_metrics_from_zip(zip_content: bytes) -> Optional[dict]:
+    """Extract metrics JSON from a zip file."""
+    try:
+        with zipfile.ZipFile(io.BytesIO(zip_content)) as zf:
+            # Find the JSON file in the archive
+            json_files = [f for f in zf.namelist() if f.endswith(".json")]
+            if not json_files:
+                return None
+
+            with zf.open(json_files[0]) as f:
+                return json.load(f)
+    except (zipfile.BadZipFile, json.JSONDecodeError) as e:
+        print(f"Failed to extract metrics: {e}", file=sys.stderr)
+        return None
+
+
+def fetch_metrics_for_run(token: Optional[str], run: dict) -> Optional[dict]:
+    """Fetch metrics for a single workflow run."""
+    run_id = run["id"]
+    print(f"Fetching metrics for run {run_id}...", file=sys.stderr)
+
+    artifacts = fetch_run_artifacts(token, run_id)
+
+    # Find consolidated metrics artifact
+    metrics_artifact = None
+    for artifact in artifacts:
+        if artifact["name"].startswith(ARTIFACT_PREFIX):
+            metrics_artifact = artifact
+            break
+
+    if not metrics_artifact:
+        print(f"No consolidated metrics found for run {run_id}", file=sys.stderr)
+        return None
+
+    # Download and extract
+    zip_content = download_artifact(token, metrics_artifact["id"])
+    if not zip_content:
+        return None
+
+    metrics = extract_metrics_from_zip(zip_content)
+    if not metrics:
+        return None
+
+    # Ensure required fields are present
+    if "run_id" not in metrics:
+        metrics["run_id"] = str(run_id)
+    if "run_date" not in metrics:
+        metrics["run_date"] = run["created_at"]
+    if "commit_sha" not in metrics:
+        metrics["commit_sha"] = run["head_sha"]
+    if "branch" not in metrics:
+        metrics["branch"] = run["head_branch"]
+
+    return metrics
+
+
+def fetch_single_run(token: Optional[str], run_id: int) -> Optional[dict]:
+    """Fetch metrics for a single run by ID."""
+    url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}"
+
+    response = requests.get(url, headers=get_headers(token), timeout=30)
+    response.raise_for_status()
+
+    run = response.json()
+    return fetch_metrics_for_run(token, run)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Fetch SGLang nightly test metrics from GitHub Actions"
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=str,
+        default="metrics_data.json",
+        help="Output JSON file path",
+    )
+    parser.add_argument(
+        "--days",
+        type=int,
+        default=30,
+        help="Number of days to fetch (default: 30)",
+    )
+    parser.add_argument(
+        "--run-id",
+        type=int,
+        help="Fetch a specific run by ID",
+    )
+    parser.add_argument(
+        "--event",
+        type=str,
+        choices=["schedule", "workflow_dispatch", "push"],
+        help="Filter by trigger event type",
+    )
+    parser.add_argument(
+        "--scheduled-only",
+        action="store_true",
+        help="Only fetch scheduled (nightly) runs",
+    )
+
+    args = parser.parse_args()
+
+    token = get_github_token()
+    if not token:
+        print(
+            "Warning: No GitHub token found. Some features may be limited.",
+            file=sys.stderr,
+        )
+        print(
+            "Set GITHUB_TOKEN env var or login with 'gh auth login'",
+            file=sys.stderr,
+        )
+
+    all_metrics = []
+
+    if args.run_id:
+        # Fetch single run
+        metrics = fetch_single_run(token, args.run_id)
+        if metrics:
+            all_metrics.append(metrics)
+    else:
+        # Fetch multiple runs
+        event = "schedule" if args.scheduled_only else args.event
+        runs = fetch_workflow_runs(token, days=args.days, event=event)
+        print(f"Found {len(runs)} workflow runs", file=sys.stderr)
+
+        for run in runs:
+            metrics = fetch_metrics_for_run(token, run)
+            if metrics:
+                all_metrics.append(metrics)
+
+    # Sort by date descending
+    all_metrics.sort(key=lambda x: x.get("run_date", ""), reverse=True)
+
+    # Write output
+    output_path = Path(args.output)
+    with open(output_path, "w") as f:
+        json.dump(all_metrics, f, indent=2)
+
+    print(f"Wrote {len(all_metrics)} metrics records to {output_path}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

docs/performance_dashboard/index.html

@@ -0,0 +1,460 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>SGLang Performance Dashboard</title>
+    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns"></script>
+    <style>
+        :root {
+            --bg-primary: #0d1117;
+            --bg-secondary: #161b22;
+            --bg-tertiary: #21262d;
+            --text-primary: #c9d1d9;
+            --text-secondary: #8b949e;
+            --border-color: #30363d;
+            --accent-color: #58a6ff;
+            --accent-green: #3fb950;
+            --accent-orange: #d29922;
+            --accent-red: #f85149;
+        }
+
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Noto Sans', Helvetica, Arial, sans-serif;
+            background-color: var(--bg-primary);
+            color: var(--text-primary);
+            line-height: 1.5;
+        }
+
+        .container {
+            max-width: 1400px;
+            margin: 0 auto;
+            padding: 20px;
+        }
+
+        header {
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            padding: 20px 0;
+            border-bottom: 1px solid var(--border-color);
+            margin-bottom: 24px;
+        }
+
+        h1 {
+            font-size: 24px;
+            font-weight: 600;
+            display: flex;
+            align-items: center;
+            gap: 12px;
+        }
+
+        h1 svg {
+            width: 32px;
+            height: 32px;
+        }
+
+        .header-actions {
+            display: flex;
+            gap: 12px;
+            align-items: center;
+        }
+
+        .btn {
+            padding: 8px 16px;
+            border-radius: 6px;
+            border: 1px solid var(--border-color);
+            background: var(--bg-secondary);
+            color: var(--text-primary);
+            cursor: pointer;
+            font-size: 14px;
+            transition: all 0.2s;
+        }
+
+        .btn:hover {
+            background: var(--bg-tertiary);
+            border-color: var(--text-secondary);
+        }
+
+        .btn-primary {
+            background: var(--accent-color);
+            border-color: var(--accent-color);
+            color: white;
+        }
+
+        .btn-primary:hover {
+            background: #4a9eff;
+        }
+
+        .filters {
+            display: flex;
+            gap: 16px;
+            flex-wrap: wrap;
+            margin-bottom: 24px;
+            padding: 16px;
+            background: var(--bg-secondary);
+            border-radius: 8px;
+            border: 1px solid var(--border-color);
+        }
+
+        .filter-group {
+            display: flex;
+            flex-direction: column;
+            gap: 6px;
+        }
+
+        .filter-group label {
+            font-size: 12px;
+            color: var(--text-secondary);
+            font-weight: 500;
+            text-transform: uppercase;
+        }
+
+        select {
+            padding: 8px 12px;
+            border-radius: 6px;
+            border: 1px solid var(--border-color);
+            background: var(--bg-tertiary);
+            color: var(--text-primary);
+            font-size: 14px;
+            min-width: 180px;
+        }
+
+        select:focus {
+            outline: none;
+            border-color: var(--accent-color);
+        }
+
+        .tabs {
+            display: flex;
+            gap: 4px;
+            margin-bottom: 24px;
+            border-bottom: 1px solid var(--border-color);
+            padding-bottom: 0;
+        }
+
+        .tab {
+            padding: 12px 20px;
+            cursor: pointer;
+            border-radius: 6px 6px 0 0;
+            background: transparent;
+            color: var(--text-secondary);
+            border: 1px solid transparent;
+            border-bottom: none;
+            transition: all 0.2s;
+            font-weight: 500;
+        }
+
+        .tab:hover {
+            color: var(--text-primary);
+            background: var(--bg-secondary);
+        }
+
+        .tab.active {
+            background: var(--bg-secondary);
+            color: var(--text-primary);
+            border-color: var(--border-color);
+            border-bottom: 1px solid var(--bg-secondary);
+            margin-bottom: -1px;
+        }
+
+        .charts-grid {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(600px, 1fr));
+            gap: 24px;
+        }
+
+        .chart-card {
+            background: var(--bg-secondary);
+            border-radius: 8px;
+            border: 1px solid var(--border-color);
+            padding: 20px;
+        }
+
+        .chart-card h3 {
+            font-size: 16px;
+            font-weight: 600;
+            margin-bottom: 16px;
+            color: var(--text-primary);
+        }
+
+        .chart-container {
+            position: relative;
+            height: 300px;
+        }
+
+        .loading {
+            display: flex;
+            justify-content: center;
+            align-items: center;
+            min-height: 400px;
+            color: var(--text-secondary);
+        }
+
+        .spinner {
+            width: 40px;
+            height: 40px;
+            border: 3px solid var(--border-color);
+            border-top-color: var(--accent-color);
+            border-radius: 50%;
+            animation: spin 1s linear infinite;
+        }
+
+        @keyframes spin {
+            to { transform: rotate(360deg); }
+        }
+
+        .error {
+            background: rgba(248, 81, 73, 0.1);
+            border: 1px solid var(--accent-red);
+            border-radius: 8px;
+            padding: 20px;
+            text-align: center;
+            color: var(--accent-red);
+        }
+
+        .stats-row {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+            gap: 16px;
+            margin-bottom: 24px;
+        }
+
+        .stat-card {
+            background: var(--bg-secondary);
+            border-radius: 8px;
+            border: 1px solid var(--border-color);
+            padding: 16px;
+        }
+
+        .stat-card .label {
+            font-size: 12px;
+            color: var(--text-secondary);
+            text-transform: uppercase;
+            margin-bottom: 4px;
+        }
+
+        .stat-card .value {
+            font-size: 24px;
+            font-weight: 600;
+            color: var(--text-primary);
+        }
+
+        .stat-card .change {
+            font-size: 12px;
+            margin-top: 4px;
+        }
+
+        .stat-card .change.positive {
+            color: var(--accent-green);
+        }
+
+        .stat-card .change.negative {
+            color: var(--accent-red);
+        }
+
+        .data-table {
+            width: 100%;
+            border-collapse: collapse;
+            margin-top: 24px;
+        }
+
+        .data-table th,
+        .data-table td {
+            padding: 12px;
+            text-align: left;
+            border-bottom: 1px solid var(--border-color);
+        }
+
+        .data-table th {
+            background: var(--bg-tertiary);
+            font-weight: 600;
+            color: var(--text-secondary);
+            font-size: 12px;
+            text-transform: uppercase;
+        }
+
+        .data-table tr:hover {
+            background: var(--bg-tertiary);
+        }
+
+        .run-link {
+            color: var(--accent-color);
+            text-decoration: none;
+        }
+
+        .run-link:hover {
+            text-decoration: underline;
+        }
+
+        .no-data {
+            text-align: center;
+            padding: 60px 20px;
+            color: var(--text-secondary);
+        }
+
+        .no-data h3 {
+            margin-bottom: 8px;
+        }
+
+        .model-badge {
+            display: inline-block;
+            padding: 2px 8px;
+            border-radius: 12px;
+            font-size: 12px;
+            background: var(--bg-tertiary);
+            border: 1px solid var(--border-color);
+        }
+
+        .metric-section {
+            margin-bottom: 24px;
+        }
+
+        .batch-charts-container {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
+            gap: 16px;
+        }
+
+        .batch-chart-wrapper {
+            background: var(--bg-tertiary);
+            border-radius: 6px;
+            padding: 12px;
+        }
+
+        .batch-chart-title {
+            font-size: 14px;
+            font-weight: 600;
+            color: var(--text-secondary);
+            margin-bottom: 8px;
+            text-align: center;
+        }
+
+        footer {
+            margin-top: 48px;
+            padding: 24px 0;
+            border-top: 1px solid var(--border-color);
+            text-align: center;
+            color: var(--text-secondary);
+            font-size: 14px;
+        }
+
+        footer a {
+            color: var(--accent-color);
+            text-decoration: none;
+        }
+
+        footer a:hover {
+            text-decoration: underline;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <header>
+            <h1>
+                <svg viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+                    <path d="M12 2L2 7L12 12L22 7L12 2Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+                    <path d="M2 17L12 22L22 17" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+                    <path d="M2 12L12 17L22 12" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+                </svg>
+                SGLang Performance Dashboard
+            </h1>
+            <div class="header-actions">
+                <button class="btn" onclick="refreshData()">Refresh</button>
+                <a href="https://github.com/sgl-project/sglang/actions/workflows/nightly-test-nvidia.yml?query=event%3Aschedule" target="_blank" class="btn">
+                    View Workflow
+                </a>
+            </div>
+        </header>
+
+        <div id="loading" class="loading">
+            <div class="spinner"></div>
+        </div>
+
+        <div id="content" style="display: none;">
+            <div class="stats-row" id="stats-row"></div>
+
+            <div class="filters">
+                <div class="filter-group">
+                    <label>GPU Configuration</label>
+                    <select id="gpu-filter" onchange="handleGpuFilterChange()">
+                    </select>
+                </div>
+                <div class="filter-group">
+                    <label>Model</label>
+                    <select id="model-filter" onchange="handleModelFilterChange(this.value)">
+                    </select>
+                </div>
+                <div class="filter-group">
+                    <label>Variant</label>
+                    <select id="variant-filter" onchange="updateCharts()">
+                        <option value="all">All Variants</option>
+                    </select>
+                </div>
+                <div class="filter-group">
+                    <label>Input/Output Length</label>
+                    <select id="io-len-filter" onchange="updateCharts()">
+                        <option value="all">All Lengths</option>
+                    </select>
+                </div>
+                <div class="filter-group">
+                    <label>Batch Size</label>
+                    <select id="batch-filter" onchange="updateCharts()">
+                        <option value="all">All Batch Sizes</option>
+                    </select>
+                </div>
+            </div>
+
+            <div class="tabs" id="metric-tabs"></div>
+
+            <div class="metric-section">
+                <div class="chart-card">
+                    <h3 id="metric-title">Overall Throughput (tokens/sec)</h3>
+                    <div class="batch-charts-container" id="charts-container">
+                    </div>
+                </div>
+            </div>
+
+            <div class="chart-card" style="margin-top: 24px;">
+                <h3>Recent Benchmark Runs</h3>
+                <table class="data-table" id="runs-table">
+                    <thead>
+                        <tr>
+                            <th>Date</th>
+                            <th>Run ID</th>
+                            <th>Commit</th>
+                            <th>Branch</th>
+                            <th>Models Tested</th>
+                        </tr>
+                    </thead>
+                    <tbody id="runs-table-body">
+                    </tbody>
+                </table>
+            </div>
+        </div>
+
+        <div id="error" class="error" style="display: none;">
+            <h3>Failed to load performance data</h3>
+            <p id="error-message"></p>
+        </div>
+
+        <footer>
+            <p>
+                SGLang Performance Dashboard -
+                <a href="https://github.com/sgl-project/sglang" target="_blank">GitHub</a> |
+                <a href="https://sgl-project.github.io/sglang" target="_blank">Documentation</a>
+            </p>
+        </footer>
+    </div>
+
+    <script src="app.js"></script>
+</body>
+</html>

docs/performance_dashboard/server.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""
+Simple development server for the SGLang Performance Dashboard.
+
+This server:
+1. Serves the static HTML/JS files
+2. Provides an API endpoint to fetch metrics from GitHub
+3. Caches metrics data to reduce API calls
+
+Usage:
+    python server.py
+    python server.py --port 8080
+    python server.py --host 0.0.0.0  # Allow external access
+    python server.py --fetch-on-start
+"""
+
+import argparse
+import http.server
+import io
+import json
+import os
+import socketserver
+import threading
+import time
+import zipfile
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from urllib.parse import urlparse
+
+import requests
+
+GITHUB_REPO = "sgl-project/sglang"
+WORKFLOW_NAME = "nightly-test-nvidia.yml"
+ARTIFACT_PREFIX = "consolidated-metrics-"
+
+# Cache for metrics data with thread-safe lock
+cache_lock = threading.Lock()
+metrics_cache = {
+    "data": [],
+    "last_updated": None,
+    "updating": False,
+}
+
+CACHE_TTL = 300  # 5 minutes
+REQUEST_TIMEOUT = 30  # seconds
+
+
+def get_github_token():
+    """Get GitHub token from environment or gh CLI."""
+    token = os.environ.get("GITHUB_TOKEN")
+    if token:
+        return token
+
+    try:
+        import subprocess
+
+        result = subprocess.run(
+            ["gh", "auth", "token"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        return result.stdout.strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        pass
+
+    return None
+
+
+def fetch_metrics_from_github(days=30):
+    """Fetch metrics from GitHub Actions artifacts."""
+    token = get_github_token()
+    headers = {"Accept": "application/vnd.github.v3+json"}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+
+    # Get workflow runs - only scheduled (nightly) runs, not workflow_dispatch
+    url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/workflows/{WORKFLOW_NAME}/runs"
+    params = {"status": "completed", "per_page": 50, "event": "schedule"}
+
+    try:
+        response = requests.get(
+            url, headers=headers, params=params, timeout=REQUEST_TIMEOUT
+        )
+        if not response.ok:
+            print(f"Failed to fetch workflow runs: {response.status_code}")
+            return []
+    except requests.exceptions.RequestException as e:
+        print(f"Network error fetching workflow runs: {e}")
+        return []
+
+    runs = response.json().get("workflow_runs", [])
+
+    # Filter by date
+    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+    runs = [
+        run
+        for run in runs
+        if datetime.fromisoformat(run["created_at"].replace("Z", "+00:00")) > cutoff
+    ]
+
+    all_metrics = []
+
+    for run in runs[:20]:  # Limit to 20 most recent
+        run_id = run["id"]
+
+        # Get artifacts
+        artifacts_url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/runs/{run_id}/artifacts"
+        try:
+            artifacts_resp = requests.get(
+                artifacts_url, headers=headers, timeout=REQUEST_TIMEOUT
+            )
+            if not artifacts_resp.ok:
+                continue
+        except requests.exceptions.RequestException as e:
+            print(f"Network error fetching artifacts for run {run_id}: {e}")
+            continue
+
+        artifacts = artifacts_resp.json().get("artifacts", [])
+
+        # Find consolidated metrics
+        for artifact in artifacts:
+            if artifact["name"].startswith(ARTIFACT_PREFIX):
+                if not token:
+                    # Without token, we can't download - return metadata only
+                    all_metrics.append(
+                        {
+                            "run_id": str(run_id),
+                            "run_date": run["created_at"],
+                            "commit_sha": run["head_sha"],
+                            "branch": run["head_branch"],
+                            "results": [],
+                        }
+                    )
+                    break
+
+                # Download artifact
+                download_url = f"https://api.github.com/repos/{GITHUB_REPO}/actions/artifacts/{artifact['id']}/zip"
+                try:
+                    download_resp = requests.get(
+                        download_url,
+                        headers=headers,
+                        allow_redirects=True,
+                        timeout=REQUEST_TIMEOUT,
+                    )
+                except requests.exceptions.RequestException as e:
+                    print(f"Network error downloading artifact: {e}")
+                    break
+
+                if download_resp.ok:
+                    try:
+                        with zipfile.ZipFile(io.BytesIO(download_resp.content)) as zf:
+                            json_files = [
+                                f for f in zf.namelist() if f.endswith(".json")
+                            ]
+                            if json_files:
+                                with zf.open(json_files[0]) as f:
+                                    metrics = json.load(f)
+                                    # Ensure required fields
+                                    metrics.setdefault("run_id", str(run_id))
+                                    metrics.setdefault("run_date", run["created_at"])
+                                    metrics.setdefault("commit_sha", run["head_sha"])
+                                    metrics.setdefault("branch", run["head_branch"])
+                                    all_metrics.append(metrics)
+                    except (zipfile.BadZipFile, json.JSONDecodeError) as e:
+                        print(f"Failed to process artifact: {e}")
+                break
+
+    return all_metrics
+
+
+def update_cache_async():
+    """Update the metrics cache in background with thread safety."""
+    with cache_lock:
+        if metrics_cache["updating"]:
+            return
+        metrics_cache["updating"] = True
+
+    try:
+        data = fetch_metrics_from_github()
+        with cache_lock:
+            metrics_cache["data"] = data
+            metrics_cache["last_updated"] = time.time()
+        print(f"Cache updated with {len(data)} metrics records")
+    finally:
+        with cache_lock:
+            metrics_cache["updating"] = False
+
+
+class DashboardHandler(http.server.SimpleHTTPRequestHandler):
+    """HTTP request handler for the dashboard."""
+
+    def __init__(self, *args, directory=None, **kwargs):
+        super().__init__(*args, directory=directory, **kwargs)
+
+    def do_GET(self):
+        parsed = urlparse(self.path)
+
+        # Prevent directory traversal attacks
+        if ".." in parsed.path or parsed.path.startswith("//"):
+            self.send_error(400, "Invalid path")
+            return
+
+        if parsed.path == "/api/metrics":
+            self.handle_metrics_api(parsed)
+        elif parsed.path == "/api/refresh":
+            self.handle_refresh_api()
+        else:
+            super().do_GET()
+
+    def handle_metrics_api(self, parsed):
+        """Handle /api/metrics endpoint."""
+        # Check cache with thread safety
+        with cache_lock:
+            cache_valid = (
+                metrics_cache["last_updated"]
+                and time.time() - metrics_cache["last_updated"] < CACHE_TTL
+            )
+            data = metrics_cache["data"].copy()
+
+        if not cache_valid:
+            # Trigger background update
+            threading.Thread(target=update_cache_async, daemon=True).start()
+
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.end_headers()
+        self.wfile.write(json.dumps(data).encode())
+
+    def handle_refresh_api(self):
+        """Handle /api/refresh endpoint."""
+        threading.Thread(target=update_cache_async, daemon=True).start()
+
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.end_headers()
+        self.wfile.write(json.dumps({"status": "refreshing"}).encode())
+
+    def log_message(self, format, *args):
+        """Custom log format."""
+        print(f"[{self.log_date_time_string()}] {args[0]}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="SGLang Performance Dashboard Server")
+    parser.add_argument("--port", type=int, default=8000, help="Port to serve on")
+    parser.add_argument(
+        "--host",
+        default="127.0.0.1",
+        help="Host to bind to (use 0.0.0.0 for external access)",
+    )
+    parser.add_argument(
+        "--fetch-on-start", action="store_true", help="Fetch metrics on startup"
+    )
+    args = parser.parse_args()
+
+    # Change to dashboard directory
+    dashboard_dir = Path(__file__).parent
+    os.chdir(dashboard_dir)
+
+    if args.fetch_on_start:
+        print("Fetching initial metrics data...")
+        update_cache_async()
+
+    handler = lambda *a, **kw: DashboardHandler(*a, directory=str(dashboard_dir), **kw)
+
+    with socketserver.TCPServer((args.host, args.port), handler) as httpd:
+        print(f"Serving dashboard at http://{args.host}:{args.port}")
+        print("Press Ctrl+C to stop")
+        try:
+            httpd.serve_forever()
+        except KeyboardInterrupt:
+            print("\nShutting down...")
+
+
+if __name__ == "__main__":
+    main()

scripts/ci/save_metrics.py

@@ -44,7 +44,11 @@ def parse_result_file(filepath: str) -> list[dict]:
 
 
 def transform_benchmark_result(result: dict, gpu_config: str, partition: int) -> dict:
-    """Transform a benchmark result to the metrics schema."""
+    """Transform a benchmark result to the metrics schema.
+
+    Note: input_len and output_len are preserved here for the flat benchmarks list,
+    but are also used as grouping keys in benchmarks_by_io_len.
+    """
     # Handle None values safely for numeric conversions
     latency = result.get("latency")
     last_ttft = result.get("last_ttft")
@@ -62,10 +66,20 @@ def transform_benchmark_result(result: dict, gpu_config: str, partition: int) ->
     }
 
 
+def get_io_len_key(input_len: int, output_len: int) -> str:
+    """Generate a key for input/output length combination."""
+    return f"{input_len}_{output_len}"
+
+
 def group_results_by_model(
     results: list[dict], gpu_config: str, partition: int
 ) -> list[dict]:
-    """Group benchmark results by model, variant, and server_args."""
+    """Group benchmark results by model, variant, and server_args.
+
+    Results are organized with two benchmark structures:
+    - benchmarks: flat list of all benchmarks (for backward compatibility)
+    - benchmarks_by_io_len: nested structure grouped by input/output length combinations
+    """
     groups = {}
 
     for result in results:
@@ -85,10 +99,34 @@ def group_results_by_model(
                 "variant": variant,
                 "server_args": server_args,
                 "benchmarks": [],
+                "benchmarks_by_io_len": {},
             }
 
-        groups[key]["benchmarks"].append(
+        transformed = transform_benchmark_result(result, gpu_config, partition)
-            transform_benchmark_result(result, gpu_config, partition)
+
+        # Add to flat benchmarks list (backward compatibility)
+        groups[key]["benchmarks"].append(transformed)
+
+        # Add to nested benchmarks_by_io_len structure
+        input_len = result.get("input_len")
+        output_len = result.get("output_len")
+        if input_len is not None and output_len is not None:
+            io_key = get_io_len_key(input_len, output_len)
+            if io_key not in groups[key]["benchmarks_by_io_len"]:
+                groups[key]["benchmarks_by_io_len"][io_key] = {
+                    "input_len": input_len,
+                    "output_len": output_len,
+                    "benchmarks": [],
+                }
+            # For the nested structure, exclude input_len and output_len from individual benchmarks
+            # since they're already in the parent
+            nested_benchmark = {
+                k: v
+                for k, v in transformed.items()
+                if k not in ("input_len", "output_len")
+            }
+            groups[key]["benchmarks_by_io_len"][io_key]["benchmarks"].append(
+                nested_benchmark
             )
 
     return list(groups.values())