feature: revamp nightly tests with combined runner (#15324)

.github/workflows/nightly-test-nvidia.yml

@@ -108,6 +108,17 @@ jobs:
         run: |
           bash scripts/ci/ci_install_dependency.sh
 
+      - name: Run common 8-GPU model tests
+        if: always()
+        timeout-minutes: 200
+        env:
+          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
+          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
+          GPU_CONFIG: "8-gpu-h200"
+        run: |
+          cd test
+          python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error
+
       - name: Run test
         timeout-minutes: 30
         env:
@@ -116,82 +127,6 @@ jobs:
           cd test
           python3 run_suite.py --hw cuda --suite nightly-8-gpu-h200 --nightly --continue-on-error
 
-      - name: Run MiniMax-M2 nightly performance test
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-h200"
-        run: |
-          rm -rf test/performance_profiles_minimax_m2/
-          cd test
-          python3 nightly/test_minimax_m2_perf.py
-
-      - name: Publish MiniMax-M2 traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2
-
-      - name: Run Qwen3-235B nightly performance test
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-h200"
-        run: |
-          rm -rf test/performance_profiles_qwen3_235b/
-          cd test
-          python3 nightly/test_qwen3_235b_perf.py
-
-      - name: Publish Qwen3-235B traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b
-
-      - name: Run Kimi-K2-Thinking nightly performance test
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-h200"
-        run: |
-          rm -rf test/performance_profiles_kimi_k2_thinking/
-          cd test
-          python3 nightly/test_kimi_k2_thinking_perf.py
-
-      - name: Publish Kimi-K2-Thinking traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking
-
-      - name: Run GLM-4.6 nightly performance test
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-h200"
-        run: |
-          rm -rf test/performance_profiles_glm_4_6/
-          cd test
-          IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py
-
-      - name: Publish GLM-4.6 traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6
-
   # General tests - 8 GPU H20
   nightly-test-general-8-gpu-h20:
     if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
@@ -446,147 +381,17 @@ jobs:
         run: |
           IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh
 
-      - name: Run Mistral-Large-3 nightly performance test
+      - name: Run common 8-GPU model tests
-        timeout-minutes: 180
+        if: always()
+        timeout-minutes: 200
         env:
           TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
           PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
           GPU_CONFIG: "8-gpu-b200"
           SGLANG_ENABLE_JIT_DEEPGEMM: "0"
         run: |
-          rm -rf test/performance_profiles_mistral_large3/
-          rm -rf test/performance_profiles_mistral_large3_eagle/
           cd test
-          IS_BLACKWELL=1 python3 nightly/test_mistral_large3_perf.py
+          IS_BLACKWELL=1 python3 run_suite.py --hw cuda --suite nightly-8-gpu-common --nightly --timeout-per-file=12000 --continue-on-error
-
-      - name: Publish Mistral-Large-3 traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_mistral_large3
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_mistral_large3_eagle
-
-      - name: Run DeepSeek v3.1 nightly performance test
-        if: always()
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-b200"
-        run: |
-          rm -rf test/performance_profiles_deepseek_v31/
-          cd test
-          IS_BLACKWELL=1 python3 nightly/test_deepseek_v31_perf.py
-
-      - name: Publish DeepSeek v3.1 traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v31
-
-      - name: Run DeepSeek v3.2 nightly performance test
-        if: always()
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-b200"
-        run: |
-          rm -rf test/performance_profiles_deepseek_v32/
-          cd test
-          IS_BLACKWELL=1 python3 nightly/test_deepseek_v32_perf.py
-
-      - name: Publish DeepSeek v3.2 traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_deepseek_v32
-
-      - name: Run Kimi-K2-Thinking nightly performance test
-        if: always()
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-b200"
-        run: |
-          rm -rf test/performance_profiles_kimi_k2_thinking/
-          cd test
-          IS_BLACKWELL=1 python3 nightly/test_kimi_k2_thinking_perf.py
-
-      - name: Publish Kimi-K2-Thinking traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_kimi_k2_thinking
-
-      - name: Run Qwen3-235B nightly performance test
-        if: always()
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-b200"
-        run: |
-          rm -rf test/performance_profiles_qwen3_235b/
-          cd test
-          IS_BLACKWELL=1 python3 nightly/test_qwen3_235b_perf.py
-
-      - name: Publish Qwen3-235B traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_qwen3_235b
-
-      - name: Run GLM-4.6 nightly performance test
-        if: always()
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-b200"
-        run: |
-          rm -rf test/performance_profiles_glm_4_6/
-          cd test
-          IS_BLACKWELL=1 python3 nightly/test_glm_4_6_perf.py
-
-      - name: Publish GLM-4.6 traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_glm_4_6
-
-      - name: Run MiniMax-M2 nightly performance test
-        if: always()
-        timeout-minutes: 180
-        env:
-          TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
-          PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
-          GPU_CONFIG: "8-gpu-b200"
-        run: |
-          rm -rf test/performance_profiles_minimax_m2/
-          cd test
-          IS_BLACKWELL=1 python3 nightly/test_minimax_m2_perf.py
-
-      - name: Publish MiniMax-M2 traces to storage repo
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
-          GITHUB_RUN_ID: ${{ github.run_id }}
-          GITHUB_RUN_NUMBER: ${{ github.run_number }}
-        run: |
-          python3 scripts/ci/publish_traces.py --traces-dir test/performance_profiles_minimax_m2
 
   # Final check job
   check-all-jobs:

test/nightly/accuracy_test_runner.py

@@ -0,0 +1,263 @@
+from dataclasses import dataclass
+from types import SimpleNamespace
+from typing import List, Optional, Tuple
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    ModelLaunchSettings,
+    popen_launch_server,
+    write_github_step_summary,
+)
+
+
+@dataclass
+class AccuracyTestParams:
+    """Parameters for accuracy testing."""
+
+    dataset: str  # e.g., "mgsm_en", "gsm8k", "mmmu", "gpqa"
+    baseline_accuracy: float  # Required: minimum accuracy threshold
+    num_examples: Optional[int] = None
+    num_threads: Optional[int] = None
+    max_tokens: Optional[int] = None
+    return_latency: bool = False
+    # Extended parameters for special evaluations (e.g., GPQA with thinking mode)
+    thinking_mode: Optional[str] = None  # e.g., "deepseek-v3"
+    temperature: Optional[float] = None
+    repeat: Optional[int] = None
+
+
+@dataclass
+class AccuracyTestResult:
+    """Result of an accuracy test."""
+
+    model: str
+    dataset: str
+    passed: bool
+    score: Optional[float]
+    baseline_accuracy: float
+    error: Optional[str]
+    latency: Optional[float] = None
+
+
+def write_accuracy_github_summary(
+    test_name: str,
+    dataset: str,
+    results: List[AccuracyTestResult],
+) -> None:
+    """Write accuracy test results to GitHub step summary.
+
+    Args:
+        test_name: Name of the test
+        dataset: Dataset name used for evaluation
+        results: List of AccuracyTestResult objects
+    """
+    summary = f"## {test_name} - Accuracy ({dataset})\n"
+    summary += "| model | status | score | baseline | error |\n"
+    summary += "| ----- | ------ | ----- | -------- | ----- |\n"
+
+    for result in results:
+        status_emoji = "✅" if result.passed else "❌"
+        score_str = f"{result.score:.4f}" if result.score is not None else "N/A"
+        baseline_str = f"{result.baseline_accuracy:.4f}"
+        error_str = result.error if result.error else "-"
+        summary += f"| {result.model} | {status_emoji} | {score_str} | {baseline_str} | {error_str} |\n"
+
+    write_github_step_summary(summary)
+
+
+def _run_simple_eval(
+    model: ModelLaunchSettings,
+    base_url: str,
+    dataset: str,
+    num_examples: Optional[int] = None,
+    num_threads: Optional[int] = None,
+    max_tokens: Optional[int] = None,
+    return_latency: bool = False,
+    thinking_mode: Optional[str] = None,
+    temperature: Optional[float] = None,
+    repeat: Optional[int] = None,
+) -> Tuple[bool, Optional[str], Optional[dict]]:
+    """Run evaluation using simple_eval backend (run_eval.py).
+
+    Returns:
+        Tuple of (success, error_message, metrics_dict)
+    """
+    process = None
+    try:
+        process = popen_launch_server(
+            model.model_path,
+            base_url,
+            other_args=model.extra_args,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+        )
+
+        args = SimpleNamespace(
+            base_url=base_url,
+            model=model.model_path,
+            eval_name=dataset,
+            num_examples=num_examples,
+            num_threads=num_threads or 1024,
+        )
+
+        if max_tokens is not None:
+            args.max_tokens = max_tokens
+
+        if return_latency:
+            args.return_latency = True
+
+        if thinking_mode is not None:
+            args.thinking_mode = thinking_mode
+
+        if temperature is not None:
+            args.temperature = temperature
+
+        if repeat is not None:
+            args.repeat = repeat
+
+        result = run_eval(args)
+
+        # Handle result format (run_eval can return metrics or (metrics, latency))
+        if return_latency and isinstance(result, tuple):
+            metrics, latency = result
+            metrics["latency"] = round(latency, 4)
+        else:
+            metrics = result
+
+        return True, None, metrics
+
+    except Exception as e:
+        return False, f"Accuracy test exception: {str(e)}", None
+
+    finally:
+        if process:
+            kill_process_tree(process.pid)
+
+
+def _run_few_shot_eval(
+    model: ModelLaunchSettings,
+    base_url: str,
+    num_questions: Optional[int] = None,
+    num_shots: int = 8,
+    max_tokens: int = 512,
+) -> Tuple[bool, Optional[str], Optional[dict]]:
+    """Run evaluation using few_shot backend (few_shot_gsm8k.py).
+
+    Returns:
+        Tuple of (success, error_message, metrics_dict)
+    """
+    from sglang.test.few_shot_gsm8k import run_eval as run_few_shot_eval
+
+    process = None
+    try:
+        process = popen_launch_server(
+            model.model_path,
+            base_url,
+            other_args=model.extra_args,
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+        )
+
+        args = SimpleNamespace(
+            num_shots=num_shots,
+            data_path=None,
+            num_questions=num_questions or 200,
+            max_new_tokens=max_tokens,
+            parallel=128,
+            host="http://127.0.0.1",
+            port=int(base_url.split(":")[-1]),
+        )
+
+        metrics = run_few_shot_eval(args)
+
+        # Normalize metrics format (few_shot returns "accuracy", simple_eval returns "score")
+        if "accuracy" in metrics and "score" not in metrics:
+            metrics["score"] = metrics["accuracy"]
+
+        return True, None, metrics
+
+    except Exception as e:
+        return False, f"Few-shot evaluation exception: {str(e)}", None
+
+    finally:
+        if process:
+            kill_process_tree(process.pid)
+
+
+def run_accuracy_test(
+    model: ModelLaunchSettings,
+    params: AccuracyTestParams,
+    base_url: Optional[str] = None,
+) -> AccuracyTestResult:
+    """Run accuracy test for a single model.
+
+    Args:
+        model: ModelLaunchSettings with model config
+        params: AccuracyTestParams with dataset, baseline, and optional settings
+        base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
+
+    Returns:
+        AccuracyTestResult with test outcome
+    """
+    base_url = base_url or DEFAULT_URL_FOR_TEST
+
+    print(f"\n{'='*60}")
+    print(f"Running ACCURACY test for {model.model_path}")
+    print(f"  Dataset: {params.dataset}")
+    print(f"  Baseline: {params.baseline_accuracy}")
+    print(f"{'='*60}\n")
+
+    # Run evaluation based on dataset type
+    if params.dataset == "gsm8k":
+        success, error, metrics = _run_few_shot_eval(
+            model=model,
+            base_url=base_url,
+            num_questions=params.num_examples,
+            max_tokens=params.max_tokens or 512,
+        )
+    else:
+        success, error, metrics = _run_simple_eval(
+            model=model,
+            base_url=base_url,
+            dataset=params.dataset,
+            num_examples=params.num_examples,
+            num_threads=params.num_threads,
+            max_tokens=params.max_tokens,
+            return_latency=params.return_latency,
+            thinking_mode=params.thinking_mode,
+            temperature=params.temperature,
+            repeat=params.repeat,
+        )
+
+    if not success:
+        print(f"✗ Accuracy test failed for {model.model_path}: {error}")
+        return AccuracyTestResult(
+            model=model.model_path,
+            dataset=params.dataset,
+            passed=False,
+            score=None,
+            baseline_accuracy=params.baseline_accuracy,
+            error=error,
+        )
+
+    # Validate against baseline
+    score = metrics.get("score", 0.0)
+    passed = score >= params.baseline_accuracy
+    latency = metrics.get("latency")
+
+    if passed:
+        print(f"✓ Accuracy {score:.3f} >= baseline {params.baseline_accuracy:.3f}")
+    else:
+        error = f"Accuracy {score:.3f} below baseline {params.baseline_accuracy:.3f}"
+        print(f"✗ {error}")
+
+    return AccuracyTestResult(
+        model=model.model_path,
+        dataset=params.dataset,
+        passed=passed,
+        score=score,
+        baseline_accuracy=params.baseline_accuracy,
+        error=error if not passed else None,
+        latency=latency,
+    )

test/nightly/nightly_utils.py

@@ -6,6 +6,8 @@ import subprocess
 import time
 from typing import List, Optional, Tuple
 
+import requests
+
 from sglang.srt.utils import kill_process_tree
 from sglang.test.nightly_bench_utils import BenchmarkResult, generate_markdown_report
 from sglang.test.test_utils import (
@@ -211,7 +213,7 @@ class NightlyBenchmarkRunner:
         other_args: Optional[List[str]] = None,
         variant: str = "",
         extra_bench_args: Optional[List[str]] = None,
-    ) -> Tuple[List[BenchmarkResult], bool]:
+    ) -> Tuple[List[BenchmarkResult], bool, Optional[float]]:
         """Run a complete benchmark for a single model with server management.
 
         This method handles:
@@ -219,6 +221,7 @@ class NightlyBenchmarkRunner:
         - Profile filename generation
         - Benchmark command construction and execution
         - Result loading and parsing
+        - Fetching speculative decoding accept length (for MTP/EAGLE)
 
         Args:
             model_path: Path to the model
@@ -230,9 +233,10 @@ class NightlyBenchmarkRunner:
             extra_bench_args: Extra arguments for the benchmark command
 
         Returns:
-            Tuple of (list of BenchmarkResult objects, success_bool)
+            Tuple of (list of BenchmarkResult objects, success_bool, avg_spec_accept_length or None)
         """
         benchmark_results = []
+        avg_spec_accept_length = None
         model_description = f"{model_path}" + (f" ({variant})" if variant else "")
 
         # Launch server
@@ -268,19 +272,42 @@ class NightlyBenchmarkRunner:
             result, cmd_success = self.run_benchmark_command(command, model_description)
 
             if not cmd_success:
-                return benchmark_results, False
+                return benchmark_results, False, None
 
             # Load results
             benchmark_results, load_success = self.load_benchmark_results(
                 json_output_file, model_description
             )
 
-            return benchmark_results, load_success
+            # Fetch speculative decoding accept length before killing server
+            avg_spec_accept_length = self._get_spec_accept_length()
+
+            return benchmark_results, load_success, avg_spec_accept_length
 
         finally:
             # Always clean up server process
             kill_process_tree(process.pid)
 
+    def _get_spec_accept_length(self) -> Optional[float]:
+        """Query the server for avg_spec_accept_length metric.
+
+        Returns:
+            The average speculative decoding accept length, or None if not available.
+        """
+        try:
+            response = requests.get(f"{self.base_url}/get_server_info", timeout=10)
+            if response.status_code == 200:
+                server_info = response.json()
+                internal_states = server_info.get("internal_states", [])
+                if internal_states and len(internal_states) > 0:
+                    accept_length = internal_states[0].get("avg_spec_accept_length")
+                    if accept_length is not None:
+                        print(f"  avg_spec_accept_length={accept_length:.2f}")
+                        return accept_length
+        except Exception as e:
+            print(f"  Warning: Could not fetch spec accept length: {e}")
+        return None
+
     def add_report(self, results: List[BenchmarkResult]) -> None:
         """Add benchmark results to the full report.
 

test/nightly/performance_test_runner.py

@@ -0,0 +1,231 @@
+from dataclasses import dataclass, field
+from typing import List, Optional, Tuple
+
+from nightly_utils import NightlyBenchmarkRunner
+
+from sglang.test.nightly_bench_utils import BenchmarkResult
+from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings
+
+
+@dataclass
+class PerformanceTestParams:
+    """Parameters for performance testing."""
+
+    batch_sizes: List[int] = field(default_factory=lambda: [1, 8, 16, 64])
+    input_lens: Tuple[int, ...] = (4096,)
+    output_lens: Tuple[int, ...] = (512,)
+    profile_dir: Optional[str] = None  # None = auto-generate based on is_vlm
+    dataset_name: str = "mmmu"  # For VLM perf test
+    # MTP/EAGLE speculative decoding: minimum accept length threshold (None = no validation)
+    spec_accept_length_threshold: Optional[float] = None
+
+
+@dataclass
+class PerformanceTestResult:
+    """Result of a performance test.
+
+    Aggregates metrics across all batch sizes tested for a single model.
+    """
+
+    model: str
+    passed: bool
+    error: Optional[str]
+    # Aggregate metrics (from the largest batch size result, or None if failed)
+    latency: Optional[float] = None
+    input_throughput: Optional[float] = None
+    output_throughput: Optional[float] = None
+    overall_throughput: Optional[float] = None
+    # All individual benchmark results
+    benchmark_results: Optional[List[BenchmarkResult]] = None
+    # MTP/EAGLE speculative decoding metric
+    avg_spec_accept_length: Optional[float] = None
+
+
+def run_performance_test(
+    model: ModelLaunchSettings,
+    perf_runner: NightlyBenchmarkRunner,
+    batch_sizes: List[int] = None,
+    input_lens: Tuple[int, ...] = (4096,),
+    output_lens: Tuple[int, ...] = (512,),
+    is_vlm: bool = False,
+    dataset_name: str = "mmmu",
+    spec_accept_length_threshold: Optional[float] = None,
+) -> PerformanceTestResult:
+
+    # Set default for mutable argument
+    if batch_sizes is None:
+        batch_sizes = [1, 8, 16, 64]
+
+    print(f"\n{'='*60}")
+    print(f"Running PERFORMANCE test for {model.model_path}")
+    print(f"  Batch sizes: {batch_sizes}")
+    print(f"  Input lens: {input_lens}")
+    print(f"  Output lens: {output_lens}")
+    if spec_accept_length_threshold is not None:
+        print(f"  Spec accept length threshold: {spec_accept_length_threshold}")
+    print(f"{'='*60}\n")
+
+    # Build extra args for benchmarks
+    extra_bench_args = ["--trust-remote-code"]
+    if is_vlm:
+        extra_bench_args.append(f"--dataset-name={dataset_name}")
+
+    try:
+        results, success, avg_spec_accept_length = perf_runner.run_benchmark_for_model(
+            model_path=model.model_path,
+            batch_sizes=batch_sizes,
+            input_lens=input_lens,
+            output_lens=output_lens,
+            other_args=model.extra_args,
+            extra_bench_args=extra_bench_args,
+        )
+
+        if success and results:
+            perf_runner.add_report(results)
+            print(f"✓ Performance test succeeded for {model.model_path}")
+
+            # Validate speculative decoding accept length if threshold is set
+            error_msg = None
+            passed = True
+            if spec_accept_length_threshold is not None:
+                if avg_spec_accept_length is None:
+                    error_msg = f"Spec accept length threshold set but no accept length reported"
+                    passed = False
+                    print(f"✗ {error_msg}")
+                elif avg_spec_accept_length < spec_accept_length_threshold:
+                    error_msg = (
+                        f"Spec accept length {avg_spec_accept_length:.2f} < "
+                        f"threshold {spec_accept_length_threshold}"
+                    )
+                    passed = False
+                    print(f"✗ {error_msg}")
+                else:
+                    print(
+                        f"✓ Spec accept length {avg_spec_accept_length:.2f} >= "
+                        f"threshold {spec_accept_length_threshold}"
+                    )
+
+            # Extract aggregate metrics from the largest batch size result
+            largest_batch_result = max(results, key=lambda r: r.batch_size)
+            return PerformanceTestResult(
+                model=model.model_path,
+                passed=passed,
+                error=error_msg,
+                latency=largest_batch_result.latency,
+                input_throughput=largest_batch_result.input_throughput,
+                output_throughput=largest_batch_result.output_throughput,
+                overall_throughput=largest_batch_result.overall_throughput,
+                benchmark_results=results,
+                avg_spec_accept_length=avg_spec_accept_length,
+            )
+        else:
+            error_msg = f"Performance test failed for {model.model_path}"
+            print(f"✗ {error_msg}")
+            return PerformanceTestResult(
+                model=model.model_path,
+                passed=False,
+                error=error_msg,
+            )
+
+    except Exception as e:
+        error_msg = f"Performance test exception for {model.model_path}: {str(e)}"
+        print(f"✗ {error_msg}")
+        return PerformanceTestResult(
+            model=model.model_path,
+            passed=False,
+            error=error_msg,
+        )
+
+
+def run_performance_for_models(
+    models: List[ModelLaunchSettings],
+    profile_dir: str,
+    test_name: str,
+    base_url: Optional[str] = None,
+    batch_sizes: List[int] = None,
+    input_lens: Tuple[int, ...] = (4096,),
+    output_lens: Tuple[int, ...] = (512,),
+    is_vlm: bool = False,
+    dataset_name: str = "mmmu",
+) -> dict:
+    """Run performance tests for multiple models.
+
+    Args:
+        models: List of ModelLaunchSettings to test
+        profile_dir: Directory for performance profiles
+        test_name: Name for the test (used in reports)
+        base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
+        batch_sizes: Batch sizes for perf test
+        input_lens: Input lengths
+        output_lens: Output lengths
+        is_vlm: Whether these are VLM models
+        dataset_name: Dataset name for VLM benchmarks
+
+    Returns:
+        dict with results:
+        {
+            "all_passed": bool,
+            "results": [PerformanceTestResult, ...]
+        }
+    """
+    base_url = base_url or DEFAULT_URL_FOR_TEST
+
+    # Setup performance runner
+    perf_runner = NightlyBenchmarkRunner(
+        profile_dir=profile_dir,
+        test_name=test_name,
+        base_url=base_url,
+    )
+    perf_runner.setup_profile_directory()
+
+    all_results = []
+    all_passed = True
+
+    for model in models:
+        print("\n" + "=" * 80)
+        print(f"PERFORMANCE TEST: {model.model_path}")
+        print(f"  TP Size: {model.tp_size}")
+        print(f"  Extra Args: {model.extra_args}")
+        print("=" * 80)
+
+        result = run_performance_test(
+            model=model,
+            perf_runner=perf_runner,
+            batch_sizes=batch_sizes,
+            input_lens=input_lens,
+            output_lens=output_lens,
+            is_vlm=is_vlm,
+            dataset_name=dataset_name,
+        )
+
+        all_results.append(result)
+
+        if not result.passed:
+            all_passed = False
+
+    # Write performance report
+    perf_runner.write_final_report()
+
+    # Print summary
+    print("\n" + "=" * 60)
+    print(f"Performance Test Summary: {test_name}")
+    print("=" * 60)
+    for result in all_results:
+        status = "PASS" if result.passed else "FAIL"
+        throughput_str = (
+            f", output: {result.output_throughput:.1f} tok/s"
+            if result.output_throughput
+            else ""
+        )
+        print(f"  {result.model}: {status}{throughput_str}")
+        if result.error:
+            print(f"    Error: {result.error}")
+
+    print("\n" + "=" * 60)
+    print(f"OVERALL: {'ALL PASSED' if all_passed else 'SOME FAILED'}")
+    print("=" * 60 + "\n")
+
+    return {
+        "all_passed": all_passed,
+        "results": all_results,
+    }

test/nightly/run_combined_tests.py

@@ -0,0 +1,192 @@
+from typing import List, Optional
+
+from accuracy_test_runner import (
+    AccuracyTestParams,
+    AccuracyTestResult,
+    run_accuracy_test,
+    write_accuracy_github_summary,
+)
+from nightly_utils import NightlyBenchmarkRunner
+from performance_test_runner import (
+    PerformanceTestParams,
+    PerformanceTestResult,
+    run_performance_test,
+)
+
+from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, ModelLaunchSettings, is_in_ci
+
+
+def run_combined_tests(
+    models: List[ModelLaunchSettings],
+    test_name: str = "NightlyTest",
+    base_url: Optional[str] = None,
+    is_vlm: bool = False,
+    accuracy_params: Optional[AccuracyTestParams] = None,
+    performance_params: Optional[PerformanceTestParams] = None,
+) -> dict:
+    """Run performance and/or accuracy tests for a list of models.
+
+    Args:
+        models: List of ModelLaunchSettings to test
+        test_name: Name for the test (used in reports)
+        base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
+        is_vlm: Whether these are VLM models (affects defaults)
+        accuracy_params: Parameters for accuracy tests (None to skip accuracy)
+        performance_params: Parameters for performance tests (None to skip perf)
+
+    Returns:
+        dict with test results:
+        {
+            "all_passed": bool,
+            "results": [
+                {
+                    "model": str,
+                    "perf_result": PerformanceTestResult/None,
+                    "accuracy_result": AccuracyTestResult/None,
+                    "errors": list,
+                },
+                ...
+            ]
+        }
+    """
+    base_url = base_url or DEFAULT_URL_FOR_TEST
+    run_perf = performance_params is not None
+    run_accuracy = accuracy_params is not None
+
+    # Print test header
+    print("\n" + "=" * 80)
+    print(f"RUNNING: {test_name}")
+    print(f"  Models: {len(models)}")
+    if run_accuracy:
+        print(f"  Accuracy dataset: {accuracy_params.dataset}")
+    if run_perf:
+        print(f"  Performance batches: {performance_params.batch_sizes}")
+    print("=" * 80)
+
+    # Set up performance parameters
+    if run_perf:
+        perf = performance_params
+        profile_dir = perf.profile_dir or (
+            "performance_profiles_vlms"
+            if is_vlm
+            else "performance_profiles_text_models"
+        )
+
+        perf_runner = NightlyBenchmarkRunner(
+            profile_dir=profile_dir,
+            test_name=test_name,
+            base_url=base_url,
+        )
+        perf_runner.setup_profile_directory()
+    else:
+        perf_runner = None
+
+    # Run tests for each model
+    all_results = []
+    all_passed = True
+
+    for model in models:
+        print("\n" + "=" * 80)
+        print(f"TESTING MODEL CONFIG: {model.model_path}")
+        print(f"  TP Size: {model.tp_size}")
+        print(f"  Extra Args: {model.extra_args}")
+        print("=" * 80)
+
+        model_result = {
+            "model": model.model_path,
+            "perf_result": None,
+            "accuracy_result": None,
+            "errors": [],
+        }
+
+        # Run performance test
+        if run_perf:
+            perf_result: PerformanceTestResult = run_performance_test(
+                model=model,
+                perf_runner=perf_runner,
+                batch_sizes=performance_params.batch_sizes,
+                input_lens=performance_params.input_lens,
+                output_lens=performance_params.output_lens,
+                is_vlm=is_vlm,
+                dataset_name=performance_params.dataset_name,
+                spec_accept_length_threshold=performance_params.spec_accept_length_threshold,
+            )
+            model_result["perf_result"] = perf_result
+            if not perf_result.passed:
+                all_passed = False
+                model_result["errors"].append(perf_result.error)
+
+        # Run accuracy test
+        if run_accuracy:
+            acc_result: AccuracyTestResult = run_accuracy_test(
+                model=model,
+                params=accuracy_params,
+                base_url=base_url,
+            )
+            model_result["accuracy_result"] = acc_result
+            if not acc_result.passed:
+                all_passed = False
+                model_result["errors"].append(acc_result.error)
+
+        all_results.append(model_result)
+
+    # Write performance report if we ran perf tests
+    if run_perf and perf_runner:
+        perf_runner.write_final_report()
+
+    # Write accuracy results to GitHub summary if in CI
+    if run_accuracy and is_in_ci():
+        accuracy_results = [
+            r["accuracy_result"] for r in all_results if r["accuracy_result"]
+        ]
+        write_accuracy_github_summary(
+            test_name, accuracy_params.dataset, accuracy_results
+        )
+
+    # Print summary
+    print("\n" + "=" * 60)
+    print(f"{test_name} Results Summary")
+    if run_accuracy:
+        print(f"Dataset: {accuracy_params.dataset}")
+        print(f"Baseline: {accuracy_params.baseline_accuracy}")
+    print("=" * 60)
+    for i, model_result in enumerate(all_results):
+        print(f"\nModel {i + 1}: {model_result['model']}")
+        if run_perf and model_result["perf_result"]:
+            perf = model_result["perf_result"]
+            throughput_str = (
+                f", output: {perf.output_throughput:.1f} tok/s"
+                if perf.output_throughput
+                else ""
+            )
+            accept_str = (
+                f", accept_len: {perf.avg_spec_accept_length:.2f}"
+                if perf.avg_spec_accept_length
+                else ""
+            )
+            print(
+                f"  Performance: {'PASS' if perf.passed else 'FAIL'}{throughput_str}{accept_str}"
+            )
+        if run_accuracy and model_result["accuracy_result"]:
+            acc = model_result["accuracy_result"]
+            print(f"  Accuracy: {'PASS' if acc.passed else 'FAIL'}")
+            if acc.score is not None:
+                print(f"  Score: {acc.score:.3f}")
+        if model_result["errors"]:
+            print(f"  Errors: {model_result['errors']}")
+
+    print("\n" + "=" * 60)
+    print(f"OVERALL: {'ALL TESTS PASSED' if all_passed else 'SOME TESTS FAILED'}")
+    print("=" * 60 + "\n")
+
+    # Raise assertion error if any test failed
+    if not all_passed:
+        failed_models = [r["model"] for r in all_results if r["errors"]]
+        raise AssertionError(
+            f"Tests failed for models: {failed_models}. See results above for details."
+        )
+
+    return {
+        "all_passed": all_passed,
+        "results": all_results,
+    }

test/nightly/test_deepseek_v31_perf.py

@@ -1,87 +0,0 @@
-import unittest
-
-from nightly_utils import NightlyBenchmarkRunner
-
-from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
-
-DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
-PROFILE_DIR = "performance_profiles_deepseek_v31"
-
-
-class TestNightlyDeepseekV31Performance(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = DEEPSEEK_V31_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.batch_sizes = [1, 1, 8, 16, 64]
-        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
-        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
-
-        # Define variant configurations
-        cls.variants = [
-            {
-                "name": "basic",
-                "other_args": [
-                    "--trust-remote-code",
-                    "--tp",
-                    "8",
-                    "--model-loader-extra-config",
-                    '{"enable_multithread_load": true}',
-                ],
-            },
-            {
-                "name": "mtp",
-                "other_args": [
-                    "--trust-remote-code",
-                    "--tp",
-                    "8",
-                    "--speculative-algorithm",
-                    "EAGLE",
-                    "--speculative-num-steps",
-                    "3",
-                    "--speculative-eagle-topk",
-                    "1",
-                    "--speculative-num-draft-tokens",
-                    "4",
-                    "--mem-frac",
-                    "0.7",
-                    "--model-loader-extra-config",
-                    '{"enable_multithread_load": true}',
-                ],
-            },
-        ]
-
-        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
-        cls.runner.setup_profile_directory()
-
-    def test_bench_one_batch(self):
-        failed_variants = []
-
-        try:
-            for variant_config in self.variants:
-                with self.subTest(variant=variant_config["name"]):
-                    results, success = self.runner.run_benchmark_for_model(
-                        model_path=self.model,
-                        batch_sizes=self.batch_sizes,
-                        input_lens=self.input_lens,
-                        output_lens=self.output_lens,
-                        other_args=variant_config["other_args"],
-                        variant=variant_config["name"],
-                    )
-
-                    if not success:
-                        failed_variants.append(variant_config["name"])
-
-                    self.runner.add_report(results)
-        finally:
-            self.runner.write_final_report()
-
-        if failed_variants:
-            raise AssertionError(
-                f"Benchmark failed for {self.model} with the following variants: "
-                f"{', '.join(failed_variants)}"
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/nightly/test_deepseek_v32_gpqa.py

@@ -1,102 +0,0 @@
-import unittest
-from types import SimpleNamespace
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
-from sglang.test.run_eval import run_eval
-from sglang.test.test_utils import (
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    is_in_ci,
-    popen_launch_server,
-    try_cached_model,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=3600, suite="nightly-8-gpu-b200", nightly=True)
-
-# Use the latest version of DeepSeek-V3.2
-DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
-SERVER_LAUNCH_TIMEOUT = 1200
-
-
-class TestDeepseekV32Accuracy(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = try_cached_model(DEEPSEEK_V32_MODEL_PATH)
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        other_args = [
-            "--trust-remote-code",
-            "--tp",
-            "8",
-            "--enable-dp-attention",
-            "--dp",
-            "8",
-            "--tool-call-parser",
-            "deepseekv32",
-            "--reasoning-parser",
-            "deepseek-v3",
-            "--model-loader-extra-config",
-            '{"enable_multithread_load": true,"num_threads": 64}',
-        ]
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=SERVER_LAUNCH_TIMEOUT,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_a_gsm8k(
-        self,
-    ):
-        args = SimpleNamespace(
-            num_shots=20,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-        )
-        metrics = run_eval_few_shot_gsm8k(args)
-        print(f"{metrics=}")
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n'
-            )
-        self.assertGreater(metrics["accuracy"], 0.935)
-
-    def test_gpqa(self):
-        args = SimpleNamespace(
-            base_url=self.base_url,
-            model=DEEPSEEK_V32_MODEL_PATH,
-            eval_name="gpqa",
-            num_examples=198,
-            # use enough threads to allow parallelism
-            num_threads=198,
-            max_tokens=120000,
-            thinking_mode="deepseek-v3",
-            temperature=0.1,
-            # Repeat 4 times for shorter runtime. Ideally we should repeat at least 8 times.
-            repeat=4,
-        )
-
-        print(f"Evaluation start for gpqa")
-        metrics = run_eval(args)
-        print(f"Evaluation end for gpqa: {metrics=}, expected_score=0.835")
-        mean_score = metrics["mean_score"]
-        self.assertGreaterEqual(mean_score, 0.835)
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_gpqa (deepseek-v32)\n" f"Mean Score: {mean_score:.3f}\n"
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/nightly/test_deepseek_v32_nsabackend.py

@@ -1,224 +0,0 @@
-import os
-import unittest
-from types import SimpleNamespace
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
-from sglang.test.test_utils import (
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    is_in_ci,
-    popen_launch_server,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=600, suite="nightly-8-gpu-h200", nightly=True)
-
-DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
-
-# Global list to collect results
-TEST_RESULTS = []
-
-
-class TestDeepseekV32NasBackend_flashmla(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = DEEPSEEK_V32_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        other_args = [
-            "--trust-remote-code",
-            "--attention-backend",
-            "nsa",
-            "--nsa-prefill-backend",
-            "flashmla_sparse",
-            "--nsa-decode-backend",
-            "flashmla_kv",
-            "--tp",
-            "8",
-            "--dp",
-            "8",
-            "--enable-dp-attention",
-        ]
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_a_gsm8k(
-        self,
-    ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
-        args = SimpleNamespace(
-            num_shots=20,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-        )
-        metrics = run_eval_few_shot_gsm8k(args)
-        print(f"{metrics=}")
-
-        if is_in_ci():
-            TEST_RESULTS.append(
-                {
-                    "variant": "flashmla",
-                    "prefill_backend": "flashmla_sparse",
-                    "decode_backend": "flashmla_kv",
-                    "kv_cache": "fp16",
-                    "accuracy": metrics["accuracy"],
-                }
-            )
-        self.assertGreater(metrics["accuracy"], 0.935)
-
-
-class TestDeepseekV32NasBackend_fa3(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = DEEPSEEK_V32_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        other_args = [
-            "--trust-remote-code",
-            "--attention-backend",
-            "nsa",
-            "--nsa-prefill-backend",
-            "fa3",
-            "--nsa-decode-backend",
-            "fa3",
-            "--tp",
-            "8",
-            "--dp",
-            "8",
-            "--enable-dp-attention",
-        ]
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_a_gsm8k(
-        self,
-    ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
-        args = SimpleNamespace(
-            num_shots=20,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-        )
-        metrics = run_eval_few_shot_gsm8k(args)
-        print(f"{metrics=}")
-
-        if is_in_ci():
-            TEST_RESULTS.append(
-                {
-                    "variant": "fa3",
-                    "prefill_backend": "fa3",
-                    "decode_backend": "fa3",
-                    "kv_cache": "fp16",
-                    "accuracy": metrics["accuracy"],
-                }
-            )
-        self.assertGreater(metrics["accuracy"], 0.935)
-
-
-class TestDeepseekV32NasBackend_fp8kvcache(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = DEEPSEEK_V32_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        other_args = [
-            "--trust-remote-code",
-            "--attention-backend",
-            "nsa",
-            "--kv-cache-dtype",
-            "fp8_e4m3",
-            "--tp",
-            "8",
-            "--dp",
-            "8",
-            "--enable-dp-attention",
-        ]
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_a_gsm8k(
-        self,
-    ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
-        args = SimpleNamespace(
-            num_shots=20,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-        )
-        metrics = run_eval_few_shot_gsm8k(args)
-        print(f"{metrics=}")
-
-        if is_in_ci():
-            TEST_RESULTS.append(
-                {
-                    "variant": "fp8kvcache",
-                    "prefill_backend": "default",
-                    "decode_backend": "default",
-                    "kv_cache": "fp8_e4m3",
-                    "accuracy": metrics["accuracy"],
-                }
-            )
-
-            # Write the summary table after all tests complete
-            _write_summary_table()
-        self.assertGreater(metrics["accuracy"], 0.935)
-
-
-def _write_summary_table():
-    """Write a markdown table with all test results."""
-    if not TEST_RESULTS:
-        return
-
-    gpu_config = os.getenv("GPU_CONFIG", "8-gpu-h200")
-
-    # Build table header
-    summary = f"### {DEEPSEEK_V32_MODEL_PATH} GSM8K Accuracy [{gpu_config}]\n\n"
-    summary += "| Variant | Prefill Backend | Decode Backend | KV Cache | Accuracy |\n"
-    summary += "|---------|-----------------|----------------|----------|----------|\n"
-
-    # Add each result as a row
-    for result in TEST_RESULTS:
-        summary += (
-            f"| {result['variant']} | {result['prefill_backend']} | "
-            f"{result['decode_backend']} | {result['kv_cache']} | "
-            f"{result['accuracy']:.3f} |\n"
-        )
-
-    write_github_step_summary(summary)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/nightly/test_deepseek_v32_perf.py

@@ -1,123 +0,0 @@
-import unittest
-
-from nightly_utils import NightlyBenchmarkRunner
-
-from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
-
-DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
-PROFILE_DIR = "performance_profiles_deepseek_v32"
-
-
-class TestNightlyDeepseekV32Performance(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = DEEPSEEK_V32_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.batch_sizes = [1, 8, 16, 64]
-        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
-        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
-
-        # Define variant configurations
-        cls.variants = [
-            {
-                "name": "dp",
-                "other_args": [
-                    "--trust-remote-code",
-                    "--tp",
-                    "8",
-                    "--dp",
-                    "8",
-                    "--enable-dp-attention",
-                    "--model-loader-extra-config",
-                    '{"enable_multithread_load": true}',
-                ],
-            },
-            {
-                "name": "dp+mtp",
-                "other_args": [
-                    "--trust-remote-code",
-                    "--tp",
-                    "8",
-                    "--dp",
-                    "8",
-                    "--enable-dp-attention",
-                    "--speculative-algorithm",
-                    "EAGLE",
-                    "--speculative-num-steps",
-                    "3",
-                    "--speculative-eagle-topk",
-                    "1",
-                    "--speculative-num-draft-tokens",
-                    "4",
-                    "--mem-frac",
-                    "0.7",
-                    "--model-loader-extra-config",
-                    '{"enable_multithread_load": true}',
-                ],
-            },
-            {
-                "name": "tp",
-                "other_args": [
-                    "--trust-remote-code",
-                    "--tp",
-                    "8",
-                    "--model-loader-extra-config",
-                    '{"enable_multithread_load": true}',
-                ],
-            },
-            {
-                "name": "tp+mtp",
-                "other_args": [
-                    "--trust-remote-code",
-                    "--tp",
-                    "8",
-                    "--speculative-algorithm",
-                    "EAGLE",
-                    "--speculative-num-steps",
-                    "3",
-                    "--speculative-eagle-topk",
-                    "1",
-                    "--speculative-num-draft-tokens",
-                    "4",
-                    "--mem-frac",
-                    "0.7",
-                    "--model-loader-extra-config",
-                    '{"enable_multithread_load": true}',
-                ],
-            },
-        ]
-
-        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
-        cls.runner.setup_profile_directory()
-
-    def test_bench_one_batch(self):
-        failed_variants = []
-
-        try:
-            for variant_config in self.variants:
-                with self.subTest(variant=variant_config["name"]):
-                    results, success = self.runner.run_benchmark_for_model(
-                        model_path=self.model,
-                        batch_sizes=self.batch_sizes,
-                        input_lens=self.input_lens,
-                        output_lens=self.output_lens,
-                        other_args=variant_config["other_args"],
-                        variant=variant_config["name"],
-                    )
-
-                    if not success:
-                        failed_variants.append(variant_config["name"])
-
-                    self.runner.add_report(results)
-        finally:
-            self.runner.write_final_report()
-
-        if failed_variants:
-            raise AssertionError(
-                f"Benchmark failed for {self.model} with the following variants: "
-                f"{', '.join(failed_variants)}"
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/nightly/test_deepseek_v32_tp.py

@@ -1,263 +0,0 @@
-import os
-import unittest
-from types import SimpleNamespace
-
-import requests
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
-from sglang.test.send_one import BenchArgs, send_one_prompt
-from sglang.test.test_utils import (
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    is_in_ci,
-    popen_launch_server,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=900, suite="nightly-8-gpu-h200", nightly=True)
-
-DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp"
-
-# Global list to collect results
-TEST_RESULTS = []
-
-
-class TestDeepseekV32_TP(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = DEEPSEEK_V32_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        # Pure TP configuration without --dp and --enable-dp-attention
-        other_args = [
-            "--trust-remote-code",
-            "--attention-backend",
-            "nsa",
-            "--nsa-prefill-backend",
-            "flashmla_sparse",
-            "--nsa-decode-backend",
-            "flashmla_kv",
-            "--tp",
-            "8",
-        ]
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_a_gsm8k(
-        self,
-    ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
-        args = SimpleNamespace(
-            num_shots=20,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-        )
-        metrics = run_eval_few_shot_gsm8k(args)
-        print(f"{metrics=}")
-
-        if is_in_ci():
-            TEST_RESULTS.append(
-                {
-                    "variant": "pure_tp",
-                    "prefill_backend": "flashmla_sparse",
-                    "decode_backend": "flashmla_kv",
-                    "kv_cache": "fp16",
-                    "accuracy": metrics["accuracy"],
-                }
-            )
-        self.assertGreater(metrics["accuracy"], 0.935)
-
-
-class TestDeepseekV32_Partial_TP(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = DEEPSEEK_V32_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        # Partial TP configuration with dp=4 and dp-attention enabled
-        other_args = [
-            "--trust-remote-code",
-            "--attention-backend",
-            "nsa",
-            "--nsa-prefill-backend",
-            "flashmla_sparse",
-            "--nsa-decode-backend",
-            "flashmla_kv",
-            "--tp",
-            "8",
-            "--dp",
-            "4",
-            "--enable-dp-attention",
-        ]
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_a_gsm8k(
-        self,
-    ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
-        args = SimpleNamespace(
-            num_shots=20,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-        )
-        metrics = run_eval_few_shot_gsm8k(args)
-        print(f"{metrics=}")
-
-        if is_in_ci():
-            TEST_RESULTS.append(
-                {
-                    "variant": "partial_tp",
-                    "prefill_backend": "flashmla_sparse",
-                    "decode_backend": "flashmla_kv",
-                    "kv_cache": "fp16",
-                    "accuracy": metrics["accuracy"],
-                }
-            )
-        self.assertGreater(metrics["accuracy"], 0.935)
-
-
-class TestDeepseekV32_TP_MTP(CustomTestCase):
-    """Test DeepSeek V3.2 with pure TP + MTP (EAGLE speculative decoding)."""
-
-    @classmethod
-    def setUpClass(cls):
-        cls.model = DEEPSEEK_V32_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        other_args = [
-            "--trust-remote-code",
-            "--tp",
-            "8",
-            "--speculative-algorithm",
-            "EAGLE",
-            "--speculative-num-steps",
-            "3",
-            "--speculative-eagle-topk",
-            "1",
-            "--speculative-num-draft-tokens",
-            "4",
-            "--mem-frac",
-            "0.7",
-        ]
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-    def test_a_gsm8k(self):
-        requests.get(self.base_url + "/flush_cache")
-        args = SimpleNamespace(
-            num_shots=20,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-        )
-        metrics = run_eval_few_shot_gsm8k(args)
-        print(f"{metrics=}")
-
-        server_info = requests.get(self.base_url + "/get_server_info")
-        avg_spec_accept_length = server_info.json()["internal_states"][0][
-            "avg_spec_accept_length"
-        ]
-        print(f"{avg_spec_accept_length=}")
-
-        if is_in_ci():
-            TEST_RESULTS.append(
-                {
-                    "variant": "tp_mtp",
-                    "prefill_backend": "flashmla_sparse",
-                    "decode_backend": "flashmla_kv",
-                    "kv_cache": "fp16",
-                    "accuracy": metrics["accuracy"],
-                    "avg_spec_accept_length": avg_spec_accept_length,
-                }
-            )
-        self.assertGreater(metrics["accuracy"], 0.935)
-        self.assertGreater(avg_spec_accept_length, 2.5)
-
-    def test_bs_1_speed(self):
-        args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
-        acc_length, speed = send_one_prompt(args)
-
-        print(f"{acc_length=:.2f} {speed=:.2f}")
-
-        if is_in_ci():
-            # Update last result with speed data
-            if TEST_RESULTS and TEST_RESULTS[-1]["variant"] == "tp_mtp":
-                TEST_RESULTS[-1]["speed"] = speed
-
-            # Write the summary table after all tests complete
-            _write_summary_table()
-
-        self.assertGreater(acc_length, 2.5)
-        self.assertGreater(speed, 110)
-
-
-def _format_optional_metric(value, fmt=".2f", suffix=""):
-    """Format an optional metric value, returning '-' if not available."""
-    if value is None:
-        return "-"
-    return f"{value:{fmt}}{suffix}"
-
-
-def _write_summary_table():
-    """Write a markdown table with all test results."""
-    if not TEST_RESULTS:
-        return
-
-    gpu_config = os.getenv("GPU_CONFIG", "8-gpu-h200")
-
-    # Build table header - keep original columns + add MTP-specific ones
-    summary = (
-        f"### {DEEPSEEK_V32_MODEL_PATH} GSM8K Accuracy (TP Tests) [{gpu_config}]\n\n"
-    )
-    summary += "| Variant | Prefill Backend | Decode Backend | KV Cache | Accuracy | Spec Acc Len | Speed |\n"
-    summary += "|---------|-----------------|----------------|----------|----------|--------------|-------|\n"
-
-    # Add each result as a row
-    for result in TEST_RESULTS:
-        summary += (
-            f"| {result['variant']} | {result['prefill_backend']} | "
-            f"{result['decode_backend']} | {result['kv_cache']} | "
-            f"{result['accuracy']:.3f} | "
-            f"{_format_optional_metric(result.get('avg_spec_accept_length'))} | "
-            f"{_format_optional_metric(result.get('speed'), '.1f', ' tok/s')} |\n"
-        )
-
-    write_github_step_summary(summary)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/nightly/test_glm_4_6_perf.py

@@ -1,49 +0,0 @@
-import unittest
-
-from nightly_utils import NightlyBenchmarkRunner
-
-from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
-
-GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6"
-PROFILE_DIR = "performance_profiles_glm_4_6"
-
-
-class TestNightlyGLM46Performance(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = GLM_4_6_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.batch_sizes = [1, 1, 8, 16, 64]
-        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
-        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
-
-        # GLM-4.6 is a 357B MoE model
-        cls.other_args = [
-            "--tp",
-            "8",
-            "--trust-remote-code",
-        ]
-
-        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
-        cls.runner.setup_profile_directory()
-
-    def test_bench_one_batch(self):
-        results, success = self.runner.run_benchmark_for_model(
-            model_path=self.model,
-            batch_sizes=self.batch_sizes,
-            input_lens=self.input_lens,
-            output_lens=self.output_lens,
-            other_args=self.other_args,
-        )
-
-        self.runner.add_report(results)
-        self.runner.write_final_report()
-
-        if not success:
-            raise AssertionError(
-                f"Benchmark failed for {self.model}. Check the logs for details."
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/nightly/test_kimi_k2_thinking_perf.py

@@ -1,54 +0,0 @@
-import unittest
-
-from nightly_utils import NightlyBenchmarkRunner
-
-from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
-
-KIMI_K2_THINKING_MODEL_PATH = "moonshotai/Kimi-K2-Thinking"
-PROFILE_DIR = "performance_profiles_kimi_k2_thinking"
-
-
-class TestNightlyKimiK2ThinkingPerformance(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = KIMI_K2_THINKING_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.batch_sizes = [1, 1, 8, 16, 64]
-        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
-        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
-
-        # Kimi-K2-Thinking requires specific launch arguments
-        cls.other_args = [
-            "--tp",
-            "8",
-            "--trust-remote-code",
-            "--tool-call-parser",
-            "kimi_k2",
-            "--reasoning-parser",
-            "kimi_k2",
-        ]
-
-        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
-        cls.runner.setup_profile_directory()
-
-    def test_bench_one_batch(self):
-        results, success = self.runner.run_benchmark_for_model(
-            model_path=self.model,
-            batch_sizes=self.batch_sizes,
-            input_lens=self.input_lens,
-            output_lens=self.output_lens,
-            other_args=self.other_args,
-            extra_bench_args=["--trust-remote-code"],
-        )
-
-        self.runner.add_report(results)
-        self.runner.write_final_report()
-
-        if not success:
-            raise AssertionError(
-                f"Benchmark failed for {self.model}. Check the logs for details."
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/nightly/test_minimax_m2_perf.py

@@ -1,54 +0,0 @@
-import unittest
-
-from nightly_utils import NightlyBenchmarkRunner
-
-from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
-
-MINIMAX_M2_MODEL_PATH = "MiniMaxAI/MiniMax-M2"
-PROFILE_DIR = "performance_profiles_minimax_m2"
-
-
-class TestNightlyMiniMaxM2Performance(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = MINIMAX_M2_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.batch_sizes = [1, 1, 8, 16, 64]
-        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
-        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
-
-        # MiniMax-M2 is a 230B MoE model with 10B active params
-        cls.other_args = [
-            "--tp",
-            "8",
-            "--ep",
-            "8",
-            "--model-loader-extra-config",
-            '{"enable_multithread_load": true}',
-            "--trust-remote-code",
-        ]
-
-        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
-        cls.runner.setup_profile_directory()
-
-    def test_bench_one_batch(self):
-        results, success = self.runner.run_benchmark_for_model(
-            model_path=self.model,
-            batch_sizes=self.batch_sizes,
-            input_lens=self.input_lens,
-            output_lens=self.output_lens,
-            other_args=self.other_args,
-            extra_bench_args=["--trust-remote-code"],
-        )
-
-        self.runner.add_report(results)
-        self.runner.write_final_report()
-
-        if not success:
-            raise AssertionError(
-                f"Benchmark failed for {self.model}. Check the logs for details."
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/nightly/test_mistral_large3_perf.py

@@ -1,201 +0,0 @@
-import os
-import unittest
-from types import SimpleNamespace
-
-from nightly_utils import NightlyBenchmarkRunner
-
-from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
-from sglang.test.test_utils import (
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    _parse_int_list_env,
-    popen_launch_server,
-)
-
-MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"
-MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle"
-PROFILE_DIR = "performance_profiles_mistral_large3"
-
-
-class TestNightlyMistralLarge3Performance(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        # Set environment variable to disable JIT DeepGemm
-        os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0"
-
-        cls.model = MISTRAL_LARGE3_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.batch_sizes = [1, 1, 8, 16, 64]
-        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
-        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
-
-        # Mistral-Large-3-675B requires TP=8 and trtllm_mla attention backend
-        cls.other_args = [
-            "--tp",
-            "8",
-            "--attention-backend",
-            "trtllm_mla",
-            "--model-loader-extra-config",
-            '{"enable_multithread_load": true}',
-            "--chat-template",
-            "mistral",
-        ]
-
-        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
-        cls.runner.setup_profile_directory()
-
-    @classmethod
-    def tearDownClass(cls):
-        # Clean up environment variable
-        if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ:
-            del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"]
-
-    def test_bench_one_batch(self):
-        results, success = self.runner.run_benchmark_for_model(
-            model_path=self.model,
-            batch_sizes=self.batch_sizes,
-            input_lens=self.input_lens,
-            output_lens=self.output_lens,
-            other_args=self.other_args,
-        )
-
-        self.runner.add_report(results)
-        self.runner.write_final_report()
-
-        if not success:
-            raise AssertionError(
-                f"Benchmark failed for {self.model}. Check the logs for details."
-            )
-
-    def test_accuracy_mgsm(self):
-        """Run MGSM accuracy evaluation for Mistral Large 3."""
-        process = popen_launch_server(
-            model=self.model,
-            base_url=self.base_url,
-            other_args=self.other_args,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-        )
-
-        try:
-            args = SimpleNamespace(
-                base_url=self.base_url,
-                model=self.model,
-                eval_name="mgsm_en",
-                num_examples=None,
-                num_threads=1024,
-            )
-            metrics = run_eval(args)
-            print(f"MGSM accuracy for {self.model}: {metrics['score']}")
-
-            # Placeholder threshold - adjust after first successful run
-            expected_threshold = 0.90
-            self.assertGreaterEqual(
-                metrics["score"],
-                expected_threshold,
-                f"MGSM accuracy {metrics['score']} below threshold {expected_threshold}",
-            )
-        finally:
-            kill_process_tree(process.pid)
-
-
-class TestNightlyMistralLarge3EaglePerformance(unittest.TestCase):
-    """Test Mistral Large 3 with Eagle speculative decoding."""
-
-    @classmethod
-    def setUpClass(cls):
-        # Set environment variable to disable JIT DeepGemm
-        os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0"
-
-        cls.model = MISTRAL_LARGE3_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.batch_sizes = [1, 1, 8, 16, 64]
-        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
-        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
-
-        # Mistral-Large-3 with Eagle speculative decoding
-        # Eagle model is used as draft model for speculative decoding
-        cls.other_args = [
-            "--tp",
-            "8",
-            "--attention-backend",
-            "trtllm_mla",
-            "--speculative-algorithm",
-            "EAGLE",
-            "--speculative-draft-model-path",
-            MISTRAL_LARGE3_EAGLE_MODEL_PATH,
-            "--speculative-num-steps",
-            "3",
-            "--speculative-eagle-topk",
-            "1",
-            "--speculative-num-draft-tokens",
-            "4",
-            "--kv-cache-dtype",
-            "auto",
-            "--model-loader-extra-config",
-            '{"enable_multithread_load": true}',
-            "--chat-template",
-            "mistral",
-        ]
-
-        cls.runner = NightlyBenchmarkRunner(
-            "performance_profiles_mistral_large3_eagle", cls.__name__, cls.base_url
-        )
-        cls.runner.setup_profile_directory()
-
-    @classmethod
-    def tearDownClass(cls):
-        # Clean up environment variable
-        if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ:
-            del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"]
-
-    def test_eagle_bench_one_batch(self):
-        results, success = self.runner.run_benchmark_for_model(
-            model_path=self.model,
-            batch_sizes=self.batch_sizes,
-            input_lens=self.input_lens,
-            output_lens=self.output_lens,
-            other_args=self.other_args,
-        )
-
-        self.runner.add_report(results)
-        self.runner.write_final_report()
-
-        if not success:
-            raise AssertionError(
-                f"Benchmark failed for {self.model} with Eagle. Check the logs for details."
-            )
-
-    def test_eagle_accuracy_mgsm(self):
-        """Run MGSM accuracy evaluation for Mistral Large 3 with Eagle."""
-        process = popen_launch_server(
-            model=self.model,
-            base_url=self.base_url,
-            other_args=self.other_args,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-        )
-
-        try:
-            args = SimpleNamespace(
-                base_url=self.base_url,
-                model=self.model,
-                eval_name="mgsm_en",
-                num_examples=None,
-                num_threads=1024,
-            )
-            metrics = run_eval(args)
-            print(f"MGSM accuracy for {self.model} with Eagle: {metrics['score']}")
-
-            # Placeholder threshold - adjust after first successful run
-            expected_threshold = 0.90
-            self.assertGreaterEqual(
-                metrics["score"],
-                expected_threshold,
-                f"MGSM accuracy {metrics['score']} below threshold {expected_threshold}",
-            )
-        finally:
-            kill_process_tree(process.pid)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/nightly/test_qwen3_235b_perf.py

@@ -1,49 +0,0 @@
-import unittest
-
-from nightly_utils import NightlyBenchmarkRunner
-
-from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
-
-QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507"
-PROFILE_DIR = "performance_profiles_qwen3_235b"
-
-
-class TestNightlyQwen3235BPerformance(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = QWEN3_235B_MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.batch_sizes = [1, 1, 8, 16, 64]
-        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
-        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
-
-        # Qwen3-235B requires TP=8 for 8 GPUs
-        cls.other_args = [
-            "--tp",
-            "8",
-            "--trust-remote-code",
-        ]
-
-        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
-        cls.runner.setup_profile_directory()
-
-    def test_bench_one_batch(self):
-        results, success = self.runner.run_benchmark_for_model(
-            model_path=self.model,
-            batch_sizes=self.batch_sizes,
-            input_lens=self.input_lens,
-            output_lens=self.output_lens,
-            other_args=self.other_args,
-        )
-
-        self.runner.add_report(results)
-        self.runner.write_final_report()
-
-        if not success:
-            raise AssertionError(
-                f"Benchmark failed for {self.model}. Check the logs for details."
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/8-gpu-models/test_deepseek_v31.py

@@ -0,0 +1,79 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
+
+
+@unittest.skipIf(not is_blackwell_system(), "Requires B200")
+class TestDeepseekV31Unified(unittest.TestCase):
+    """Unified test class for DeepSeek-V3.1 performance and accuracy.
+
+    Two variants:
+    - basic: Standard TP=8
+    - mtp: TP=8 + EAGLE speculative decoding
+
+    Each variant runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_deepseek_v31_all_variants(self):
+        """Run performance and accuracy for all DeepSeek-V3.1 variants."""
+        # Define base arguments shared by most variants
+        base_args = [
+            "--tp=8",
+            "--trust-remote-code",
+            "--model-loader-extra-config",
+            '{"enable_multithread_load": true}',
+        ]
+        mtp_args = [
+            "--speculative-algorithm=EAGLE",
+            "--speculative-num-steps=3",
+            "--speculative-eagle-topk=1",
+            "--speculative-num-draft-tokens=4",
+            "--mem-frac=0.7",
+        ]
+
+        variants = [
+            # Variant: "basic" - Standard TP=8
+            ModelLaunchSettings(
+                DEEPSEEK_V31_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+            # Variant: "mtp" - TP=8 + EAGLE speculative decoding
+            ModelLaunchSettings(
+                DEEPSEEK_V31_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args + mtp_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="DeepSeek-V3.1 Unified",
+            accuracy_params=AccuracyTestParams(
+                dataset="gsm8k", baseline_accuracy=0.935
+            ),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_deepseek_v31",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/8-gpu-models/test_deepseek_v32.py

@@ -0,0 +1,194 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
+
+register_cuda_ci(est_time=8000, suite="nightly-8-gpu-common", nightly=True)
+
+DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
+
+BASE_ARGS = [
+    "--trust-remote-code",
+    "--model-loader-extra-config",
+    '{"enable_multithread_load": true}',
+]
+
+DP_ARGS = [
+    "--tp=8",
+    "--dp=8",
+    "--enable-dp-attention",
+]
+
+# Accuracy thresholds
+GSM8K_BASELINE = 0.935
+GPQA_BASELINE = 0.835
+
+
+class TestDeepseekV32Unified(unittest.TestCase):
+    """Unified test class for DeepSeek V3.2 performance and accuracy.
+
+    Tests multiple variants with both performance and accuracy tests:
+    - dp: Standard TP=8 + DP=8 with dp-attention
+    - dp+mtp: DP + EAGLE speculative decoding
+    - tp: Pure TP=8 only
+    - tp+mtp: Pure TP=8 + EAGLE speculative decoding
+    """
+
+    @unittest.skipIf(is_blackwell_system(), "Requires H200 system")
+    def test_deepseek_v32_all_variants(self):
+        """Run performance and accuracy for all DeepSeek V3.2 variants."""
+        TP_ARGS = [
+            "--tp=8",
+        ]
+        MTP_ARGS = [
+            "--speculative-algorithm=EAGLE",
+            "--speculative-num-steps=3",
+            "--speculative-eagle-topk=1",
+            "--speculative-num-draft-tokens=4",
+            "--mem-frac=0.7",
+        ]
+        variants = [
+            # Variant: "dp" - Standard TP=8 + DP=8 with dp-attention
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS,
+            ),
+            # Variant: "dp+mtp" - DP + EAGLE speculative decoding
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + MTP_ARGS,
+            ),
+            # Variant: "tp" - Pure TP=8 only
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + TP_ARGS,
+            ),
+            # Variant: "tp+mtp" - Pure TP=8 + EAGLE speculative decoding
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + TP_ARGS + MTP_ARGS,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="DeepSeek-V3.2 Unified",
+            accuracy_params=AccuracyTestParams(
+                dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE
+            ),
+            performance_params=PerformanceTestParams(
+                batch_sizes=[1, 8, 16, 64],
+                profile_dir="performance_profiles_deepseek_v32",
+            ),
+        )
+
+    @unittest.skipIf(is_blackwell_system(), "Requires H200 system")
+    def test_deepseek_v32_nsa_backends(self):
+        """Test NSA attention backend variants (H200 only).
+
+        Tests three NSA backend configurations:
+        - flashmla: flashmla_sparse prefill + flashmla_kv decode
+        - fa3: FA3 prefill + FA3 decode
+        - fp8kvcache: default backends with FP8 KV cache
+        """
+        NSA_FLASHMLA_ARGS = [
+            "--attention-backend=nsa",
+            "--nsa-prefill-backend=flashmla_sparse",
+            "--nsa-decode-backend=flashmla_kv",
+        ]
+
+        NSA_FA3_ARGS = [
+            "--attention-backend=nsa",
+            "--nsa-prefill-backend=fa3",
+            "--nsa-decode-backend=fa3",
+        ]
+
+        NSA_FP8KV_ARGS = [
+            "--attention-backend=nsa",
+            "--kv-cache-dtype=fp8_e4m3",
+        ]
+
+        nsa_variants = [
+            # flashmla backend
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + NSA_FLASHMLA_ARGS,
+            ),
+            # fa3 backend
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + NSA_FA3_ARGS,
+            ),
+            # fp8 kv cache
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + NSA_FP8KV_ARGS,
+            ),
+        ]
+
+        run_combined_tests(
+            models=nsa_variants,
+            test_name="DeepSeek-V3.2 NSA Backends",
+            accuracy_params=AccuracyTestParams(
+                dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE
+            ),
+            performance_params=PerformanceTestParams(
+                batch_sizes=[1, 8, 16, 64],
+                profile_dir="performance_profiles_deepseek_v32_nsa",
+            ),
+        )
+
+    @unittest.skipIf(not is_blackwell_system(), "Requires B200")
+    def test_deepseek_v32_b200(self):
+        """Test DeepSeek V3.2 with GPQA evaluation using thinking mode (B200 only).
+
+        This test runs GPQA evaluation with the reasoning parser enabled.
+        """
+        B200_REASONING_ARGS = [
+            "--tool-call-parser=deepseekv32",
+            "--reasoning-parser=deepseek-v3",
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + B200_REASONING_ARGS,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="DeepSeek-V3.2 GPQA (B200)",
+            accuracy_params=AccuracyTestParams(
+                dataset="gpqa",
+                baseline_accuracy=GPQA_BASELINE,
+                num_examples=198,
+                num_threads=198,
+                max_tokens=120000,
+                thinking_mode="deepseek-v3",
+                temperature=0.1,
+                repeat=4,
+            ),
+            performance_params=None,  # Skip performance test for GPQA
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/8-gpu-models/test_glm_46.py

@@ -0,0 +1,57 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6"
+
+
+class TestGLM46Unified(unittest.TestCase):
+    """Unified test class for GLM-4.6 performance and accuracy.
+
+    Single variant with simple TP=8 configuration.
+    GLM-4.6 is a 357B MoE model.
+    Runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_glm_46(self):
+        """Run performance and accuracy for GLM-4.6."""
+        base_args = [
+            "--tp=8",
+            "--trust-remote-code",
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                GLM_4_6_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="GLM-4.6 Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_glm_4_6",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/8-gpu-models/test_kimi_k2.py

@@ -0,0 +1,58 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+KIMI_K2_THINKING_MODEL_PATH = "moonshotai/Kimi-K2-Thinking"
+
+
+class TestKimiK2Unified(unittest.TestCase):
+    """Unified test class for Kimi-K2-Thinking performance and accuracy.
+
+    Single variant with TP=8 + tool/reasoning parsers.
+    Runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner with extra_bench_args)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_kimi_k2(self):
+        """Run performance and accuracy for Kimi-K2-Thinking."""
+        base_args = [
+            "--tp=8",
+            "--trust-remote-code",
+            "--tool-call-parser=kimi_k2",
+            "--reasoning-parser=kimi_k2",
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                KIMI_K2_THINKING_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Kimi-K2-Thinking Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.95),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_kimi_k2_thinking",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/8-gpu-models/test_minimax_m2.py

@@ -0,0 +1,60 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+MINIMAX_M2_MODEL_PATH = "MiniMaxAI/MiniMax-M2"
+
+
+class TestMiniMaxM2Unified(unittest.TestCase):
+    """Unified test class for MiniMax-M2 performance and accuracy.
+
+    Single variant with TP=8 + EP=8 configuration.
+    MiniMax-M2 is a 230B MoE model with 10B active params.
+    Runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner with extra_bench_args)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_minimax_m2(self):
+        """Run performance and accuracy for MiniMax-M2."""
+        base_args = [
+            "--tp=8",
+            "--ep=8",
+            "--trust-remote-code",
+            "--model-loader-extra-config",
+            '{"enable_multithread_load": true}',
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                MINIMAX_M2_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="MiniMax-M2 Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_minimax_m2",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/8-gpu-models/test_mistral_large3.py

@@ -0,0 +1,92 @@
+import os
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+# Note: trtllm_mla backend may have hardware-specific behavior
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"
+MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle"
+
+
+@unittest.skipIf(not is_blackwell_system(), "Requires B200")
+class TestMistralLarge3Unified(unittest.TestCase):
+    """Unified test class for Mistral-Large-3 performance and accuracy.
+
+    Two variants:
+    - basic: TP=8 + trtllm_mla backend
+    - eagle: basic + EAGLE speculative decoding with draft model
+
+    Each variant runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        # Set environment variable to disable JIT DeepGemm
+        os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0"
+
+    @classmethod
+    def tearDownClass(cls):
+        # Clean up environment variable
+        if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ:
+            del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"]
+
+    def test_mistral_large3_all_variants(self):
+        """Run performance and accuracy for all Mistral-Large-3 variants."""
+        base_args = [
+            "--tp=8",
+            "--attention-backend=trtllm_mla",
+            "--model-loader-extra-config",
+            '{"enable_multithread_load": true}',
+            "--chat-template=mistral",
+        ]
+        eagle_args = [
+            "--speculative-algorithm=EAGLE",
+            f"--speculative-draft-model-path={MISTRAL_LARGE3_EAGLE_MODEL_PATH}",
+            "--speculative-num-steps=3",
+            "--speculative-eagle-topk=1",
+            "--speculative-num-draft-tokens=4",
+            "--kv-cache-dtype=auto",
+        ]
+
+        variants = [
+            # Variant: "basic" - TP=8 + trtllm_mla backend
+            ModelLaunchSettings(
+                MISTRAL_LARGE3_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+            # Variant: "eagle" - TP=8 + trtllm_mla + EAGLE with draft model
+            ModelLaunchSettings(
+                MISTRAL_LARGE3_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args + eagle_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Mistral-Large-3 Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.90),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_mistral_large3",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/8-gpu-models/test_qwen3_235b.py

@@ -0,0 +1,57 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507"
+
+
+@unittest.skipIf(not is_blackwell_system(), "Requires B200")
+class TestQwen3235BUnified(unittest.TestCase):
+    """Unified test class for Qwen3-235B performance and accuracy.
+
+    Single variant with simple TP=8 configuration.
+    Runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_qwen3_235b(self):
+        """Run performance and accuracy for Qwen3-235B."""
+        base_args = [
+            "--tp=8",
+            "--trust-remote-code",
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                QWEN3_235B_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Qwen3-235B Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.88),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_qwen3_235b",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/run_suite.py

@@ -39,6 +39,9 @@ NIGHTLY_SUITES = {
         "nightly-8-gpu-h200",
         "nightly-8-gpu-h20",
         "nightly-8-gpu-b200",
+        "nightly-8-gpu-h200-basic",  # Basic tests for large models on H200
+        "nightly-8-gpu-b200-basic",  # Basic tests for large models on B200
+        "nightly-8-gpu-common",  # Common tests that run on both H200 and B200
     ],
     HWBackend.AMD: ["nightly-amd", "nightly-amd-8-gpu"],
     HWBackend.CPU: [],