From 3d508d614f28d2c35da2fdafe422c9bfe454c1b8 Mon Sep 17 00:00:00 2001 From: em-dreyer Date: Wed, 21 Oct 2026 18:53:24 -0700 Subject: [PATCH] [CI] [NVIDIA] gb300: Run DSV4.1-Flash tests on 2 GPUs with engram host tables (#45294) Co-authored-by: Claude Opus 5.5 --- .github/workflows/pr-test.yml | 1 + python/sglang/test/dsv41_flash_common.py | 12 +++++++++--- scripts/ci/runner_configs.yml | 1 + ...p4.py => test_deepseek_v41_flash_gb300_dp2ep2.py} | 10 +++++----- .../test_deepseek_v41_flash_gb300_encoder_bcg.py | 8 ++++---- ...p4.py => test_deepseek_v41_flash_gb300_tp2ep2.py} | 10 +++++----- ...st_deepseek_v41_flash_gb300_trtllm_bounded_bcg.py | 8 ++++---- test/run_suite.py | 1 + 8 files changed, 30 insertions(+), 21 deletions(-) rename test/registered/e2e/models/{test_deepseek_v41_flash_gb300_dp4ep4.py => test_deepseek_v41_flash_gb300_dp2ep2.py} (75%) rename test/registered/e2e/models/{test_deepseek_v41_flash_gb300_tp4ep4.py => test_deepseek_v41_flash_gb300_tp2ep2.py} (57%) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 1df7aa44e3..d421541fe7 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -446,6 +446,7 @@ jobs: fail-fast: false matrix: include: + - { runner_config: 2-gpu-gb300, run_timeout: '30', timeout_per_file: '1800' } - { runner_config: 4-gpu-gb300, run_timeout: '30', timeout_per_file: '1800' } uses: ./.github/workflows/_pr-test-stage.yml with: diff --git a/python/sglang/test/dsv41_flash_common.py b/python/sglang/test/dsv41_flash_common.py index 17d4360416..be9105cdc9 100644 --- a/python/sglang/test/dsv41_flash_common.py +++ b/python/sglang/test/dsv41_flash_common.py @@ -4,6 +4,11 @@ Each server configuration lives in its own file under ``test/registered/e2e/models/test_deepseek_v41_flash_gb300_*.py`` so the configurations partition independently across GB300 runners. All use default decoder bounded replay and explicitly enabled breakable prefill graphs. + +They run on two GB300 GPUs (TP2) with the engram tables in host memory +(SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE). The two engram tables are ~189 GiB in FP8 +(2 x ~384M rows x 264 B), so at TP2 offloading them frees ~94 GiB per GPU and the +remaining weights (~143 GiB/GPU) leave room for KV cache, graphs and DSPARK. """ import requests @@ -19,7 +24,7 @@ from sglang.test.test_utils import ( MODEL = "deepseek-ai/DeepSeek-V4.1-Flash" # Tensor-parallel attention and expert-parallel MoE, without DP attention. -TP4_EP4_ARGS = ["--attn-dp-size", "1", "--ep-size", "4"] +TP2_EP2_ARGS = ["--attn-dp-size", "1", "--ep-size", "2"] class DSV41FlashAccuracyMixin(GSM8KMixin): @@ -49,7 +54,8 @@ class DSV41FlashAccuracyMixin(GSM8KMixin): "8192", ] parallel_args = [] - server_env = {} + # Keep the engram tables in host memory so DSV4.1-Flash fits on two GB300s. + server_env = {"SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE": "1"} @classmethod def setUpClass(cls): @@ -62,7 +68,7 @@ class DSV41FlashAccuracyMixin(GSM8KMixin): other_args=[ "--trust-remote-code", "--tp", - "4", + "2", "--speculative-algorithm", "DSPARK", "--speculative-dspark-block-size", diff --git a/scripts/ci/runner_configs.yml b/scripts/ci/runner_configs.yml index b9c032b1b0..e06d0b29bf 100644 --- a/scripts/ci/runner_configs.yml +++ b/scripts/ci/runner_configs.yml @@ -22,6 +22,7 @@ runner_configs: 1-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 1-gpu-h100 } 2-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 2-gpu-h100 } 4-gpu-b200: { install: *default, artifact_version: v6, install_timeout: "20", runs_on: $b200_runner } + 2-gpu-gb300: { install: *default, artifact_version: v6, install_timeout: "20", runs_on: 2-gpu-gb300 } 4-gpu-gb300: { install: *default, artifact_version: v6, install_timeout: "20", runs_on: 4-gpu-gb300 } 4-gpu-h100: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 4-gpu-h100 } 8-gpu-h200: { install: *default, artifact_version: v4, install_timeout: "20", runs_on: 8-gpu-h200 } diff --git a/test/registered/e2e/models/test_deepseek_v41_flash_gb300_dp4ep4.py b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_dp2ep2.py similarity index 75% rename from test/registered/e2e/models/test_deepseek_v41_flash_gb300_dp4ep4.py rename to test/registered/e2e/models/test_deepseek_v41_flash_gb300_dp2ep2.py index ad82a647a7..dd72f9858f 100644 --- a/test/registered/e2e/models/test_deepseek_v41_flash_gb300_dp4ep4.py +++ b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_dp2ep2.py @@ -1,4 +1,4 @@ -"""DeepSeek-V4.1-Flash + DSPARK accuracy on four GB300 GPUs: DP-attention4 + EP4. +"""DeepSeek-V4.1-Flash + DSPARK accuracy on two GB300 GPUs: DP-attention2 + EP2. The DP+EP recipe is intentionally exercised rather than skipped: startup or accuracy failures must be visible in CI while this path is being validated. @@ -10,10 +10,10 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.dsv41_flash_common import DSV41FlashAccuracyMixin from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=600, stage="base-c", runner_config="4-gpu-gb300") +register_cuda_ci(est_time=600, stage="base-c", runner_config="2-gpu-gb300") -class TestDSV41FlashDP4EP4DSpark(DSV41FlashAccuracyMixin, CustomTestCase): +class TestDSV41FlashDP2EP2DSpark(DSV41FlashAccuracyMixin, CustomTestCase): """DP attention and EP MoE using the built-in gather/reduce path. DSPARK with DP attention does not support the DeepEP backend. @@ -21,9 +21,9 @@ class TestDSV41FlashDP4EP4DSpark(DSV41FlashAccuracyMixin, CustomTestCase): parallel_args = [ "--attn-dp-size", - "4", + "2", "--ep-size", - "4", + "2", "--enable-dp-lm-head", "--moe-a2a-backend", "none", diff --git a/test/registered/e2e/models/test_deepseek_v41_flash_gb300_encoder_bcg.py b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_encoder_bcg.py index 28ce1715ba..99442e3272 100644 --- a/test/registered/e2e/models/test_deepseek_v41_flash_gb300_encoder_bcg.py +++ b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_encoder_bcg.py @@ -1,20 +1,20 @@ """DeepSeek-V4.1-Flash + DSPARK accuracy on GB300 with encoder bounded replay. -Run GSM8K and MMMU-Pro with TP4 + EP4, encoder SWA bounded replay, and +Run GSM8K and MMMU-Pro with TP2 + EP2, encoder SWA bounded replay, and breakable prefill graphs using the shared accuracy recipe. """ import unittest from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.dsv41_flash_common import TP4_EP4_ARGS, DSV41FlashAccuracyMixin +from sglang.test.dsv41_flash_common import TP2_EP2_ARGS, DSV41FlashAccuracyMixin from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=600, stage="base-c", runner_config="4-gpu-gb300") +register_cuda_ci(est_time=600, stage="base-c", runner_config="2-gpu-gb300") class TestDSV41FlashEncoderBoundedBCG(DSV41FlashAccuracyMixin, CustomTestCase): - parallel_args = TP4_EP4_ARGS + parallel_args = TP2_EP2_ARGS graph_args = [ *DSV41FlashAccuracyMixin.graph_args, "--enable-encoder-swa-bounded-replay", diff --git a/test/registered/e2e/models/test_deepseek_v41_flash_gb300_tp4ep4.py b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_tp2ep2.py similarity index 57% rename from test/registered/e2e/models/test_deepseek_v41_flash_gb300_tp4ep4.py rename to test/registered/e2e/models/test_deepseek_v41_flash_gb300_tp2ep2.py index f9426bda6f..e328489da4 100644 --- a/test/registered/e2e/models/test_deepseek_v41_flash_gb300_tp4ep4.py +++ b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_tp2ep2.py @@ -1,4 +1,4 @@ -"""DeepSeek-V4.1-Flash + DSPARK accuracy on four GB300 GPUs: TP4 + EP4. +"""DeepSeek-V4.1-Flash + DSPARK accuracy on two GB300 GPUs: TP2 + EP2. GSM8K and MMMU-Pro with tensor-parallel attention and expert-parallel MoE, without DP attention. @@ -7,16 +7,16 @@ without DP attention. import unittest from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.dsv41_flash_common import TP4_EP4_ARGS, DSV41FlashAccuracyMixin +from sglang.test.dsv41_flash_common import TP2_EP2_ARGS, DSV41FlashAccuracyMixin from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=600, stage="base-c", runner_config="4-gpu-gb300") +register_cuda_ci(est_time=600, stage="base-c", runner_config="2-gpu-gb300") -class TestDSV41FlashTP4EP4DSpark(DSV41FlashAccuracyMixin, CustomTestCase): +class TestDSV41FlashTP2EP2DSpark(DSV41FlashAccuracyMixin, CustomTestCase): """Tensor-parallel attention and expert-parallel MoE, without DP attention.""" - parallel_args = TP4_EP4_ARGS + parallel_args = TP2_EP2_ARGS if __name__ == "__main__": diff --git a/test/registered/e2e/models/test_deepseek_v41_flash_gb300_trtllm_bounded_bcg.py b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_trtllm_bounded_bcg.py index 33b143b356..550db6d186 100644 --- a/test/registered/e2e/models/test_deepseek_v41_flash_gb300_trtllm_bounded_bcg.py +++ b/test/registered/e2e/models/test_deepseek_v41_flash_gb300_trtllm_bounded_bcg.py @@ -1,6 +1,6 @@ -"""DeepSeek-V4.1-Flash + DSPARK accuracy on four GB300 GPUs: TRT-LLM bounded BCG. +"""DeepSeek-V4.1-Flash + DSPARK accuracy on two GB300 GPUs: TRT-LLM bounded BCG. -TP4 + EP4 with TRT-LLM varlen attention and decoder SWA bounded replay inside +TP2 + EP2 with TRT-LLM varlen attention and decoder SWA bounded replay inside breakable prefill graphs. """ @@ -10,7 +10,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.dsv41_flash_common import DSV41FlashAccuracyMixin from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=600, stage="base-c", runner_config="4-gpu-gb300") +register_cuda_ci(est_time=600, stage="base-c", runner_config="2-gpu-gb300") class TestDSV41FlashTrtllmBoundedBCG(DSV41FlashAccuracyMixin, CustomTestCase): @@ -20,7 +20,7 @@ class TestDSV41FlashTrtllmBoundedBCG(DSV41FlashAccuracyMixin, CustomTestCase): "--attn-dp-size", "1", "--ep-size", - "4", + "2", "--dsv4-attn-backend", "trtllm", ] diff --git a/test/run_suite.py b/test/run_suite.py index c56b709508..116f6461ef 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -92,6 +92,7 @@ PER_COMMIT_SUITES = { "base-b-test-diffusion-unit-1-gpu-h100", "base-c-test-4-gpu-h100", "base-c-test-4-gpu-b200", + "base-c-test-2-gpu-gb300", "base-c-test-4-gpu-gb300", "base-c-test-8-gpu-h20", "base-c-test-8-gpu-h200",