From ed316a26efa474190a18ee517cebffba46e4bcdf Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Mon, 23 Mar 2026 12:34:10 -0700 Subject: [PATCH] Fix CP in-seq-split method for DeepSeek V32 and update related tests (#21192) --- python/sglang/srt/server_args.py | 2 +- test/manual/nightly/test_deepseek_v32_perf.py | 2 +- .../8-gpu-models/test_deepseek_v32_basic.py | 2 +- .../test_deepseek_v32_cp_single_node.py | 92 ---------- .../8-gpu-models/test_deepseek_v32_mtp.py | 2 +- .../cp/test_deepseek_v32_cp_single_node.py | 157 ++++++++++++++++++ test/registered/ep/test_deepep_large.py | 2 +- 7 files changed, 162 insertions(+), 97 deletions(-) delete mode 100644 test/registered/8-gpu-models/test_deepseek_v32_cp_single_node.py create mode 100644 test/registered/cp/test_deepseek_v32_cp_single_node.py diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index f94da569a..a0f0704f3 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1530,7 +1530,7 @@ class ServerArgs: assert ( self.tp_size == 8 ), "Current multi-machine CP support suffers from precision issues. So context parallel only support Single machine(tp_size == 8)" - self.attn_cp_size = self.tp_size + self.attn_cp_size = self.tp_size // self.dp_size logger.warning( f"Enable Context Parallel opt for deeeseekv3.2-DSA, Setting dp_size == {self.dp_size} and moe_dense_tp_size == {self.moe_dense_tp_size}, ep_size == {self.ep_size}, tp_size == {self.tp_size}, kv_cache_dtype == {self.kv_cache_dtype}, moe_a2a_backend {self.moe_a2a_backend} " diff --git a/test/manual/nightly/test_deepseek_v32_perf.py b/test/manual/nightly/test_deepseek_v32_perf.py index b14a189ed..75b258fdd 100644 --- a/test/manual/nightly/test_deepseek_v32_perf.py +++ b/test/manual/nightly/test_deepseek_v32_perf.py @@ -3,7 +3,7 @@ import unittest from sglang.test.nightly_utils import NightlyBenchmarkRunner from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env -DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" PROFILE_DIR = "performance_profiles_deepseek_v32" diff --git a/test/registered/8-gpu-models/test_deepseek_v32_basic.py b/test/registered/8-gpu-models/test_deepseek_v32_basic.py index cd937824f..b1c3c3cf9 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32_basic.py +++ b/test/registered/8-gpu-models/test_deepseek_v32_basic.py @@ -16,7 +16,7 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=360, suite="stage-c-test-8-gpu-h200") -DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" class TestDeepseekV32DP(CustomTestCase): diff --git a/test/registered/8-gpu-models/test_deepseek_v32_cp_single_node.py b/test/registered/8-gpu-models/test_deepseek_v32_cp_single_node.py deleted file mode 100644 index 95684b63a..000000000 --- a/test/registered/8-gpu-models/test_deepseek_v32_cp_single_node.py +++ /dev/null @@ -1,92 +0,0 @@ -import unittest - -from sglang.test.accuracy_test_runner import AccuracyTestParams -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_combined_tests import run_combined_tests -from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system - -register_cuda_ci(est_time=5400, suite="nightly-8-gpu-common", nightly=True) - -DEEPSEEK_V32_EXP_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" - -BASE_ARGS = [ - "--trust-remote-code", - "--model-loader-extra-config", - '{"enable_multithread_load": true, "num_threads": 64}', -] - -DP_ARGS = [ - "--tp=8", - "--dp=2", - "--attn-cp-size=4", - "--enable-dp-attention", -] - -MTP_ARGS = [ - "--speculative-algorithm=EAGLE", - "--speculative-num-steps=3", - "--speculative-eagle-topk=1", - "--speculative-num-draft-tokens=4", - "--mem-frac=0.7", - "--cuda-graph-max-bs=32", - "--max-running-requests=32", -] - -# Accuracy thresholds -GSM8K_BASELINE = 0.935 - -# CP mode arguments -CP_IN_SEQ_SPLIT_ARGS = [ - "--enable-nsa-prefill-context-parallel", - "--nsa-prefill-cp-mode=in-seq-split", -] - -CP_ROUND_ROBIN_ARGS = [ - "--enable-nsa-prefill-context-parallel", - "--nsa-prefill-cp-mode=round-robin-split", - "--attn-cp-size=8", -] - - -class TestDeepseekV32CPSingleNode(unittest.TestCase): - """Test class for DeepSeek V3.2 with NSA context parallelism. - - Tests context parallelism modes with DP+MTP: - - in-seq-split: In-sequence split CP mode - - round-robin-split: Round-robin split CP mode - """ - - @unittest.skipIf(is_blackwell_system(), "Skip on B200 systems") - def test_deepseek_v32_cp_variants(self): - """Run accuracy tests for DeepSeek V3.2 CP variants.""" - variants = [ - # Variant: in-seq-split CP mode with DP+MTP - ModelLaunchSettings( - DEEPSEEK_V32_EXP_MODEL_PATH, - tp_size=8, - extra_args=BASE_ARGS + DP_ARGS + MTP_ARGS + CP_IN_SEQ_SPLIT_ARGS, - env={"SGLANG_ENABLE_SPEC_V2": "1"}, - variant="CP-in-seq-split", - ), - # Variant: round-robin-split CP mode (TP only, no DP) - ModelLaunchSettings( - DEEPSEEK_V32_EXP_MODEL_PATH, - tp_size=8, - extra_args=BASE_ARGS + MTP_ARGS + CP_ROUND_ROBIN_ARGS, - env={"SGLANG_ENABLE_SPEC_V2": "1"}, - variant="CP-round-robin-split", - ), - ] - - run_combined_tests( - models=variants, - test_name="DeepSeek-V3.2-Exp CP Single Node", - accuracy_params=AccuracyTestParams( - dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE - ), - performance_params=None, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/8-gpu-models/test_deepseek_v32_mtp.py b/test/registered/8-gpu-models/test_deepseek_v32_mtp.py index 2b1e529a6..75498322f 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32_mtp.py +++ b/test/registered/8-gpu-models/test_deepseek_v32_mtp.py @@ -19,7 +19,7 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=720, suite="stage-c-test-8-gpu-h200") -FULL_DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" +FULL_DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" class TestDeepseekV32DPMTP(CustomTestCase): diff --git a/test/registered/cp/test_deepseek_v32_cp_single_node.py b/test/registered/cp/test_deepseek_v32_cp_single_node.py new file mode 100644 index 000000000..8b3255777 --- /dev/null +++ b/test/registered/cp/test_deepseek_v32_cp_single_node.py @@ -0,0 +1,157 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +register_cuda_ci(est_time=360, suite="stage-c-test-8-gpu-h200") +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" + + +class TestDeepseekV32CPInSeqSplit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "2", + "--attn-cp-size", + "4", + "--enable-nsa-prefill-context-parallel", + "--nsa-prefill-cp-mode", + "in-seq-split", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', + ] + with envs.SGLANG_ENABLE_SPEC_V2.override(True): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=500, + parallel=32, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_a_gsm8k (deepseek-v32-cp-in-seq-split)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.935) + + +class TestDeepseekV32CPRoundRobinSplit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEEPSEEK_V32_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--tp", + "8", + "--attn-cp-size", + "8", + "--enable-nsa-prefill-context-parallel", + "--nsa-prefill-cp-mode", + "round-robin-split", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-frac", + "0.7", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', + ] + with envs.SGLANG_ENABLE_SPEC_V2.override(True): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + num_shots=20, + data_path=None, + num_questions=500, + parallel=32, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_a_gsm8k (deepseek-v32-cp-in-seq-split)\n" + f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.935) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index e093f86e2..f107b4c40 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=563, suite="stage-c-test-deepep-8-gpu-h200") -DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2-Exp" +DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2" @unittest.skip("Skip for saving ci time")