diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 2163e33dc..319e41e04 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -902,9 +902,9 @@ jobs: docker exec ci_sglang bash -c "cd /sglang-checkout && NCCL_DEBUG=INFO RCCL_DEBUG=INFO torchrun --nproc_per_node=8 scripts/ci/amd/test_rccl_multi_gpu.py" - name: Run test - timeout-minutes: 60 + timeout-minutes: 120 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 5400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd-mi35x-rocm720: needs: [check-changes] diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 6a219d417..9e46a5b41 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -985,7 +985,7 @@ jobs: - name: Run test timeout-minutes: 120 run: | - bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 5400 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} stage-c-test-large-8-gpu-amd-mi35x: needs: [check-changes, call-gate, wait-for-stage-b-amd] diff --git a/test/registered/amd/test_moriep_small.py b/test/registered/amd/test_moriep_small.py index 64a22208c..d32cf3002 100644 --- a/test/registered/amd/test_moriep_small.py +++ b/test/registered/amd/test_moriep_small.py @@ -11,6 +11,7 @@ from sglang.test.ci.ci_register import register_amd_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, + DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST_NEXTN, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -41,7 +42,7 @@ def wait_all_ports_release(base_url, timeout_s=60): print(f"Warning: some ports still occupied after {timeout_s}s") -register_amd_ci(est_time=1200, suite="stage-c-test-large-8-gpu-amd") +register_amd_ci(est_time=5400, suite="stage-c-test-large-8-gpu-amd") common_args = [ "--tp-size", @@ -60,15 +61,15 @@ common_args = [ "1", "--enable-dp-lm-head", "--mem-fraction-static", - "0.72", # relax for mi300x + "0.7", # relax for mi300x "--chunked-prefill-size", - "16384", + "1024", "--max-running-requests", "128", "--context-length", - "12288", + "4096", "--max-total-tokens", - "131072", + "32768", "--attention-backend", "aiter", "--cuda-graph-max-bs", @@ -84,6 +85,8 @@ mtp_args = [ "1", "--speculative-num-draft-tokens", "4", + "--speculative-draft-model-path", + DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST_NEXTN, ] @@ -98,7 +101,7 @@ class TestPureDP(CustomTestCase): env = dict(os.environ) env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" - env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "4096" + env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory cls.process = popen_launch_server( @@ -129,7 +132,7 @@ class TestPureDP(CustomTestCase): metrics = run_eval_few_shot_gsm8k(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], 0.935) + self.assertGreaterEqual(metrics["accuracy"], 0.90) class TestMTP(CustomTestCase): @@ -143,7 +146,7 @@ class TestMTP(CustomTestCase): env = dict(os.environ) env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" - env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "4096" + env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory cls.process = popen_launch_server( @@ -173,7 +176,7 @@ class TestMTP(CustomTestCase): ) metrics = run_eval_few_shot_gsm8k(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], 0.92) + self.assertGreaterEqual(metrics["accuracy"], 0.90) server_info = requests.get(self.base_url + "/server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ @@ -197,7 +200,7 @@ class TestNormal(CustomTestCase): env = dict(os.environ) env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" - env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "4096" + env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory cls.process = popen_launch_server( @@ -228,7 +231,7 @@ class TestNormal(CustomTestCase): metrics = run_eval_few_shot_gsm8k(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], 0.935) + self.assertGreaterEqual(metrics["accuracy"], 0.90) class TestLowLatency(CustomTestCase): @@ -245,7 +248,7 @@ class TestLowLatency(CustomTestCase): env = dict(os.environ) env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" - env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "4096" + env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory # FIXME(billishyahao): enable p2p due to no rdma devices on CI machine # env["MORI_DISABLE_P2P"] = "1" @@ -278,7 +281,7 @@ class TestLowLatency(CustomTestCase): metrics = run_eval_few_shot_gsm8k(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], 0.935) + self.assertGreaterEqual(metrics["accuracy"], 0.90) class TestTBOwithNormal(CustomTestCase): @@ -296,7 +299,7 @@ class TestTBOwithNormal(CustomTestCase): env = dict(os.environ) env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" - env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "4096" + env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory cls.process = popen_launch_server( @@ -327,7 +330,7 @@ class TestTBOwithNormal(CustomTestCase): metrics = run_eval_few_shot_gsm8k(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], 0.935) + self.assertGreaterEqual(metrics["accuracy"], 0.90) class TestTBOwithLowLatency(CustomTestCase): @@ -345,7 +348,7 @@ class TestTBOwithLowLatency(CustomTestCase): env = dict(os.environ) env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" - env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "4096" + env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory # FIXME(billishyahao): enable p2p due to no rdma devices on CI machine # env["MORI_DISABLE_P2P"] = "1" @@ -378,7 +381,7 @@ class TestTBOwithLowLatency(CustomTestCase): metrics = run_eval_few_shot_gsm8k(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], 0.935) + self.assertGreaterEqual(metrics["accuracy"], 0.90) class TestMTPwithTBONormal(CustomTestCase): @@ -400,9 +403,8 @@ class TestMTPwithTBONormal(CustomTestCase): env = dict(os.environ) env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" - env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "4096" + env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory - env["SGLANG_ENABLE_SPEC_V2"] = "false" env["MORI_ENABLE_SDMA"] = "true" cls.process = popen_launch_server( @@ -432,7 +434,7 @@ class TestMTPwithTBONormal(CustomTestCase): ) metrics = run_eval_few_shot_gsm8k(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], 0.92) + self.assertGreaterEqual(metrics["accuracy"], 0.90) server_info = requests.get(self.base_url + "/server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ @@ -461,7 +463,7 @@ class TestMTPwithTBOLowLatency(CustomTestCase): env = dict(os.environ) env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" - env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "4096" + env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" env["SGLANG_ENABLE_SPEC_V2"] = "false" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory # FIXME(billishyahao): enable p2p due to no rdma devices on CI machine @@ -495,7 +497,7 @@ class TestMTPwithTBOLowLatency(CustomTestCase): ) metrics = run_eval_few_shot_gsm8k(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], 0.92) + self.assertGreaterEqual(metrics["accuracy"], 0.90) server_info = requests.get(self.base_url + "/server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][