diff --git a/test/registered/cuda_graph/piecewise/test_piecewise_cuda_graph_support_1_gpu.py b/test/registered/cuda_graph/piecewise/test_piecewise_cuda_graph_support_1_gpu.py index cd7ca1ae7..4ec0637d6 100644 --- a/test/registered/cuda_graph/piecewise/test_piecewise_cuda_graph_support_1_gpu.py +++ b/test/registered/cuda_graph/piecewise/test_piecewise_cuda_graph_support_1_gpu.py @@ -13,6 +13,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, SimpleNamespace, + is_in_amd_ci, popen_launch_server, ) @@ -21,6 +22,13 @@ register_cuda_ci(est_time=180, suite="nightly-1-gpu", nightly=True) register_amd_ci(est_time=180, suite="stage-b-test-1-gpu-large-amd") +# The 192GB mi300x runners have less headroom than the 256GB mi325x ones they +# replaced: the auto-derived fraction left too little room for the ViT +# activations plus the piecewise graph private pools, and the server died under +# the 1024-thread gsm8k load. +AMD_MEM_FRACTION_STATIC = 0.6 + + class TestPiecewiseCudaGraphQwen25VL(CustomTestCase): """Test piecewise CUDA graph with Qwen2.5-VL-7B-Instruct model""" @@ -28,14 +36,17 @@ class TestPiecewiseCudaGraphQwen25VL(CustomTestCase): def setUpClass(cls): cls.model = "Qwen/Qwen2.5-VL-7B-Instruct" cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--cuda-graph-backend-prefill=tc_piecewise", + "--disable-radix-cache", + ] + if is_in_amd_ci(): + other_args += ["--mem-fraction-static", str(AMD_MEM_FRACTION_STATIC)] cls.process = popen_launch_server( cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--cuda-graph-backend-prefill=tc_piecewise", - "--disable-radix-cache", - ], + other_args=other_args, ) @classmethod @@ -64,12 +75,16 @@ class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase): model_path = "Qwen/Qwen2.5-VL-3B-Instruct" chat_template = get_chat_template_by_model_path(model_path) text = f"{chat_template.image_token}What is in this picture? Answer: " + extra_args = ( + {"mem_fraction_static": AMD_MEM_FRACTION_STATIC} if is_in_amd_ci() else {} + ) engine = Engine( model_path=model_path, enable_multimodal=True, is_embedding=True, cuda_graph_backend_prefill="tc_piecewise", + **extra_args, ) out = engine.encode([text], image_data=[DEFAULT_IMAGE_URL])[0]["embedding"] engine.shutdown() @@ -80,6 +95,7 @@ class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase): enable_multimodal=True, is_embedding=True, cuda_graph_backend_prefill="disabled", + **extra_args, ) out_without_pcg = engine.encode([text], image_data=[DEFAULT_IMAGE_URL])[0][ "embedding"