From 9ba4aac47b540eab4874724f768d47da2259f22d Mon Sep 17 00:00:00 2001 From: fzyzcjy <5236035+fzyzcjy@users.noreply.github.com> Date: Thu, 7 May 2026 12:06:04 +0800 Subject: [PATCH] Fix weight_checker e2e OOM on 32GB GPU + move to nightly (#24559) --- test/registered/rl/test_weight_checker_e2e.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/registered/rl/test_weight_checker_e2e.py b/test/registered/rl/test_weight_checker_e2e.py index 29f8b35e4..bff4ad102 100644 --- a/test/registered/rl/test_weight_checker_e2e.py +++ b/test/registered/rl/test_weight_checker_e2e.py @@ -34,7 +34,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=150, suite="stage-b-test-1-gpu-large") +register_cuda_ci(est_time=150, suite="nightly-1-gpu", nightly=True) _MODEL_NAME = "Qwen/Qwen3-0.6B" # We address the up half via the HF-style unfused name "up_proj.weight". sglang's @@ -55,10 +55,17 @@ class TestWeightCheckerE2E(CustomTestCase): @classmethod def setUpClass(cls): cls.url = DEFAULT_URL_FOR_TEST + # --mem-fraction-static 0.7 leaves enough free GPU for _check_tensors's + # CPU->GPU round trip: snapshot lives on CPU, then _compare moves each + # snapshot tensor back to GPU for byte equality. With the default 0.88, + # sglang holds ~29GB on a 32GB GPU and only ~200MB is free, so the + # vocab-embedding round-trip (~600MB) OOMs the snapshot/reset/compare + # cycle in test_z_*. cls.process = popen_launch_server( _MODEL_NAME, cls.url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--mem-fraction-static", "0.7"], ) @classmethod