Add nightly test for GLM5.2 LayerSplit (#31512)
This commit is contained in:
@@ -9,8 +9,7 @@ PD-disaggregated GLM-5.2 deployment: a layer-split prefill worker running
|
|||||||
interleave prefill-CP + layer split, and an ordinary decode worker that receives
|
interleave prefill-CP + layer split, and an ordinary decode worker that receives
|
||||||
full cache shards via PD transfer.
|
full cache shards via PD transfer.
|
||||||
|
|
||||||
Sized for the 4-GPU B200 runner (prefill TP=2 + decode TP=2) rather than an
|
Runs nightly on an 8-GPU B200 runner (prefill TP=4 + decode TP=4).
|
||||||
8-GPU deployment, since the 8-gpu-b200 runner is nightly-only.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
@@ -22,10 +21,9 @@ from sglang.test.server_fixtures.disaggregation_fixture import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
register_cuda_ci(
|
register_cuda_ci(
|
||||||
est_time=1200,
|
est_time=450,
|
||||||
stage="extra-b",
|
suite="nightly-8-gpu-b200",
|
||||||
runner_config="4-gpu-b200",
|
nightly=True,
|
||||||
disabled="Temporarily disabled",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -36,13 +34,15 @@ class TestGLM52DSACacheLayerSplit(PDDisaggregationServerBase, GSM8KMixin):
|
|||||||
gsm8k_accuracy_thres = 0.935
|
gsm8k_accuracy_thres = 0.935
|
||||||
gsm8k_num_questions = 1319
|
gsm8k_num_questions = 1319
|
||||||
gsm8k_num_threads = 200
|
gsm8k_num_threads = 200
|
||||||
gsm8k_num_shots = 0
|
gsm8k_num_shots = 20
|
||||||
|
|
||||||
# Prefill worker: interleave prefill-CP + DSA cache layer split on 2 GPUs
|
# Prefill worker: interleave prefill-CP + DSA cache layer split on 4 GPUs
|
||||||
# (TP=2 -> attn_cp_size=2, so KV/indexer layers shard 2-way across CP ranks).
|
# (TP=4 -> attn_cp_size=4, so KV/indexer layers shard 4-way across CP ranks).
|
||||||
extra_prefill_args = [
|
extra_prefill_args = [
|
||||||
"--tp",
|
"--tp",
|
||||||
"2",
|
"4",
|
||||||
|
"--attn-cp-size",
|
||||||
|
"4",
|
||||||
"--dsa-prefill-backend",
|
"--dsa-prefill-backend",
|
||||||
"trtllm",
|
"trtllm",
|
||||||
"--kv-cache-dtype",
|
"--kv-cache-dtype",
|
||||||
@@ -57,20 +57,36 @@ class TestGLM52DSACacheLayerSplit(PDDisaggregationServerBase, GSM8KMixin):
|
|||||||
"4096",
|
"4096",
|
||||||
"--max-prefill-tokens",
|
"--max-prefill-tokens",
|
||||||
"4096",
|
"4096",
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"EAGLE",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"5",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"6",
|
||||||
]
|
]
|
||||||
# Decode worker: ordinary local decode cache on the other 2 GPUs, receives
|
# Decode worker: ordinary local decode cache, receives full shards via PD
|
||||||
# full shards via PD transfer.
|
# transfer.
|
||||||
extra_decode_args = [
|
extra_decode_args = [
|
||||||
"--tp",
|
"--tp",
|
||||||
"2",
|
"4",
|
||||||
"--dsa-decode-backend",
|
"--dsa-decode-backend",
|
||||||
"trtllm",
|
"trtllm",
|
||||||
"--kv-cache-dtype",
|
"--kv-cache-dtype",
|
||||||
"fp8_e4m3",
|
"fp8_e4m3",
|
||||||
"--mem-fraction-static",
|
"--mem-fraction-static",
|
||||||
"0.85",
|
"0.85",
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"EAGLE",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"5",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"6",
|
||||||
"--base-gpu-id",
|
"--base-gpu-id",
|
||||||
"2",
|
"4",
|
||||||
]
|
]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
Reference in New Issue
Block a user