[diffusion] feat: add performance mode server args (#24491)
This commit is contained in:
@@ -53,12 +53,10 @@ class TestDFlashServerBase(CustomTestCase, MatchedStopMixin, GSM8KMixin):
|
||||
old_value = os.environ.get("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN")
|
||||
os.environ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN"] = "1"
|
||||
try:
|
||||
with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(
|
||||
1
|
||||
), envs.SGLANG_SPEC_NAN_DETECTION.override(
|
||||
True
|
||||
), envs.SGLANG_SPEC_OOB_DETECTION.override(
|
||||
True
|
||||
with (
|
||||
envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1),
|
||||
envs.SGLANG_SPEC_NAN_DETECTION.override(True),
|
||||
envs.SGLANG_SPEC_OOB_DETECTION.override(True),
|
||||
):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
|
||||
@@ -49,9 +49,10 @@ class TestDeepseekV3FP4MTP(CustomTestCase):
|
||||
"--model-loader-extra-config",
|
||||
'{"enable_multithread_load": true,"num_threads": 64}',
|
||||
]
|
||||
with envs.SGLANG_SPEC_NAN_DETECTION.override(
|
||||
True
|
||||
), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
|
||||
with (
|
||||
envs.SGLANG_SPEC_NAN_DETECTION.override(True),
|
||||
envs.SGLANG_SPEC_OOB_DETECTION.override(True),
|
||||
):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
|
||||
@@ -59,12 +59,10 @@ class TestEagleConstrainedDecoding(
|
||||
cls.grammar_backend,
|
||||
]
|
||||
launch_args.extend(cls.other_launch_args)
|
||||
with envs.SGLANG_ENABLE_SPEC_V2.override(
|
||||
cls.spec_v2
|
||||
), envs.SGLANG_SPEC_NAN_DETECTION.override(
|
||||
True
|
||||
), envs.SGLANG_SPEC_OOB_DETECTION.override(
|
||||
True
|
||||
with (
|
||||
envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2),
|
||||
envs.SGLANG_SPEC_NAN_DETECTION.override(True),
|
||||
envs.SGLANG_SPEC_OOB_DETECTION.override(True),
|
||||
):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
|
||||
@@ -57,9 +57,10 @@ class TestEAGLE3EngineDPAttention(CustomTestCase):
|
||||
"--cuda-graph-max-bs",
|
||||
"64",
|
||||
]
|
||||
with envs.SGLANG_SPEC_NAN_DETECTION.override(
|
||||
True
|
||||
), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
|
||||
with (
|
||||
envs.SGLANG_SPEC_NAN_DETECTION.override(True),
|
||||
envs.SGLANG_SPEC_OOB_DETECTION.override(True),
|
||||
):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
|
||||
@@ -63,14 +63,11 @@ class TestEagle3ServerBase(CustomTestCase, MatchedStopMixin):
|
||||
*[str(i) for i in range(1, cls.max_running_requests + 1)],
|
||||
]
|
||||
launch_args.extend(cls.other_launch_args)
|
||||
with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(
|
||||
1
|
||||
), envs.SGLANG_SPEC_NAN_DETECTION.override(
|
||||
True
|
||||
), envs.SGLANG_SPEC_OOB_DETECTION.override(
|
||||
True
|
||||
), envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(
|
||||
True
|
||||
with (
|
||||
envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1),
|
||||
envs.SGLANG_SPEC_NAN_DETECTION.override(True),
|
||||
envs.SGLANG_SPEC_OOB_DETECTION.override(True),
|
||||
envs.SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN.override(True),
|
||||
):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
|
||||
@@ -65,9 +65,10 @@ class TestEagleDPAttnServerSmall(CustomTestCase):
|
||||
"--speculative-num-draft-tokens",
|
||||
"4",
|
||||
]
|
||||
with envs.SGLANG_SPEC_NAN_DETECTION.override(
|
||||
True
|
||||
), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
|
||||
with (
|
||||
envs.SGLANG_SPEC_NAN_DETECTION.override(True),
|
||||
envs.SGLANG_SPEC_OOB_DETECTION.override(True),
|
||||
):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
|
||||
@@ -73,9 +73,10 @@ class TestEagleDPAttnServerLarge(CustomTestCase):
|
||||
"--model-loader-extra-config",
|
||||
'{"enable_multithread_load": true,"num_threads": 64}',
|
||||
]
|
||||
with envs.SGLANG_SPEC_NAN_DETECTION.override(
|
||||
True
|
||||
), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
|
||||
with (
|
||||
envs.SGLANG_SPEC_NAN_DETECTION.override(True),
|
||||
envs.SGLANG_SPEC_OOB_DETECTION.override(True),
|
||||
):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
|
||||
@@ -51,9 +51,10 @@ class ServerWithGrammar(CustomTestCase):
|
||||
"--speculative-num-draft-tokens=8",
|
||||
]
|
||||
|
||||
with envs.SGLANG_SPEC_NAN_DETECTION.override(
|
||||
True
|
||||
), envs.SGLANG_SPEC_OOB_DETECTION.override(True):
|
||||
with (
|
||||
envs.SGLANG_SPEC_NAN_DETECTION.override(True),
|
||||
envs.SGLANG_SPEC_OOB_DETECTION.override(True),
|
||||
):
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
|
||||
Reference in New Issue
Block a user