[AMD] skip deterministic inference for MLA FP8 test (#23382)
Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com>
This commit is contained in:
co-authored by
YC Yen-Ching Tseng
parent
1340d8b3bd
commit
fd88a1c562
@@ -8,6 +8,7 @@ from sglang.test.test_utils import (
|
|||||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
DEFAULT_URL_FOR_TEST,
|
DEFAULT_URL_FOR_TEST,
|
||||||
CustomTestCase,
|
CustomTestCase,
|
||||||
|
is_in_amd_ci,
|
||||||
popen_launch_server,
|
popen_launch_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,23 +24,26 @@ class TestMLA(CustomTestCase, MGSMEnMixin):
|
|||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
cls.model = DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST
|
cls.model = DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
|
other_args = [
|
||||||
|
"--trust-remote-code",
|
||||||
|
"--kv-cache-dtype",
|
||||||
|
"fp8_e5m2",
|
||||||
|
# Pin MoE expert dispatch and kernel reduction order so MGSM
|
||||||
|
# scores don't drift across runs. The eval already uses greedy
|
||||||
|
# decoding, but FP8 dequant + non-deterministic MoE top-k
|
||||||
|
# tie-breaks produce ~1–3 point swings without this flag and
|
||||||
|
# straddle the 0.8 threshold. With deterministic inference,
|
||||||
|
# the score becomes a fixed function of (model, weights, CUDA
|
||||||
|
# stack), so threshold-edge flakes stop being random noise.
|
||||||
|
]
|
||||||
|
if not is_in_amd_ci():
|
||||||
|
# On AMD, the default attention backend (aiter) is not in the deterministic-inference allowlist, so the server fails to start, disable it.
|
||||||
|
other_args.append("--enable-deterministic-inference")
|
||||||
cls.process = popen_launch_server(
|
cls.process = popen_launch_server(
|
||||||
cls.model,
|
cls.model,
|
||||||
cls.base_url,
|
cls.base_url,
|
||||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||||
other_args=[
|
other_args=other_args,
|
||||||
"--trust-remote-code",
|
|
||||||
"--kv-cache-dtype",
|
|
||||||
"fp8_e5m2",
|
|
||||||
# Pin MoE expert dispatch and kernel reduction order so MGSM
|
|
||||||
# scores don't drift across runs. The eval already uses greedy
|
|
||||||
# decoding, but FP8 dequant + non-deterministic MoE top-k
|
|
||||||
# tie-breaks produce ~1–3 point swings without this flag and
|
|
||||||
# straddle the 0.8 threshold. With deterministic inference,
|
|
||||||
# the score becomes a fixed function of (model, weights, CUDA
|
|
||||||
# stack), so threshold-edge flakes stop being random noise.
|
|
||||||
"--enable-deterministic-inference",
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
Reference in New Issue
Block a user