[CI] Remove the GLM-5.3-Flash PTX KDA prefill nightly test (#43470)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
8b2aaf7060
commit
16be9df25f
@@ -1,270 +0,0 @@
|
||||
"""GLM-5.3-Flash accuracy parity for the PTX KDA prefill backend on Blackwell.
|
||||
|
||||
``--linear-attn-prefill-backend ptx_kda`` selects the hand-written tcgen05
|
||||
chunked-prefill kernel, JIT-built for the running device's arch (sm_100a on
|
||||
B200, sm_103a on GB300 / B300). GLM-5.3-Flash is the target: its 34 KDA layers
|
||||
use the safe gate (``gate_lower_bound=-5``), the only gate mode the kernel
|
||||
handles. Without a lower bound (Kimi-Linear) the per-chunk decay overflows and
|
||||
the kernel returns NaN.
|
||||
|
||||
Its failure modes are silent: the dispatcher swaps in Triton when it rejects
|
||||
the arch, and the wrapper hands individual batches to Triton (spec
|
||||
draft-extend, track-state snapshots, unsupported shapes, a kernel exception).
|
||||
None of that reaches the HTTP result, so this file
|
||||
|
||||
* reads the server log to prove the dispatcher picked ``PtxKDAKernel`` and
|
||||
that no batch fell back, and
|
||||
* scores the kernel against the same server on the default Triton prefill:
|
||||
|
||||
- GSM8K, 20-shot completion, greedy, 500 questions. The many-sequence
|
||||
batches take the kernel's builder + chain route, and every batch has a new
|
||||
shape, which is what exposes per-shape state in the kernel.
|
||||
- Prompt logprobs of one 2040-token prefill (the head of
|
||||
``python/sglang/test/long_prompt.txt``), which takes the single-sequence
|
||||
fused route through the wrapper's padding path (2040 is not a multiple of
|
||||
the 64-token chunk). It stays below the DSA indexer's top-k (2048): past
|
||||
that the sparse key selection is not run-to-run reproducible, and the
|
||||
comparison would measure that instead of the KDA kernel.
|
||||
|
||||
Server args follow the cookbook's Blackwell TP4 recipe with two changes, both
|
||||
made on the Triton and PTX servers alike:
|
||||
|
||||
* ``--disable-radix-cache``. With the radix cache on, the mamba cache
|
||||
strategy resolves to ``extra_buffer`` (``no_buffer`` needs page size 1, and
|
||||
DSA pins 64), whose tracked prefill batches carry a track-state buffer; the
|
||||
PTX wrapper routes exactly those batches to Triton, so the kernel under
|
||||
test would barely run. It also sends every GSM8K few-shot prefix through
|
||||
prefill instead of the cache.
|
||||
* ``--chunked-prefill-size 16384``, the B200 default, pinned so GB300 runs the
|
||||
same batch sizes.
|
||||
|
||||
python -m pytest \\
|
||||
test/registered/e2e/models/test_glm53_flash_kda_ptx_prefill_blackwell.py -v
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import requests
|
||||
import torch
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
import sglang.test
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
_wait_for_gpu_idle_in_ci,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
terminate_and_kill_process_tree,
|
||||
try_cached_model,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
# Measured on B200: 822 s for both servers, including the PTX extension build.
|
||||
register_cuda_ci(est_time=900, stage="nightly", runner_config="4-gpu-b200")
|
||||
register_cuda_ci(est_time=900, stage="nightly", runner_config="4-gpu-gb300")
|
||||
|
||||
MODEL = os.environ.get("SGLANG_TEST_GLM53_FLASH_MODEL") or try_cached_model(
|
||||
"zai-org/GLM-5.3-Flash"
|
||||
)
|
||||
# Offline hosts can point this at a local copy of the GSM8K test.jsonl.
|
||||
GSM8K_DATA_PATH = os.environ.get("SGLANG_TEST_GSM8K_DATA_PATH")
|
||||
SERVER_LAUNCH_TIMEOUT = 3600
|
||||
GSM8K_NUM_EXAMPLES = 500
|
||||
GSM8K_NUM_SHOTS = 20
|
||||
GSM8K_NUM_THREADS = 128
|
||||
# Collapse guard only; test_glm53_flash_b200.py owns the recipe's 0.93 floor.
|
||||
GSM8K_SCORE_FLOOR = 0.90
|
||||
# Measured gap 0.000 at 500 questions (0.930 / 0.930); 1 sigma is ~0.011.
|
||||
GSM8K_MAX_GAP = 0.03
|
||||
PROBE_TOKENS = 2040
|
||||
# Measured on B200 over the probe: mean |dlogprob| 0.080 and 0.2% of tokens
|
||||
# past 1 nat, identical across fresh and loaded servers because both backends
|
||||
# are bit-reproducible here. That is the kernels' numeric difference, amplified
|
||||
# by MoE routing; a wrong state or a NaN lands far above both bars.
|
||||
MAX_MEAN_ABS_DLOGPROB = 0.2
|
||||
MAX_FRAC_ABOVE_1NAT = 0.02
|
||||
|
||||
LONG_PROMPT_PATH = os.path.join(
|
||||
os.path.dirname(sglang.test.__file__), "long_prompt.txt"
|
||||
)
|
||||
|
||||
COMMON_ARGS = [
|
||||
"--tp-size",
|
||||
"4",
|
||||
"--dsa-prefill-backend",
|
||||
"trtllm",
|
||||
"--dsa-decode-backend",
|
||||
"trtllm",
|
||||
"--kv-cache-dtype",
|
||||
"fp8_e4m3",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_trtllm",
|
||||
"--chunked-prefill-size",
|
||||
"16384",
|
||||
"--disable-radix-cache",
|
||||
]
|
||||
PTX_ARGS = COMMON_ARGS + ["--linear-attn-prefill-backend", "ptx_kda"]
|
||||
|
||||
DISPATCHER_PTX = "extend=PtxKDAKernel"
|
||||
DISPATCHER_TRITON = "extend=TritonKDAKernel"
|
||||
PTX_LOADED = "Using PTX KDA chunked prefill"
|
||||
FALLBACK_MARKERS = (
|
||||
"falling back to Triton extend", # dispatcher rejected the arch
|
||||
"Falling back to Triton", # wrapper: unsupported shape / dtype
|
||||
"fell back to Triton", # wrapper: kernel raised, state restored
|
||||
)
|
||||
|
||||
|
||||
def _ptx_supported() -> bool:
|
||||
if not torch.cuda.is_available():
|
||||
return False
|
||||
from sglang.kernels.ops.attention.linear.kda_ptx_prefill import SM_ARCHS
|
||||
|
||||
return torch.cuda.get_device_capability() in SM_ARCHS
|
||||
|
||||
|
||||
class TestGLM53FlashKdaPtxPrefill(CustomTestCase):
|
||||
"""PTX KDA prefill vs Triton on the same weights: GSM8K and prompt logprobs."""
|
||||
|
||||
process = None
|
||||
log_file = None
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not _ptx_supported():
|
||||
raise unittest.SkipTest("PTX KDA prefill requires SM100 or SM103")
|
||||
cls.model = MODEL
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
with open(LONG_PROMPT_PATH) as f:
|
||||
text = f.read()
|
||||
tokenizer = AutoTokenizer.from_pretrained(cls.model, trust_remote_code=True)
|
||||
cls.probe_ids = tokenizer.encode(text)[:PROBE_TOKENS]
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls._stop_server()
|
||||
|
||||
@classmethod
|
||||
def _stop_server(cls):
|
||||
if cls.process is not None:
|
||||
terminate_and_kill_process_tree(cls.process)
|
||||
cls.process = None
|
||||
_wait_for_gpu_idle_in_ci()
|
||||
if cls.log_file is not None:
|
||||
cls.log_file.close()
|
||||
cls.log_file = None
|
||||
|
||||
@classmethod
|
||||
def _start_server(cls, other_args):
|
||||
cls._stop_server()
|
||||
cls.log_file = tempfile.NamedTemporaryFile(
|
||||
"w", prefix="kda_ptx_prefill_", suffix=".log", delete=False
|
||||
)
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=SERVER_LAUNCH_TIMEOUT,
|
||||
other_args=other_args,
|
||||
return_stdout_stderr=(cls.log_file, cls.log_file),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _server_log(cls) -> str:
|
||||
cls.log_file.flush()
|
||||
with open(cls.log_file.name) as f:
|
||||
return f.read()
|
||||
|
||||
def _gsm8k(self) -> float:
|
||||
requests.get(self.base_url + "/flush_cache")
|
||||
args = SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="gsm8k",
|
||||
api="completion",
|
||||
max_tokens=512,
|
||||
num_examples=GSM8K_NUM_EXAMPLES,
|
||||
num_shots=GSM8K_NUM_SHOTS,
|
||||
num_threads=GSM8K_NUM_THREADS,
|
||||
gsm8k_data_path=GSM8K_DATA_PATH,
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
print(f"gsm8k {metrics=}")
|
||||
return metrics["score"]
|
||||
|
||||
def _probe_logprobs(self) -> list:
|
||||
requests.get(self.base_url + "/flush_cache")
|
||||
response = requests.post(
|
||||
self.base_url + "/generate",
|
||||
json={
|
||||
"input_ids": self.probe_ids,
|
||||
"sampling_params": {"temperature": 0, "max_new_tokens": 1},
|
||||
"return_logprob": True,
|
||||
"logprob_start_len": 0,
|
||||
},
|
||||
timeout=300,
|
||||
)
|
||||
response.raise_for_status()
|
||||
# Entries are [logprob, token_id, text]. The first token has no logprob,
|
||||
# and the JSON encoder writes NaN as null, so count the nulls.
|
||||
entries = response.json()["meta_info"]["input_token_logprobs"][1:]
|
||||
logprobs = [lp for lp, *_ in entries]
|
||||
num_null = sum(lp is None for lp in logprobs)
|
||||
self.assertEqual(num_null, 0, f"{num_null} null (NaN) prompt logprobs")
|
||||
self.assertEqual(len(logprobs), PROBE_TOKENS - 1)
|
||||
return logprobs
|
||||
|
||||
def _run_backend(self, other_args):
|
||||
self._start_server(other_args)
|
||||
score = self._gsm8k()
|
||||
logprobs = self._probe_logprobs()
|
||||
log = self._server_log()
|
||||
self._stop_server()
|
||||
return score, logprobs, log
|
||||
|
||||
def test_ptx_prefill_matches_triton(self):
|
||||
triton_score, triton_lp, triton_log = self._run_backend(COMMON_ARGS)
|
||||
self.assertIn(DISPATCHER_TRITON, triton_log)
|
||||
|
||||
ptx_score, ptx_lp, ptx_log = self._run_backend(PTX_ARGS)
|
||||
|
||||
# The arch gate and the per-batch fallbacks only show in the log.
|
||||
self.assertIn(DISPATCHER_PTX, ptx_log, "dispatcher did not pick PtxKDAKernel")
|
||||
self.assertIn(PTX_LOADED, ptx_log, "PTX extension never ran a batch")
|
||||
for marker in FALLBACK_MARKERS:
|
||||
self.assertNotIn(marker, ptx_log, f"PTX prefill fell back: {marker}")
|
||||
|
||||
diff = [p - t for p, t in zip(ptx_lp, triton_lp)]
|
||||
mean_abs = sum(abs(d) for d in diff) / len(diff)
|
||||
frac_above_1 = sum(abs(d) > 1 for d in diff) / len(diff)
|
||||
max_abs = max(abs(d) for d in diff)
|
||||
# k3 (the KL estimator of kl_test_utils) is carried by a few tail tokens;
|
||||
# reported for reference, not asserted.
|
||||
k3 = sum(math.exp(d) - 1 - d for d in diff) / len(diff)
|
||||
|
||||
summary = (
|
||||
f"gsm8k triton={triton_score:.3f} ptx_kda={ptx_score:.3f} "
|
||||
f"gap={ptx_score - triton_score:+.3f} | {PROBE_TOKENS}-token prefill: "
|
||||
f"mean|dlogprob|={mean_abs:.4f} frac>1nat={frac_above_1:.4f} "
|
||||
f"max={max_abs:.3f} k3={k3:.2e}"
|
||||
)
|
||||
print(summary)
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_glm53_flash_kda_ptx_prefill\n{summary}\n"
|
||||
)
|
||||
|
||||
self.assertGreaterEqual(ptx_score, GSM8K_SCORE_FLOOR)
|
||||
self.assertGreaterEqual(ptx_score, triton_score - GSM8K_MAX_GAP)
|
||||
self.assertLess(mean_abs, MAX_MEAN_ABS_DLOGPROB)
|
||||
self.assertLess(frac_above_1, MAX_FRAC_ABOVE_1NAT)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user