[Test] Fold EAGLE return_hidden_states regression into spec triton suite (#28577)

This commit is contained in:
Liangsheng Yin
2026-06-17 16:42:02 -07:00
committed by GitHub
parent b88bada64e
commit a663500ea9
4 changed files with 62 additions and 72 deletions
@@ -574,3 +574,49 @@ class SpecFeatureKit:
self.assertIsInstance(content, dict)
except Exception:
self.fail(f"parse JSON failed: {content_json}")
class SpecHiddenStatesKit:
"""return_hidden_states under spec V2 (regression for issue #26163).
Requires the server launched with --enable-return-hidden-states
(set ``enable_return_hidden_states = True`` on the fixture class).
"""
def test_return_hidden_states(self):
# Two prompts of different lengths to exercise the per-req stride
# window: under spec V2 hidden_states is [bs * num_draft_tokens, dim],
# so a wrong index aliases a neighbor request's accepted rows.
prompts = [
"Repeat: the quick brown fox the quick brown fox the quick brown fox",
"Count down from ten: ten nine eight",
]
res = requests.post(
self.base_url + "/generate",
json={
"text": prompts,
"sampling_params": {"temperature": 0, "max_new_tokens": 32},
"return_hidden_states": True,
},
)
self.assertEqual(res.status_code, 200)
outputs = res.json()
for out in outputs:
meta = out["meta_info"]
hs = meta["hidden_states"]
ct = meta["completion_tokens"]
# One hidden-state entry per completion token: hs[0] is the prefill
# block (List[List[float]]), hs[1:] are per-decode-token rows.
self.assertEqual(
len(hs),
ct,
f"len(hidden_states)={len(hs)} but completion_tokens={ct}",
)
decode_rows = hs[1:]
self.assertGreater(len(decode_rows), 0)
hidden_dim = len(decode_rows[0])
self.assertGreater(hidden_dim, 0)
for row in decode_rows:
self.assertIsInstance(row, list)
self.assertEqual(len(row), hidden_dim)
@@ -66,6 +66,9 @@ class SpecEagleServerBase(CustomTestCase):
dtype = "bfloat16"
cuda_graph_max_bs = None
trust_remote_code = True
# Launch with --enable-return-hidden-states so SpecHiddenStatesKit can probe
# per-request hidden states; per-request gated, so other requests don't pay.
enable_return_hidden_states = False
# -- extras --
# env_overrides: (env_var_obj, value) pairs applied only around launch.
@@ -104,6 +107,8 @@ class SpecEagleServerBase(CustomTestCase):
args.append("--disable-overlap-schedule")
if cls.trust_remote_code:
args.append("--trust-remote-code")
if cls.enable_return_hidden_states:
args.append("--enable-return-hidden-states")
if cls.cuda_graph_max_bs is not None:
args += ["--cuda-graph-max-bs", str(cls.cuda_graph_max_bs)]
args += [str(a) for a in cls.extra_args]