[Test] Fold EAGLE return_hidden_states regression into spec triton suite (#28577)
This commit is contained in:
@@ -574,3 +574,49 @@ class SpecFeatureKit:
|
||||
self.assertIsInstance(content, dict)
|
||||
except Exception:
|
||||
self.fail(f"parse JSON failed: {content_json}")
|
||||
|
||||
|
||||
class SpecHiddenStatesKit:
|
||||
"""return_hidden_states under spec V2 (regression for issue #26163).
|
||||
|
||||
Requires the server launched with --enable-return-hidden-states
|
||||
(set ``enable_return_hidden_states = True`` on the fixture class).
|
||||
"""
|
||||
|
||||
def test_return_hidden_states(self):
|
||||
# Two prompts of different lengths to exercise the per-req stride
|
||||
# window: under spec V2 hidden_states is [bs * num_draft_tokens, dim],
|
||||
# so a wrong index aliases a neighbor request's accepted rows.
|
||||
prompts = [
|
||||
"Repeat: the quick brown fox the quick brown fox the quick brown fox",
|
||||
"Count down from ten: ten nine eight",
|
||||
]
|
||||
res = requests.post(
|
||||
self.base_url + "/generate",
|
||||
json={
|
||||
"text": prompts,
|
||||
"sampling_params": {"temperature": 0, "max_new_tokens": 32},
|
||||
"return_hidden_states": True,
|
||||
},
|
||||
)
|
||||
self.assertEqual(res.status_code, 200)
|
||||
outputs = res.json()
|
||||
|
||||
for out in outputs:
|
||||
meta = out["meta_info"]
|
||||
hs = meta["hidden_states"]
|
||||
ct = meta["completion_tokens"]
|
||||
# One hidden-state entry per completion token: hs[0] is the prefill
|
||||
# block (List[List[float]]), hs[1:] are per-decode-token rows.
|
||||
self.assertEqual(
|
||||
len(hs),
|
||||
ct,
|
||||
f"len(hidden_states)={len(hs)} but completion_tokens={ct}",
|
||||
)
|
||||
decode_rows = hs[1:]
|
||||
self.assertGreater(len(decode_rows), 0)
|
||||
hidden_dim = len(decode_rows[0])
|
||||
self.assertGreater(hidden_dim, 0)
|
||||
for row in decode_rows:
|
||||
self.assertIsInstance(row, list)
|
||||
self.assertEqual(len(row), hidden_dim)
|
||||
|
||||
@@ -66,6 +66,9 @@ class SpecEagleServerBase(CustomTestCase):
|
||||
dtype = "bfloat16"
|
||||
cuda_graph_max_bs = None
|
||||
trust_remote_code = True
|
||||
# Launch with --enable-return-hidden-states so SpecHiddenStatesKit can probe
|
||||
# per-request hidden states; per-request gated, so other requests don't pay.
|
||||
enable_return_hidden_states = False
|
||||
|
||||
# -- extras --
|
||||
# env_overrides: (env_var_obj, value) pairs applied only around launch.
|
||||
@@ -104,6 +107,8 @@ class SpecEagleServerBase(CustomTestCase):
|
||||
args.append("--disable-overlap-schedule")
|
||||
if cls.trust_remote_code:
|
||||
args.append("--trust-remote-code")
|
||||
if cls.enable_return_hidden_states:
|
||||
args.append("--enable-return-hidden-states")
|
||||
if cls.cuda_graph_max_bs is not None:
|
||||
args += ["--cuda-graph-max-bs", str(cls.cuda_graph_max_bs)]
|
||||
args += [str(a) for a in cls.extra_args]
|
||||
|
||||
Reference in New Issue
Block a user