[CI][RFC] Replace black-jupyter with ruff-format (#37210)
Co-authored-by: Alison Shao <a.shao@wustl.edu>
This commit is contained in:
co-authored by
Alison Shao
parent
2641e427be
commit
28262c20df
@@ -58,7 +58,7 @@ class TestDeepseekV3Basic(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v3)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v3)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["score"], 0.935)
|
||||
|
||||
@@ -70,7 +70,7 @@ class TestDeepseekV3Basic(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (deepseek-v3)\n" f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (deepseek-v3)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(speed, 12)
|
||||
|
||||
@@ -61,7 +61,7 @@ class TestDeepseekV32DP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v32)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["score"], 0.935)
|
||||
|
||||
@@ -73,7 +73,7 @@ class TestDeepseekV32DP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (deepseek-v32)\n" f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (deepseek-v32)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(speed, 50)
|
||||
|
||||
@@ -119,7 +119,7 @@ class TestDeepseekV32TP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v32)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["score"], 0.935)
|
||||
|
||||
@@ -131,7 +131,7 @@ class TestDeepseekV32TP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (deepseek-v32)\n" f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (deepseek-v32)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(speed, 80)
|
||||
|
||||
@@ -180,7 +180,7 @@ class TestGLM5DP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (glm-5)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (glm-5)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["score"], 0.935)
|
||||
|
||||
@@ -192,7 +192,7 @@ class TestGLM5DP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (glm-5)\n" f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (glm-5)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(speed, 40)
|
||||
|
||||
@@ -238,7 +238,7 @@ class TestGLM5TP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (glm-5)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (glm-5)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["score"], 0.935)
|
||||
|
||||
@@ -250,7 +250,7 @@ class TestGLM5TP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (glm-5)\n" f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (glm-5)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(speed, 60)
|
||||
|
||||
|
||||
@@ -138,7 +138,7 @@ def get_rdma_devices_args():
|
||||
if not (base_rdma_group <= gpu_idx < base_rdma_group + 4):
|
||||
warnings.warn(
|
||||
f"GPU index {gpu_idx} is outside expected group "
|
||||
f"{base_rdma_group}-{base_rdma_group+3}"
|
||||
f"{base_rdma_group}-{base_rdma_group + 3}"
|
||||
)
|
||||
|
||||
# 3. Generate RDMA device names
|
||||
|
||||
@@ -21,7 +21,6 @@ TEST_MODEL_MATRIX = {
|
||||
|
||||
|
||||
class TestAscendDeepSeekMTP(CustomTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.models = TEST_MODEL_MATRIX.keys()
|
||||
|
||||
@@ -274,12 +274,12 @@ def compare_outputs(trtllm_out, reference_out, tolerance=1e-2):
|
||||
"""Compare outputs with detailed analysis."""
|
||||
|
||||
# Basic checks
|
||||
assert (
|
||||
trtllm_out.shape == reference_out.shape
|
||||
), f"Shape mismatch: {trtllm_out.shape} vs {reference_out.shape}"
|
||||
assert (
|
||||
trtllm_out.dtype == reference_out.dtype
|
||||
), f"Dtype mismatch: {trtllm_out.dtype} vs {reference_out.dtype}"
|
||||
assert trtllm_out.shape == reference_out.shape, (
|
||||
f"Shape mismatch: {trtllm_out.shape} vs {reference_out.shape}"
|
||||
)
|
||||
assert trtllm_out.dtype == reference_out.dtype, (
|
||||
f"Dtype mismatch: {trtllm_out.dtype} vs {reference_out.dtype}"
|
||||
)
|
||||
|
||||
# Check for NaN/Inf
|
||||
assert not torch.isnan(trtllm_out).any(), "TRTLLM output contains NaN"
|
||||
@@ -310,7 +310,7 @@ def compare_outputs(trtllm_out, reference_out, tolerance=1e-2):
|
||||
trt_val = trtllm_out[idx_tuple].item()
|
||||
ref_val = reference_out[idx_tuple].item()
|
||||
print(
|
||||
f" [{idx_tuple}]: TRTLLM={trt_val:.6f}, Reference={ref_val:.6f}, diff={abs(trt_val-ref_val):.6f}"
|
||||
f" [{idx_tuple}]: TRTLLM={trt_val:.6f}, Reference={ref_val:.6f}, diff={abs(trt_val - ref_val):.6f}"
|
||||
)
|
||||
|
||||
return all_close
|
||||
|
||||
@@ -72,9 +72,9 @@ class _BeamSweepBase(CustomTestCase):
|
||||
cls.prompts = [r.prompt for r in rows if r.prompt_len < MAX_PROMPT_LEN][
|
||||
:NUM_PROMPTS
|
||||
]
|
||||
assert (
|
||||
len(cls.prompts) == NUM_PROMPTS
|
||||
), f"only {len(cls.prompts)} short prompts sampled"
|
||||
assert len(cls.prompts) == NUM_PROMPTS, (
|
||||
f"only {len(cls.prompts)} short prompts sampled"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
|
||||
@@ -53,15 +53,15 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
"finished",
|
||||
"unknown",
|
||||
), f"after abort r should be finished/unknown, got {r.status}"
|
||||
assert (
|
||||
r.kv_pages == 0
|
||||
), f"abort must release KV; r.kv_pages={r.kv_pages} after abort"
|
||||
assert (
|
||||
r.req is None or r.req.kv.req_pool_idx is None
|
||||
), f"abort must release row; r.req={r.req} after abort"
|
||||
assert (
|
||||
r.lock_refs == 0
|
||||
), f"abort must release lock_refs; r.lock_refs={r.lock_refs}"
|
||||
assert r.kv_pages == 0, (
|
||||
f"abort must release KV; r.kv_pages={r.kv_pages} after abort"
|
||||
)
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None, (
|
||||
f"abort must release row; r.req={r.req} after abort"
|
||||
)
|
||||
assert r.lock_refs == 0, (
|
||||
f"abort must release lock_refs; r.lock_refs={r.lock_refs}"
|
||||
)
|
||||
|
||||
def test_abort_at_chunk_0(self):
|
||||
self.server.execute_script(self._script_abort_at_chunk_0)
|
||||
@@ -436,9 +436,9 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
assert not r.is_chunking, "aborted gap req must stay out of chunking"
|
||||
|
||||
if r.req is not None:
|
||||
assert (
|
||||
r.req.inflight_middle_chunks == 0
|
||||
), f"inflight_middle_chunks not cleared; got {r.req.inflight_middle_chunks}"
|
||||
assert r.req.inflight_middle_chunks == 0, (
|
||||
f"inflight_middle_chunks not cleared; got {r.req.inflight_middle_chunks}"
|
||||
)
|
||||
|
||||
def test_abort_when_chunked_only_then_idle(self):
|
||||
self.server.execute_script(self._script_abort_when_chunked_only_then_idle)
|
||||
@@ -509,10 +509,10 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
yield from _drain_until_released(t, r1)
|
||||
|
||||
assert r1.kv_pages == 0, (
|
||||
f"force_retract + abort same yield must release KV; got " f"{r1.kv_pages}"
|
||||
f"force_retract + abort same yield must release KV; got {r1.kv_pages}"
|
||||
)
|
||||
assert r1.req is None or r1.req.kv.req_pool_idx is None, (
|
||||
f"force_retract + abort same yield must release row; got " f"{r1.req}"
|
||||
f"force_retract + abort same yield must release row; got {r1.req}"
|
||||
)
|
||||
assert r1.lock_refs == 0, (
|
||||
f"force_retract + abort same yield must release lock_refs; "
|
||||
|
||||
@@ -73,9 +73,9 @@ class TestChunkSizeDefault(ScriptedTestCase):
|
||||
r = t.start_req(prompt_len=1, max_new_tokens=2)
|
||||
yield from run_until_finished(r)
|
||||
assert r.finished
|
||||
assert (
|
||||
r.chunks_done == 0
|
||||
), f"single-token prompt should not chunk, got chunks_done={r.chunks_done}"
|
||||
assert r.chunks_done == 0, (
|
||||
f"single-token prompt should not chunk, got chunks_done={r.chunks_done}"
|
||||
)
|
||||
|
||||
def test_chunk_size_256_prompt_100x(self):
|
||||
self.server.execute_script(self._script_chunk_size_256_prompt_100x)
|
||||
|
||||
@@ -26,9 +26,9 @@ class TestScriptedHttpSmoke(ScriptedTestCase):
|
||||
break
|
||||
yield
|
||||
assert r.finished
|
||||
assert (
|
||||
saw_chunking
|
||||
), "expected the req to hold the chunked_req slot at least once"
|
||||
assert saw_chunking, (
|
||||
"expected the req to hold the chunked_req slot at least once"
|
||||
)
|
||||
|
||||
def test_two_reqs_finish(self):
|
||||
self.server.execute_script(self._script_two_reqs_finish)
|
||||
|
||||
@@ -123,9 +123,9 @@ class TestSWAHalfWindowChunk(ScriptedTestCase):
|
||||
r = t.start_req(prompt_len=2 * _SWA_WINDOW, max_new_tokens=4)
|
||||
yield from run_until_finished(r, max_steps=800)
|
||||
assert r.finished
|
||||
assert (
|
||||
r.chunks_done >= 4
|
||||
), f"expected >=4 chunks for 2*window / (window/2), got {r.chunks_done}"
|
||||
assert r.chunks_done >= 4, (
|
||||
f"expected >=4 chunks for 2*window / (window/2), got {r.chunks_done}"
|
||||
)
|
||||
assert len(r.req.output_ids) == 4
|
||||
|
||||
|
||||
@@ -171,9 +171,9 @@ class TestSWARadix(ScriptedTestCase):
|
||||
)
|
||||
yield from run_until_finished(r2, max_steps=800)
|
||||
assert r2.finished
|
||||
assert (
|
||||
r2.req.cached_tokens > 0
|
||||
), f"r2 must hit the radix prefix, got cached_tokens={r2.req.cached_tokens}"
|
||||
assert r2.req.cached_tokens > 0, (
|
||||
f"r2 must hit the radix prefix, got cached_tokens={r2.req.cached_tokens}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -38,9 +38,9 @@ class TestInvariantsBasic(ScriptedTestCase):
|
||||
for _ in range(DEFAULT_MAX_STEPS):
|
||||
if r.is_chunking:
|
||||
observed_chunking = True
|
||||
assert (
|
||||
r.kv_pages > 0
|
||||
), f"kv_pages must be > 0 while is_chunking; got {r.kv_pages}"
|
||||
assert r.kv_pages > 0, (
|
||||
f"kv_pages must be > 0 while is_chunking; got {r.kv_pages}"
|
||||
)
|
||||
if r.finished:
|
||||
break
|
||||
yield
|
||||
@@ -63,9 +63,9 @@ class TestInvariantsBasic(ScriptedTestCase):
|
||||
+ comp.get("decode", [])
|
||||
+ comp.get("chunked", [])
|
||||
)
|
||||
assert (
|
||||
r.rid in all_rids
|
||||
), f"running but not in batch_composition: {comp}"
|
||||
assert r.rid in all_rids, (
|
||||
f"running but not in batch_composition: {comp}"
|
||||
)
|
||||
if r.finished:
|
||||
return
|
||||
yield
|
||||
@@ -157,9 +157,9 @@ class TestInvariantsBasic(ScriptedTestCase):
|
||||
t.flush_cache()
|
||||
yield
|
||||
final = t.engine_stats()
|
||||
assert (
|
||||
final["kv_pool_free"] >= baseline["kv_pool_free"]
|
||||
), f"KV leak: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
|
||||
assert final["kv_pool_free"] >= baseline["kv_pool_free"], (
|
||||
f"KV leak: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
|
||||
)
|
||||
assert final["req_pool_free"] >= baseline["req_pool_free"]
|
||||
|
||||
def test_long_lived_engine_reps_chunked(self):
|
||||
@@ -212,9 +212,9 @@ class TestInvariantsBasic(ScriptedTestCase):
|
||||
t.flush_cache()
|
||||
yield
|
||||
final_kv = t.engine_stats()["kv_pool_free"]
|
||||
assert (
|
||||
final_kv >= baseline_kv
|
||||
), f"KV leak after sustained chunked load: {baseline_kv} -> {final_kv}"
|
||||
assert final_kv >= baseline_kv, (
|
||||
f"KV leak after sustained chunked load: {baseline_kv} -> {final_kv}"
|
||||
)
|
||||
|
||||
def test_round_robin_short_and_chunked(self):
|
||||
self.server.execute_script(self._script_round_robin_short_and_chunked)
|
||||
@@ -356,9 +356,9 @@ class TestInvariantsBasic(ScriptedTestCase):
|
||||
)
|
||||
yield from run_until_finished(r)
|
||||
assert r.finished
|
||||
assert (
|
||||
r.chunks_done >= 2
|
||||
), f"VERY_LONG_PROMPT_LEN should chunk; got chunks_done={r.chunks_done}"
|
||||
assert r.chunks_done >= 2, (
|
||||
f"VERY_LONG_PROMPT_LEN should chunk; got chunks_done={r.chunks_done}"
|
||||
)
|
||||
assert len(r.req.output_ids) == n, (
|
||||
f"ignore_eos=True + max_new_tokens={n} must produce exactly "
|
||||
f"{n} output tokens; got len(output_tokens)={len(r.req.output_ids)}"
|
||||
|
||||
@@ -62,9 +62,9 @@ class TestKVPressureBasic(ScriptedTestCase):
|
||||
f"long req must really chunk under pinned cache; got chunks_done="
|
||||
f"{r_long.chunks_done}"
|
||||
)
|
||||
assert (
|
||||
r_long.lock_refs == 0
|
||||
), f"req {r_long.rid} leaked {r_long.lock_refs} lock_refs after finish"
|
||||
assert r_long.lock_refs == 0, (
|
||||
f"req {r_long.rid} leaked {r_long.lock_refs} lock_refs after finish"
|
||||
)
|
||||
|
||||
t._release_exhausted_pools()
|
||||
final_lock_refs = t.get_all_node_lock_refs()
|
||||
|
||||
@@ -171,9 +171,9 @@ class TestLifecycleBasic(ScriptedTestCase):
|
||||
assert "running" in seen, f"never observed running status; seen={seen}"
|
||||
assert seen[-1] == "finished", f"final status must be finished; seen={seen}"
|
||||
finished_idx = seen.index("finished")
|
||||
assert all(
|
||||
s in ("finished",) for s in seen[finished_idx:]
|
||||
), f"status regressed after finish; seen={seen}"
|
||||
assert all(s in ("finished",) for s in seen[finished_idx:]), (
|
||||
f"status regressed after finish; seen={seen}"
|
||||
)
|
||||
|
||||
def test_long_prompt_only_one_decode(self):
|
||||
self.server.execute_script(self._script_long_prompt_only_one_decode)
|
||||
@@ -279,8 +279,7 @@ class TestLifecycleBasic(ScriptedTestCase):
|
||||
assert r1.finished and r2.finished
|
||||
assert r2.chunks_done == 0
|
||||
assert r2.req.cached_tokens > 0, (
|
||||
f"r2 must hit r1's radix prefix; got cached_tokens="
|
||||
f"{r2.req.cached_tokens}"
|
||||
f"r2 must hit r1's radix prefix; got cached_tokens={r2.req.cached_tokens}"
|
||||
)
|
||||
assert len(r2.req.output_ids) == 2
|
||||
|
||||
@@ -408,9 +407,9 @@ class TestLifecycleBasic(ScriptedTestCase):
|
||||
t.flush_cache()
|
||||
yield
|
||||
final = t.engine_stats()["kv_pool_free"]
|
||||
assert (
|
||||
final >= baseline - 1
|
||||
), f"KV pool drift: baseline={baseline}, final={final}"
|
||||
assert final >= baseline - 1, (
|
||||
f"KV pool drift: baseline={baseline}, final={final}"
|
||||
)
|
||||
|
||||
def test_abort_all_during_chunked(self):
|
||||
self.server.execute_script(self._script_abort_all_during_chunked)
|
||||
|
||||
@@ -143,14 +143,13 @@ class TestMaxNewTokensFirstDecodeAdjacent(ScriptedTestCase):
|
||||
|
||||
decode_records = _decode_records(batch_log, r.rid)
|
||||
assert len(decode_records) == max_new_tokens, (
|
||||
f"expected {max_new_tokens} decode forwards, got " f"{len(decode_records)}"
|
||||
f"expected {max_new_tokens} decode forwards, got {len(decode_records)}"
|
||||
)
|
||||
|
||||
rid_modes = [rec.mode for rec in rid_records]
|
||||
first_decode_pos = rid_modes.index("decode")
|
||||
assert first_decode_pos >= 1, (
|
||||
f"first decode must be preceded by an extend chunk; rid_modes="
|
||||
f"{rid_modes}"
|
||||
f"first decode must be preceded by an extend chunk; rid_modes={rid_modes}"
|
||||
)
|
||||
assert rid_modes[first_decode_pos - 1] == "extend", (
|
||||
f"record immediately before the first decode (in this rid's "
|
||||
|
||||
@@ -20,9 +20,9 @@ def _drain_flush_then_assert_no_kv_leak(t: ScriptedContext, baseline: dict):
|
||||
t.flush_cache()
|
||||
yield
|
||||
final = t.engine_stats()
|
||||
assert (
|
||||
final["kv_pool_free"] >= baseline["kv_pool_free"]
|
||||
), f"KV leak: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
|
||||
assert final["kv_pool_free"] >= baseline["kv_pool_free"], (
|
||||
f"KV leak: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
|
||||
)
|
||||
|
||||
|
||||
class TestMultiReqBasic(ScriptedTestCase):
|
||||
@@ -39,9 +39,9 @@ class TestMultiReqBasic(ScriptedTestCase):
|
||||
yield
|
||||
|
||||
assert r1.is_chunking, "r1 should still be chunking"
|
||||
assert (
|
||||
not r2.is_chunking
|
||||
), "r2 must wait for r1's chunk loop to clear before chunking"
|
||||
assert not r2.is_chunking, (
|
||||
"r2 must wait for r1's chunk loop to clear before chunking"
|
||||
)
|
||||
|
||||
yield from run_until_all_finished([r1, r2])
|
||||
assert r1.finished and r2.finished
|
||||
@@ -144,9 +144,9 @@ class TestMultiReqBasic(ScriptedTestCase):
|
||||
r2 = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN + 8, max_new_tokens=2)
|
||||
yield from run_until_finished(r2)
|
||||
assert r1.finished and r2.finished
|
||||
assert (
|
||||
r2.chunks_done < r1.chunks_done
|
||||
), "r2 reuses r1's cached prefix, so it should chunk fewer times"
|
||||
assert r2.chunks_done < r1.chunks_done, (
|
||||
"r2 reuses r1's cached prefix, so it should chunk fewer times"
|
||||
)
|
||||
|
||||
def test_trickle_per_yield_50(self):
|
||||
self.server.execute_script(self._script_trickle_per_yield_50)
|
||||
|
||||
@@ -175,9 +175,9 @@ class TestPPPdmux(ScriptedTestCase):
|
||||
def _script_pp_split_prefill_chunked_no_merge_assert(t: ScriptedContext):
|
||||
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
|
||||
yield from run_until_finished(r, max_steps=800)
|
||||
assert (
|
||||
r.finished
|
||||
), "engine died before req finished — merge_batch assert may have tripped"
|
||||
assert r.finished, (
|
||||
"engine died before req finished — merge_batch assert may have tripped"
|
||||
)
|
||||
assert r.chunks_done >= 2, (
|
||||
f"pdmux + chunked path must produce >=2 chunks to exercise "
|
||||
f"split_prefill_batch filter; got chunks_done={r.chunks_done}"
|
||||
|
||||
@@ -30,9 +30,9 @@ class TestPriorityBasic(ScriptedTestCase):
|
||||
t.pause_generation(mode="retract")
|
||||
yield
|
||||
|
||||
assert (
|
||||
r.status == "waiting"
|
||||
), f"force-retracted chunked req must be back in waiting; got {r.status}"
|
||||
assert r.status == "waiting", (
|
||||
f"force-retracted chunked req must be back in waiting; got {r.status}"
|
||||
)
|
||||
assert r.kv_pages == 0, f"retract must release KV; got {r.kv_pages}"
|
||||
|
||||
t.continue_generation()
|
||||
|
||||
@@ -76,9 +76,9 @@ class TestRadixBasic(ScriptedTestCase):
|
||||
f"after eviction r2 must re-chunk from scratch; "
|
||||
f"chunks_done={r2.chunks_done} cached_tokens={r2.req.cached_tokens}"
|
||||
)
|
||||
assert (
|
||||
r2.req.cached_tokens == 0
|
||||
), f"eviction must clear r1's prefix; cached_tokens={r2.req.cached_tokens}"
|
||||
assert r2.req.cached_tokens == 0, (
|
||||
f"eviction must clear r1's prefix; cached_tokens={r2.req.cached_tokens}"
|
||||
)
|
||||
assert r2.kv_pages == 0
|
||||
assert r2.lock_refs == 0
|
||||
|
||||
@@ -219,9 +219,9 @@ class TestRadixBasic(ScriptedTestCase):
|
||||
r = t.start_req(prompt_len=prompt_len, max_new_tokens=2)
|
||||
yield from run_until_finished(r, max_steps=400)
|
||||
assert r.finished
|
||||
assert (
|
||||
r.chunks_done == 0
|
||||
), f"full prefix hit must skip chunked path; got chunks_done={r.chunks_done}"
|
||||
assert r.chunks_done == 0, (
|
||||
f"full prefix hit must skip chunked path; got chunks_done={r.chunks_done}"
|
||||
)
|
||||
|
||||
def test_radix_evict_race_concurrent_chunked_admit(self):
|
||||
self.server.execute_script(
|
||||
@@ -319,9 +319,9 @@ class TestRadixNoTailChunked(ScriptedTestCase):
|
||||
"test must observe r as the in-flight chunked_req at least once; the "
|
||||
"no-tail else branch was never exercised"
|
||||
)
|
||||
assert (
|
||||
r.kv_pages == 0
|
||||
), f"finished chunked req must release KV; got {r.kv_pages}"
|
||||
assert r.kv_pages == 0, (
|
||||
f"finished chunked req must release KV; got {r.kv_pages}"
|
||||
)
|
||||
|
||||
|
||||
class TestRadixHitCountInvariant(ScriptedTestCase):
|
||||
|
||||
@@ -100,9 +100,9 @@ class TestRegressionBasic(ScriptedTestCase):
|
||||
f"observed max={observed_max} (pre-fix bug would bump to 2 "
|
||||
f"at the last-chunk admit boundary)"
|
||||
)
|
||||
assert (
|
||||
cleared_inflight
|
||||
), "inflight_middle_chunks should be 0 once the chunk loop clears"
|
||||
assert cleared_inflight, (
|
||||
"inflight_middle_chunks should be 0 once the chunk loop clears"
|
||||
)
|
||||
|
||||
yield from run_until_finished(r)
|
||||
assert r.finished
|
||||
@@ -247,15 +247,15 @@ class TestRegressionBasic(ScriptedTestCase):
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
|
||||
assert (
|
||||
r.req.kv.req_pool_idx is None
|
||||
), f"96d4749094: abort must release row; got row_idx={r.req.kv.req_pool_idx!r}"
|
||||
assert (
|
||||
r.kv_pages == 0
|
||||
), f"96d4749094: abort must release KV; got kv_pages={r.kv_pages}"
|
||||
assert (
|
||||
r.lock_refs == 0
|
||||
), f"96d4749094: abort must release lock_ref; got lock_refs={r.lock_refs}"
|
||||
assert r.req.kv.req_pool_idx is None, (
|
||||
f"96d4749094: abort must release row; got row_idx={r.req.kv.req_pool_idx!r}"
|
||||
)
|
||||
assert r.kv_pages == 0, (
|
||||
f"96d4749094: abort must release KV; got kv_pages={r.kv_pages}"
|
||||
)
|
||||
assert r.lock_refs == 0, (
|
||||
f"96d4749094: abort must release lock_ref; got lock_refs={r.lock_refs}"
|
||||
)
|
||||
assert not r.is_chunking
|
||||
assert r.req.inflight_middle_chunks == 0
|
||||
assert sum(t.get_all_node_lock_refs().values()) == baseline_refs
|
||||
|
||||
@@ -34,8 +34,7 @@ class TestSamplingBasic(ScriptedTestCase):
|
||||
if r.rid in rec.rids and rec.mode == "decode"
|
||||
]
|
||||
assert len(decode_records) == 0, (
|
||||
f"max_new_tokens=0 must run zero decode forwards; got "
|
||||
f"{len(decode_records)}"
|
||||
f"max_new_tokens=0 must run zero decode forwards; got {len(decode_records)}"
|
||||
)
|
||||
|
||||
def test_max_new_tokens_one_long_chunked(self):
|
||||
@@ -100,8 +99,7 @@ class TestSamplingBasic(ScriptedTestCase):
|
||||
assert r.chunks_done >= 2
|
||||
assert len(r.req.output_ids) == 16
|
||||
assert isinstance(r.req.finished_reason, FINISH_LENGTH), (
|
||||
f"ignore_eos=True must finish via length cap; got "
|
||||
f"{r.req.finished_reason!r}"
|
||||
f"ignore_eos=True must finish via length cap; got {r.req.finished_reason!r}"
|
||||
)
|
||||
|
||||
def test_return_logprob_top_logprobs_chunked(self):
|
||||
@@ -176,9 +174,9 @@ class TestSamplingBasic(ScriptedTestCase):
|
||||
)
|
||||
yield from run_until_finished(r)
|
||||
assert r.finished
|
||||
assert (
|
||||
r.chunks_done >= 2
|
||||
), f"prompt should span multiple chunks, got chunks_done={r.chunks_done}"
|
||||
assert r.chunks_done >= 2, (
|
||||
f"prompt should span multiple chunks, got chunks_done={r.chunks_done}"
|
||||
)
|
||||
assert r.req.logprob is not None
|
||||
input_lp = r.req.logprob.input_token_logprobs_val
|
||||
assert len(input_lp) == prompt_len, (
|
||||
@@ -201,9 +199,9 @@ class TestSamplingBasic(ScriptedTestCase):
|
||||
)
|
||||
yield from run_until_finished(r)
|
||||
assert r.finished
|
||||
assert (
|
||||
r.chunks_done >= 3
|
||||
), f"prompt should span 3+ chunks, got chunks_done={r.chunks_done}"
|
||||
assert r.chunks_done >= 3, (
|
||||
f"prompt should span 3+ chunks, got chunks_done={r.chunks_done}"
|
||||
)
|
||||
assert r.req.logprob is not None
|
||||
input_lp = r.req.logprob.input_token_logprobs_val
|
||||
assert len(input_lp) == prompt_len - start_len, (
|
||||
@@ -246,9 +244,9 @@ class TestSamplingBasic(ScriptedTestCase):
|
||||
)
|
||||
yield from run_until_finished(r_eos, max_steps=2000)
|
||||
assert r_eos.finished
|
||||
assert (
|
||||
r_eos.chunks_done >= 2
|
||||
), f"scenario 1 should chunk; got chunks_done={r_eos.chunks_done}"
|
||||
assert r_eos.chunks_done >= 2, (
|
||||
f"scenario 1 should chunk; got chunks_done={r_eos.chunks_done}"
|
||||
)
|
||||
assert isinstance(r_eos.req.finished_reason, FINISH_MATCHED_TOKEN), (
|
||||
f"a stop token the model deterministically produces under greedy must "
|
||||
f"finish via the matched-token path; got {r_eos.req.finished_reason!r}"
|
||||
@@ -261,9 +259,9 @@ class TestSamplingBasic(ScriptedTestCase):
|
||||
)
|
||||
yield from run_until_finished(r_length)
|
||||
assert r_length.finished
|
||||
assert (
|
||||
r_length.chunks_done >= 2
|
||||
), f"scenario 2 should chunk; got chunks_done={r_length.chunks_done}"
|
||||
assert r_length.chunks_done >= 2, (
|
||||
f"scenario 2 should chunk; got chunks_done={r_length.chunks_done}"
|
||||
)
|
||||
assert isinstance(r_length.req.finished_reason, FINISH_LENGTH), (
|
||||
f"ignore_eos=True + max_new_tokens=4 chunked must finish via "
|
||||
f"length cap; got {r_length.req.finished_reason!r}"
|
||||
|
||||
@@ -45,9 +45,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
for _ in range(DEFAULT_MAX_STEPS):
|
||||
if r.is_chunking:
|
||||
saw_chunking = True
|
||||
assert (
|
||||
not t.is_idle
|
||||
), "scheduler must not idle while chunked_req is in flight"
|
||||
assert not t.is_idle, (
|
||||
"scheduler must not idle while chunked_req is in flight"
|
||||
)
|
||||
if r.finished:
|
||||
break
|
||||
yield
|
||||
@@ -102,9 +102,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
break
|
||||
yield
|
||||
|
||||
assert (
|
||||
t.scheduler.chunked_req is None
|
||||
), f"abort must clear the chunked slot; got {t.scheduler.chunked_req!r}"
|
||||
assert t.scheduler.chunked_req is None, (
|
||||
f"abort must clear the chunked slot; got {t.scheduler.chunked_req!r}"
|
||||
)
|
||||
assert r.kv_pages == 0
|
||||
assert r.lock_refs == 0
|
||||
|
||||
@@ -193,19 +193,19 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
if r1.is_chunking:
|
||||
saw_r1_chunking = True
|
||||
comp = t.batch_composition()
|
||||
assert r1.rid in comp.get(
|
||||
"chunked", []
|
||||
), f"mid-chunk r1 must occupy the chunked role; got {comp!r}"
|
||||
assert r1.rid not in comp.get(
|
||||
"running", []
|
||||
), f"chunked r1 must be excluded from the running role; got {comp!r}"
|
||||
assert r1.rid in comp.get("chunked", []), (
|
||||
f"mid-chunk r1 must occupy the chunked role; got {comp!r}"
|
||||
)
|
||||
assert r1.rid not in comp.get("running", []), (
|
||||
f"chunked r1 must be excluded from the running role; got {comp!r}"
|
||||
)
|
||||
if r1.finished and r2.finished:
|
||||
break
|
||||
yield
|
||||
assert r1.finished and r2.finished
|
||||
assert (
|
||||
saw_r1_chunking
|
||||
), "r1 must have chunked at some point to exercise the exclude branch"
|
||||
assert saw_r1_chunking, (
|
||||
"r1 must have chunked at some point to exercise the exclude branch"
|
||||
)
|
||||
|
||||
@unittest.skip(
|
||||
"pdmux split_prefill_batch requires the pdmux topology — "
|
||||
@@ -257,9 +257,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
t.pause_generation(mode="retract")
|
||||
yield
|
||||
|
||||
assert (
|
||||
t.scheduler.chunked_req is None
|
||||
), f"pause(retract) must clear chunked_req; got {t.scheduler.chunked_req!r}"
|
||||
assert t.scheduler.chunked_req is None, (
|
||||
f"pause(retract) must clear chunked_req; got {t.scheduler.chunked_req!r}"
|
||||
)
|
||||
assert not r.finished, "retract must re-queue r, not finish or abort it"
|
||||
assert r.status == "waiting", (
|
||||
f"retracted chunked req must return to the waiting queue; "
|
||||
@@ -268,9 +268,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
|
||||
t.continue_generation()
|
||||
yield from run_until_finished(r)
|
||||
assert (
|
||||
r.finished
|
||||
), "continue_generation must drive the re-queued req to completion"
|
||||
assert r.finished, (
|
||||
"continue_generation must drive the re-queued req to completion"
|
||||
)
|
||||
|
||||
def test_retract_during_gap_inflight_middle_chunks_positive(self):
|
||||
self.server.execute_script(
|
||||
@@ -304,9 +304,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
|
||||
t.continue_generation()
|
||||
yield from run_until_finished(r, max_steps=2000)
|
||||
assert (
|
||||
r.finished
|
||||
), "continue_generation must drive the re-queued req to completion"
|
||||
assert r.finished, (
|
||||
"continue_generation must drive the re-queued req to completion"
|
||||
)
|
||||
assert r.kv_pages == 0
|
||||
assert len(r.req.output_ids) == 2
|
||||
|
||||
@@ -344,9 +344,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
break
|
||||
yield
|
||||
assert r.finished
|
||||
assert (
|
||||
saw_chunking
|
||||
), "test must observe the dual-queue chunked state at least once"
|
||||
assert saw_chunking, (
|
||||
"test must observe the dual-queue chunked state at least once"
|
||||
)
|
||||
assert saw_dedup, (
|
||||
"test must observe the chunked req with a committed prefix so the "
|
||||
"dedup subtraction is actually exercised"
|
||||
@@ -491,9 +491,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
break
|
||||
yield
|
||||
assert r.finished
|
||||
assert (
|
||||
saw_mid_chunk
|
||||
), "test must observe the fill_ids reset boundary at least once"
|
||||
assert saw_mid_chunk, (
|
||||
"test must observe the fill_ids reset boundary at least once"
|
||||
)
|
||||
assert r.finished
|
||||
|
||||
def test_chunked_req_slot_cleared_when_chunk_completes(self):
|
||||
@@ -516,9 +516,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
yield
|
||||
assert r.finished
|
||||
assert saw_chunking, "req should have occupied the chunked_req slot mid-chunk"
|
||||
assert (
|
||||
s.chunked_req is None
|
||||
), f"chunked_req slot must clear after last chunk; got {s.chunked_req!r}"
|
||||
assert s.chunked_req is None, (
|
||||
f"chunked_req slot must clear after last chunk; got {s.chunked_req!r}"
|
||||
)
|
||||
|
||||
def test_second_chunked_admit_blocked_when_chunked_req_set(self):
|
||||
self.server.execute_script(
|
||||
@@ -571,9 +571,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
|
||||
progressed = False
|
||||
for _ in range(DEFAULT_MAX_STEPS):
|
||||
if r.is_chunking:
|
||||
assert (
|
||||
not t.is_idle
|
||||
), "scheduler must not go idle while a chunked req is in flight"
|
||||
assert not t.is_idle, (
|
||||
"scheduler must not go idle while a chunked req is in flight"
|
||||
)
|
||||
cur_chunks_done = r.chunks_done
|
||||
if cur_chunks_done > prev_chunks_done:
|
||||
progressed = True
|
||||
@@ -681,9 +681,9 @@ class TestSpecialCaseMixedChunk(ScriptedTestCase):
|
||||
prompt_token=310,
|
||||
)
|
||||
yield from run_until(r, lambda h: h.is_chunking)
|
||||
assert (
|
||||
t.last_batch_forward_mode != "MIXED"
|
||||
), f"return_logprob must disable mixed-chunk path; got {t.last_batch_forward_mode!r}"
|
||||
assert t.last_batch_forward_mode != "MIXED", (
|
||||
f"return_logprob must disable mixed-chunk path; got {t.last_batch_forward_mode!r}"
|
||||
)
|
||||
yield from run_until_finished(r)
|
||||
|
||||
def test_mixed_chunk_with_running_batch(self):
|
||||
@@ -700,9 +700,9 @@ class TestSpecialCaseMixedChunk(ScriptedTestCase):
|
||||
yield
|
||||
yield from run_until(r_chunk, lambda h: h.is_chunking)
|
||||
|
||||
assert (
|
||||
t.last_batch_forward_mode == "MIXED"
|
||||
), f"chunked admission with running batch must enter MIXED; got {t.last_batch_forward_mode!r}"
|
||||
assert t.last_batch_forward_mode == "MIXED", (
|
||||
f"chunked admission with running batch must enter MIXED; got {t.last_batch_forward_mode!r}"
|
||||
)
|
||||
for _ in range(DEFAULT_MAX_STEPS * 2):
|
||||
if r_chunk.finished and r_dec.finished:
|
||||
break
|
||||
@@ -743,9 +743,9 @@ class TestSpecialCaseNoChunking(ScriptedTestCase):
|
||||
def _script_chunk_size_negative_disables_chunking(t: ScriptedContext):
|
||||
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
|
||||
for _ in range(DEFAULT_MAX_STEPS):
|
||||
assert (
|
||||
not r.is_chunking
|
||||
), "chunked_prefill_size=-1 should disable chunked path"
|
||||
assert not r.is_chunking, (
|
||||
"chunked_prefill_size=-1 should disable chunked path"
|
||||
)
|
||||
if r.finished:
|
||||
return
|
||||
yield
|
||||
@@ -825,9 +825,9 @@ class TestSpecialCaseHiCache(ScriptedTestCase):
|
||||
yield
|
||||
assert r.finished
|
||||
assert saw_chunking, "test must observe r mid-chunk at least once"
|
||||
assert (
|
||||
first_chunk_snap is not None
|
||||
), "test must snapshot cached_tokens at the first chunk boundary"
|
||||
assert first_chunk_snap is not None, (
|
||||
"test must snapshot cached_tokens at the first chunk boundary"
|
||||
)
|
||||
|
||||
def test_hicache_cached_tokens_set_once_invariant(self):
|
||||
self.server.execute_script(
|
||||
@@ -862,9 +862,9 @@ class TestSpecialCaseHiCache(ScriptedTestCase):
|
||||
break
|
||||
yield
|
||||
assert r.finished
|
||||
assert (
|
||||
saw_chunking
|
||||
), "test must observe the req mid-chunk (chunks_done >= 1) at least once"
|
||||
assert saw_chunking, (
|
||||
"test must observe the req mid-chunk (chunks_done >= 1) at least once"
|
||||
)
|
||||
assert snap is not None, "test must snapshot the cached_tokens_* breakdown"
|
||||
|
||||
|
||||
@@ -1011,24 +1011,23 @@ class TestSpecialCaseRetractMerge(ScriptedTestCase):
|
||||
t.pause_generation(mode="retract")
|
||||
yield
|
||||
|
||||
assert (
|
||||
s.last_batch is None
|
||||
), "retract must clear last_batch after merging the extend chunk batch"
|
||||
assert s.last_batch is None, (
|
||||
"retract must clear last_batch after merging the extend chunk batch"
|
||||
)
|
||||
assert len(s.running_batch.reqs) == 0, (
|
||||
"the merged extend chunk batch must be retracted out of running_batch, "
|
||||
f"not stranded; got {len(s.running_batch.reqs)} reqs"
|
||||
)
|
||||
assert (
|
||||
r.status == "waiting"
|
||||
), f"retracted chunked req must return to the waiting queue; got {r.status!r}"
|
||||
assert r.status == "waiting", (
|
||||
f"retracted chunked req must return to the waiting queue; got {r.status!r}"
|
||||
)
|
||||
assert r.kv_pages == 0
|
||||
|
||||
t.continue_generation()
|
||||
yield from run_until_finished(r)
|
||||
assert r.finished
|
||||
assert len(r.req.output_ids) == 2, (
|
||||
f"resumed req must emit exactly max_new_tokens; got "
|
||||
f"{len(r.req.output_ids)}"
|
||||
f"resumed req must emit exactly max_new_tokens; got {len(r.req.output_ids)}"
|
||||
)
|
||||
|
||||
|
||||
@@ -1183,9 +1182,9 @@ class TestSpecialCaseRetractedStain(ScriptedTestCase):
|
||||
assert r.finished
|
||||
|
||||
req = r.req
|
||||
assert (
|
||||
req.retracted_stain is True
|
||||
), "retract must set retracted_stain so the cached-token recount is suppressed"
|
||||
assert req.retracted_stain is True, (
|
||||
"retract must set retracted_stain so the cached-token recount is suppressed"
|
||||
)
|
||||
assert req.cached_tokens == cached_before, (
|
||||
f"retracted_stain must suppress re-adding pre_len-already_computed on "
|
||||
f"resume; cached_tokens grew from {cached_before} to {req.cached_tokens}"
|
||||
@@ -1237,17 +1236,17 @@ class TestSpecialCaseMiddleChunkNoToken(ScriptedTestCase):
|
||||
f"middle chunk must not append an output token; got "
|
||||
f"output_ids len {len(r.req.output_ids)}"
|
||||
)
|
||||
assert (
|
||||
r.status != "finished"
|
||||
), "middle chunk must not finish the req (skip_stream_req)"
|
||||
assert r.status != "finished", (
|
||||
"middle chunk must not finish the req (skip_stream_req)"
|
||||
)
|
||||
if r.finished:
|
||||
break
|
||||
yield
|
||||
assert r.finished
|
||||
assert saw_middle_chunk, "test must observe r mid-chunk at least once"
|
||||
assert (
|
||||
len(r.req.output_ids) >= 1
|
||||
), "output tokens must appear only after the chunked prefill completes"
|
||||
assert len(r.req.output_ids) >= 1, (
|
||||
"output tokens must appear only after the chunked prefill completes"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -264,7 +264,6 @@ def _ref_compress(
|
||||
|
||||
|
||||
class TestFusedCompressAttn(unittest.TestCase):
|
||||
|
||||
def _run_test(
|
||||
self,
|
||||
ratio: int,
|
||||
@@ -397,7 +396,6 @@ class TestFusedCompressAttn(unittest.TestCase):
|
||||
|
||||
|
||||
class TestStateOrdering(unittest.TestCase):
|
||||
|
||||
def test_write_then_compress(self):
|
||||
"""Verify write-first, compress-second matches reference."""
|
||||
device = torch.device("cuda")
|
||||
|
||||
@@ -149,7 +149,7 @@ def test_main(
|
||||
for with_topk in (False, True):
|
||||
if local_rank == 0:
|
||||
print(
|
||||
f'[testing] Running with {"FP8" if isinstance(current_x, tuple) else "BF16"}, {"with" if with_topk else "without"} top-k (async={async_mode}, previous={previous_mode}) ...',
|
||||
f"[testing] Running with {'FP8' if isinstance(current_x, tuple) else 'BF16'}, {'with' if with_topk else 'without'} top-k (async={async_mode}, previous={previous_mode}) ...",
|
||||
flush=True,
|
||||
end="",
|
||||
)
|
||||
@@ -192,9 +192,9 @@ def test_main(
|
||||
|
||||
# Checks
|
||||
recv_gbl_rank_prefix_sum = handle[-4]
|
||||
assert gbl_num_tokens_per_rank[rank].item() == recv_x.size(
|
||||
0
|
||||
), f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}"
|
||||
assert gbl_num_tokens_per_rank[rank].item() == recv_x.size(0), (
|
||||
f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}"
|
||||
)
|
||||
assert (
|
||||
gbl_num_tokens_per_expert.view(num_ranks, -1)[rank].tolist()
|
||||
== recv_num_tokens_per_expert_list
|
||||
@@ -316,10 +316,13 @@ def test_main(
|
||||
tune_args = {"x": current_x, "handle": handle, "config": config}
|
||||
t = bench(lambda: buffer.dispatch(**tune_args))[0]
|
||||
if t < best_time:
|
||||
best_time, best_results = t, (
|
||||
num_sms,
|
||||
nvl_chunk_size,
|
||||
rdma_chunk_size,
|
||||
best_time, best_results = (
|
||||
t,
|
||||
(
|
||||
num_sms,
|
||||
nvl_chunk_size,
|
||||
rdma_chunk_size,
|
||||
),
|
||||
)
|
||||
if local_rank == 0:
|
||||
print(
|
||||
@@ -328,7 +331,7 @@ def test_main(
|
||||
)
|
||||
if local_rank == 0:
|
||||
print(
|
||||
f'[tuning] Best dispatch ({"FP8" if isinstance(current_x, tuple) else "BF16"}): SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {rdma_send_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)',
|
||||
f"[tuning] Best dispatch ({'FP8' if isinstance(current_x, tuple) else 'BF16'}): SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {rdma_send_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)",
|
||||
flush=True,
|
||||
)
|
||||
print("", flush=True)
|
||||
@@ -385,10 +388,13 @@ def test_main(
|
||||
flush=True,
|
||||
)
|
||||
if t < best_time:
|
||||
best_time, best_results = t, (
|
||||
num_sms,
|
||||
nvl_chunk_size,
|
||||
rdma_chunk_size,
|
||||
best_time, best_results = (
|
||||
t,
|
||||
(
|
||||
num_sms,
|
||||
nvl_chunk_size,
|
||||
rdma_chunk_size,
|
||||
),
|
||||
)
|
||||
|
||||
if local_rank == 0:
|
||||
|
||||
@@ -120,7 +120,7 @@ def test_main(
|
||||
for with_topk in (False, True):
|
||||
if local_rank == 0:
|
||||
print(
|
||||
f'[testing] Running with {"FP8" if isinstance(current_x, tuple) else "BF16"}, {"with" if with_topk else "without"} top-k (async={async_mode}, previous={previous_mode}) ...',
|
||||
f"[testing] Running with {'FP8' if isinstance(current_x, tuple) else 'BF16'}, {'with' if with_topk else 'without'} top-k (async={async_mode}, previous={previous_mode}) ...",
|
||||
flush=True,
|
||||
end="",
|
||||
)
|
||||
@@ -162,9 +162,9 @@ def test_main(
|
||||
|
||||
# Checks
|
||||
rank_prefix_matrix = handle[0]
|
||||
assert gbl_num_tokens_per_rank[rank].item() == recv_x.size(
|
||||
0
|
||||
), f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}"
|
||||
assert gbl_num_tokens_per_rank[rank].item() == recv_x.size(0), (
|
||||
f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}"
|
||||
)
|
||||
assert (
|
||||
gbl_num_tokens_per_expert.view(num_ranks, -1)[rank].tolist()
|
||||
== recv_num_tokens_per_expert_list
|
||||
@@ -280,7 +280,7 @@ def test_main(
|
||||
)
|
||||
if local_rank == 0:
|
||||
print(
|
||||
f'[tuning] Best dispatch ({"FP8" if isinstance(current_x, tuple) else "BF16"}): SMs {best_results[0]}, NVL chunk {best_results[1]}, {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)',
|
||||
f"[tuning] Best dispatch ({'FP8' if isinstance(current_x, tuple) else 'BF16'}): SMs {best_results[0]}, NVL chunk {best_results[1]}, {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)",
|
||||
flush=True,
|
||||
)
|
||||
print("", flush=True)
|
||||
|
||||
@@ -36,9 +36,9 @@ def test_main(
|
||||
|
||||
# NOTES: the integers greater than 256 exceeds the BF16 precision limit
|
||||
rank_offset = 128
|
||||
assert (
|
||||
num_ranks - rank_offset < 257
|
||||
), "Too many ranks (exceeding test precision limit)"
|
||||
assert num_ranks - rank_offset < 257, (
|
||||
"Too many ranks (exceeding test precision limit)"
|
||||
)
|
||||
|
||||
x = torch.ones((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") * (
|
||||
rank - rank_offset
|
||||
@@ -55,9 +55,9 @@ def test_main(
|
||||
|
||||
# Randomly mask some positions
|
||||
for i in range(10):
|
||||
topk_idx[random.randint(0, num_tokens - 1), random.randint(0, num_topk - 1)] = (
|
||||
-1
|
||||
)
|
||||
topk_idx[
|
||||
random.randint(0, num_tokens - 1), random.randint(0, num_topk - 1)
|
||||
] = -1
|
||||
|
||||
# Check dispatch correctness
|
||||
do_check = True
|
||||
@@ -114,9 +114,9 @@ def test_main(
|
||||
assert (
|
||||
num_valid_tokens == (recv_layout_range & int_mask).sum().item()
|
||||
), f"{num_valid_tokens} != {recv_layout_range & int_mask}.sum().item()"
|
||||
assert (
|
||||
num_valid_tokens == (all_topk_idx == expert_id).sum().item()
|
||||
), f"{num_valid_tokens} != {(all_topk_idx == expert_id).sum().item()}"
|
||||
assert num_valid_tokens == (all_topk_idx == expert_id).sum().item(), (
|
||||
f"{num_valid_tokens} != {(all_topk_idx == expert_id).sum().item()}"
|
||||
)
|
||||
|
||||
# Check received data
|
||||
recv_x = recv_x[:num_valid_tokens]
|
||||
@@ -127,9 +127,10 @@ def test_main(
|
||||
recv_x[:, -128:] - recv_src_info.view(-1, 1) % num_tokens
|
||||
).sum().item() == 0
|
||||
for j in range(num_ranks):
|
||||
begin_idx, count = (recv_layout_range[j] >> 32).item(), (
|
||||
recv_layout_range[j] & int_mask
|
||||
).item()
|
||||
begin_idx, count = (
|
||||
(recv_layout_range[j] >> 32).item(),
|
||||
(recv_layout_range[j] & int_mask).item(),
|
||||
)
|
||||
assert (recv_x_amin == j - rank_offset).sum().item() == (
|
||||
all_topk_idx[j] == expert_id
|
||||
).sum().item()
|
||||
@@ -145,9 +146,9 @@ def test_main(
|
||||
# Check combine correctness
|
||||
for zero_copy in (False, True):
|
||||
if zero_copy:
|
||||
buffer.get_next_low_latency_combine_buffer(handle)[
|
||||
:, :, :
|
||||
] = simulated_gemm_x
|
||||
buffer.get_next_low_latency_combine_buffer(handle)[:, :, :] = (
|
||||
simulated_gemm_x
|
||||
)
|
||||
out = torch.empty(
|
||||
(num_tokens, hidden), dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
@@ -203,9 +204,9 @@ def test_main(
|
||||
)
|
||||
large_gemm_with_hook(hook) if return_recv_hook else None
|
||||
if zero_copy:
|
||||
buffer.get_next_low_latency_combine_buffer(handle)[
|
||||
:, :, :
|
||||
] = simulated_gemm_x
|
||||
buffer.get_next_low_latency_combine_buffer(handle)[:, :, :] = (
|
||||
simulated_gemm_x
|
||||
)
|
||||
combined_x, event, hook = buffer.low_latency_combine(
|
||||
simulated_gemm_x,
|
||||
topk_idx,
|
||||
|
||||
@@ -238,8 +238,7 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase):
|
||||
join_end = rank_offset + join_tp
|
||||
if join_end > len(visible_devices):
|
||||
raise RuntimeError(
|
||||
f"Scale-up requires {join_end} visible GPUs, got "
|
||||
f"{len(visible_devices)}"
|
||||
f"Scale-up requires {join_end} visible GPUs, got {len(visible_devices)}"
|
||||
)
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(visible_devices[rank_offset:join_end])
|
||||
base_joining_log = os.environ.get(
|
||||
|
||||
@@ -16,7 +16,6 @@ from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class TestFlashinferDispatcher(CustomTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
server_args = ServerArgs(model_path="dummy")
|
||||
@@ -125,8 +124,7 @@ class TestFlashinferDispatcher(CustomTestCase):
|
||||
self.assertTrue(
|
||||
torch.all(
|
||||
received_hidden_states[
|
||||
expected_source_rank
|
||||
* num_tokens : (expected_source_rank + 1)
|
||||
expected_source_rank * num_tokens : (expected_source_rank + 1)
|
||||
* num_tokens
|
||||
]
|
||||
== 100.0 + expected_source_rank
|
||||
@@ -229,8 +227,7 @@ class TestFlashinferDispatcher(CustomTestCase):
|
||||
self.assertTrue(
|
||||
torch.all(
|
||||
received_hidden_states[
|
||||
expected_source_rank
|
||||
* num_tokens : (expected_source_rank + 1)
|
||||
expected_source_rank * num_tokens : (expected_source_rank + 1)
|
||||
* num_tokens
|
||||
]
|
||||
== 100.0 + expected_source_rank
|
||||
|
||||
@@ -242,7 +242,7 @@ class TestDisaggregationDecodeWithHiCache(DisaggregationHiCacheBase):
|
||||
self.assertGreater(
|
||||
cached_tokens,
|
||||
previous_cached_tokens,
|
||||
f"Turn {turn} should have more cached tokens than turn {turn-1}",
|
||||
f"Turn {turn} should have more cached tokens than turn {turn - 1}",
|
||||
)
|
||||
|
||||
# Update context and cached tokens for next iteration
|
||||
|
||||
@@ -380,14 +380,14 @@ def test_plan_then_io_cuda_graph_replay() -> None:
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
# Anchor slot table matches the synchronous layer-0 result.
|
||||
assert torch.equal(
|
||||
out.cpu(), ref_slots[s][0].cpu()
|
||||
), f"slots differ at step {s}"
|
||||
assert torch.equal(out.cpu(), ref_slots[s][0].cpu()), (
|
||||
f"slots differ at step {s}"
|
||||
)
|
||||
# Every layer's device buffer stays bit-identical to synchronous swap-in.
|
||||
for layer in range(_PIO_LAYERS):
|
||||
assert torch.equal(
|
||||
buf[layer].cpu(), ref_snap[s][layer].cpu()
|
||||
), f"buffer differs at step {s}, layer {layer}"
|
||||
assert torch.equal(buf[layer].cpu(), ref_snap[s][layer].cpu()), (
|
||||
f"buffer differs at step {s}, layer {layer}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -53,7 +53,6 @@ MOCK_CHOICES_INPUT_DATA = {
|
||||
|
||||
|
||||
class TestChoices(CustomTestCase):
|
||||
|
||||
def test_token_length_normalized(self):
|
||||
"""Confirm 'antidisestablishmentarianism' is selected due to high confidences for
|
||||
its later tokens resulting in highest token length normalized prompt logprob."""
|
||||
|
||||
@@ -47,6 +47,7 @@ json_jump_forward = (
|
||||
+ r"""\}\n"""
|
||||
)
|
||||
|
||||
|
||||
# fmt: off
|
||||
@sgl.function
|
||||
def json_gen(s):
|
||||
|
||||
@@ -175,7 +175,7 @@ def get_k_and_s_triton():
|
||||
|
||||
end_time = time.perf_counter()
|
||||
print(
|
||||
f"_get_k_and_s_triton_kernel triton kernel infer time is {((end_time-start_time)*1000):.4f} ms\n"
|
||||
f"_get_k_and_s_triton_kernel triton kernel infer time is {((end_time - start_time) * 1000):.4f} ms\n"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -30,7 +30,6 @@ from sglang.test.test_utils import CustomTestCase, is_in_ci
|
||||
|
||||
|
||||
class TestLoRABackend(CustomTestCase):
|
||||
|
||||
def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]):
|
||||
for model_case in model_cases:
|
||||
# If skip_long_prompt is True, filter out prompts longer than 1000 characters
|
||||
|
||||
@@ -44,7 +44,6 @@ TEST_CUDA_GRAPH_PADDING_PROMPTS = [
|
||||
|
||||
|
||||
class TestLoRACudaGraph(CustomTestCase):
|
||||
|
||||
def _run_without_cuda_graph_on_model_cases(self, model_cases: List[LoRAModelCase]):
|
||||
# Since we have already enabled CUDA graph by default in other lora tests,
|
||||
# we only need to run lora tests without CUDA graph here.
|
||||
|
||||
@@ -9,7 +9,6 @@ from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class TestTorchNativeLoRABackend(CustomTestCase):
|
||||
|
||||
device = "cpu"
|
||||
|
||||
# set duplicate weights to test merging during prepare_lora_batch
|
||||
|
||||
@@ -74,7 +74,9 @@ class _FakeMHATokenToKVPool(_FakeKVCache):
|
||||
self.v_head_dim = (
|
||||
swa_v_head_dim
|
||||
if swa_v_head_dim is not None
|
||||
else v_head_dim if v_head_dim is not None else head_dim
|
||||
else v_head_dim
|
||||
if v_head_dim is not None
|
||||
else head_dim
|
||||
)
|
||||
self._create_buffers()
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ TORCH_DTYPES = [torch.float16]
|
||||
|
||||
|
||||
class TestClipModels(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
mp.set_start_method("spawn", force=True)
|
||||
@@ -61,12 +60,12 @@ class TestClipModels(unittest.TestCase):
|
||||
)
|
||||
print("text similarity diff", abs(text_similarity - 1))
|
||||
print("image similarity diff", abs(image_similarity - 1))
|
||||
assert torch.all(
|
||||
abs(text_similarity - 1) < prefill_tolerance
|
||||
), "embeddings are not all close"
|
||||
assert torch.all(
|
||||
abs(image_similarity - 1) < prefill_tolerance
|
||||
), "embeddings are not all close"
|
||||
assert torch.all(abs(text_similarity - 1) < prefill_tolerance), (
|
||||
"embeddings are not all close"
|
||||
)
|
||||
assert torch.all(abs(image_similarity - 1) < prefill_tolerance), (
|
||||
"embeddings are not all close"
|
||||
)
|
||||
|
||||
def test_accuracy(self):
|
||||
for model, prefill_tolerance in MODELS:
|
||||
|
||||
@@ -64,16 +64,16 @@ class TestQmeQwenModels(CustomTestCase):
|
||||
hf_text_embeddings.embed_logits[0], srt_text_embeddings.embed_logits[0]
|
||||
)
|
||||
print("texts similarity diff", abs(similarity - 1))
|
||||
assert torch.all(
|
||||
abs(similarity - 1) < prefill_tolerance
|
||||
), "embeddings are not all close"
|
||||
assert torch.all(abs(similarity - 1) < prefill_tolerance), (
|
||||
"embeddings are not all close"
|
||||
)
|
||||
similarity = get_similarities(
|
||||
hf_image_embeddings.embed_logits[0], srt_image_embeddings.embed_logits[0]
|
||||
)
|
||||
print("images similarity diff", abs(similarity - 1))
|
||||
assert torch.all(
|
||||
abs(similarity - 1) < prefill_tolerance
|
||||
), "embeddings are not all close"
|
||||
assert torch.all(abs(similarity - 1) < prefill_tolerance), (
|
||||
"embeddings are not all close"
|
||||
)
|
||||
|
||||
def test_accuracy(self):
|
||||
for model, prefill_tolerance in MODELS:
|
||||
|
||||
@@ -61,7 +61,7 @@ class TestKimiK2Thinking(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (Kimi-K2-Thinking)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (Kimi-K2-Thinking)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["score"], 0.95)
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@ class TestMistralLarge3Basic(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (mistral-large-3)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (mistral-large-3)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["score"], 0.90)
|
||||
|
||||
@@ -79,7 +79,7 @@ class TestMistralLarge3Basic(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (mistral-large-3)\n" f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (mistral-large-3)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(speed, 50)
|
||||
|
||||
|
||||
@@ -156,9 +156,9 @@ async def _stream_websocket_async(
|
||||
|
||||
async with websockets.connect(websocket_url) as websocket:
|
||||
created = json.loads(await websocket.recv())
|
||||
assert (
|
||||
created.get("type") == "session.created"
|
||||
), f"expected session.created, got {created!r}"
|
||||
assert created.get("type") == "session.created", (
|
||||
f"expected session.created, got {created!r}"
|
||||
)
|
||||
session_id = created["session"]["id"]
|
||||
|
||||
transcription_cfg = {"model": "qwen3-asr"}
|
||||
@@ -200,9 +200,9 @@ async def _stream_websocket_async(
|
||||
if t == "conversation.item.input_audio_transcription.delta":
|
||||
deltas.append(resp["delta"])
|
||||
elif t == "conversation.item.input_audio_transcription.completed":
|
||||
assert (
|
||||
"usage" in resp
|
||||
), f"transcription.completed missing required usage field: {resp!r}"
|
||||
assert "usage" in resp, (
|
||||
f"transcription.completed missing required usage field: {resp!r}"
|
||||
)
|
||||
assert resp["usage"].get("type") == "duration", resp["usage"]
|
||||
completed_msg.update(resp)
|
||||
return
|
||||
@@ -317,7 +317,7 @@ class TestQwen3ASRTranscription(CustomTestCase):
|
||||
self.assertEqual(
|
||||
results[0],
|
||||
results[i],
|
||||
f"Request {i+1} differs from first request",
|
||||
f"Request {i + 1} differs from first request",
|
||||
)
|
||||
print(f"[Consistency] All 3 requests match: {results[0][:80]}...")
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ class TestVLMPiecewiseCudaGraph(CustomTestCase):
|
||||
os.makedirs(output_path, exist_ok=True)
|
||||
|
||||
# -------- compose --model_args --------
|
||||
model_args = f'model_version="{model_version}",' f"tp={tp}"
|
||||
model_args = f'model_version="{model_version}",tp={tp}'
|
||||
|
||||
# -------- build command list --------
|
||||
cmd = [
|
||||
|
||||
@@ -69,7 +69,7 @@ class TestVLMViTCudaGraph(CustomTestCase):
|
||||
os.makedirs(output_path, exist_ok=True)
|
||||
|
||||
# -------- compose --model_args --------
|
||||
model_args = f'model_version="{model_version}",' f"tp={tp}"
|
||||
model_args = f'model_version="{model_version}",tp={tp}'
|
||||
|
||||
# -------- build command list --------
|
||||
cmd = [
|
||||
|
||||
@@ -66,7 +66,7 @@ class TestVLMViTFlashinferCudnn(CustomTestCase):
|
||||
os.makedirs(output_path, exist_ok=True)
|
||||
|
||||
# -------- compose --model_args --------
|
||||
model_args = f'model_version="{model_version}",' f"tp={tp}"
|
||||
model_args = f'model_version="{model_version}",tp={tp}'
|
||||
|
||||
# -------- build command list --------
|
||||
cmd = [
|
||||
|
||||
@@ -250,12 +250,12 @@ class TestCacheReport(CustomTestCase):
|
||||
)
|
||||
|
||||
# Verify cache hit for same salt
|
||||
assert (
|
||||
cached_tokens_1_second > cached_tokens_1_first
|
||||
), "Should have cache hit with same cache_salt"
|
||||
assert (
|
||||
cached_tokens_1_second == prompt_tokens_1 - 1
|
||||
), "Should cache all prompt tokens except the last one"
|
||||
assert cached_tokens_1_second > cached_tokens_1_first, (
|
||||
"Should have cache hit with same cache_salt"
|
||||
)
|
||||
assert cached_tokens_1_second == prompt_tokens_1 - 1, (
|
||||
"Should cache all prompt tokens except the last one"
|
||||
)
|
||||
|
||||
# Third request with different cache_salt "salt2" - should not get cache hit
|
||||
response3 = self.client.chat.completions.create(
|
||||
@@ -269,9 +269,9 @@ class TestCacheReport(CustomTestCase):
|
||||
print(f"First request with salt2 - cached_tokens: {cached_tokens_2_first}")
|
||||
|
||||
# Verify no cache hit for different salt (should be similar to first request with salt1)
|
||||
assert (
|
||||
cached_tokens_2_first <= cached_tokens_1_first + self.min_cached
|
||||
), "Different cache_salt should not share cache"
|
||||
assert cached_tokens_2_first <= cached_tokens_1_first + self.min_cached, (
|
||||
"Different cache_salt should not share cache"
|
||||
)
|
||||
|
||||
# Fourth request with same cache_salt "salt2" - should now get cache hit
|
||||
response4 = self.client.chat.completions.create(
|
||||
@@ -285,9 +285,9 @@ class TestCacheReport(CustomTestCase):
|
||||
print(f"Second request with salt2 - cached_tokens: {cached_tokens_2_second}")
|
||||
|
||||
# Verify cache hit for salt2
|
||||
assert (
|
||||
cached_tokens_2_second > cached_tokens_2_first
|
||||
), "Should have cache hit with same cache_salt for salt2"
|
||||
assert cached_tokens_2_second > cached_tokens_2_first, (
|
||||
"Should have cache hit with same cache_salt for salt2"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -23,7 +23,6 @@ from sglang.test.test_utils import (
|
||||
|
||||
|
||||
class TestToolChoiceLlama32(CustomTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# Mark flaky tests for this model
|
||||
@@ -379,15 +378,15 @@ class TestToolChoiceLlama32(CustomTestCase):
|
||||
|
||||
# Update function name if present (first chunk)
|
||||
if tool_call_delta.function and tool_call_delta.function.name:
|
||||
tool_calls_by_index[tool_index]["function"][
|
||||
"name"
|
||||
] = tool_call_delta.function.name
|
||||
tool_calls_by_index[tool_index]["function"]["name"] = (
|
||||
tool_call_delta.function.name
|
||||
)
|
||||
|
||||
# Accumulate arguments (all chunks)
|
||||
if tool_call_delta.function and tool_call_delta.function.arguments:
|
||||
tool_calls_by_index[tool_index]["function"][
|
||||
"arguments"
|
||||
] += tool_call_delta.function.arguments
|
||||
tool_calls_by_index[tool_index]["function"]["arguments"] += (
|
||||
tool_call_delta.function.arguments
|
||||
)
|
||||
|
||||
self.assertGreater(len(tool_calls_by_index), 0)
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ from sglang.test.test_utils import (
|
||||
|
||||
|
||||
class TestBenchOneBatch1GPU(CustomTestCase):
|
||||
|
||||
def test_bs1_small(self):
|
||||
_, output_throughput, _ = run_bench_one_batch(
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs-decode", "2"]
|
||||
|
||||
@@ -20,7 +20,6 @@ TORCH_DTYPES = [torch.float32]
|
||||
|
||||
|
||||
class TestCrossEncoderModels(CustomTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
mp.set_start_method("spawn", force=True)
|
||||
@@ -55,9 +54,9 @@ class TestCrossEncoderModels(CustomTestCase):
|
||||
for i in range(len(srt_scores)):
|
||||
score_difference = abs(hf_scores[i] - srt_scores[i])
|
||||
|
||||
assert (
|
||||
score_difference < score_tolerance
|
||||
), "cross encoder scores are not all close"
|
||||
assert score_difference < score_tolerance, (
|
||||
"cross encoder scores are not all close"
|
||||
)
|
||||
|
||||
def preprocess_prompts(self, prompt):
|
||||
processed_prompts = []
|
||||
|
||||
@@ -37,7 +37,6 @@ sgl_to_st_ratio = []
|
||||
|
||||
|
||||
class TestEncoderEmbeddingModels(CustomTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
mp.set_start_method("spawn", force=True)
|
||||
@@ -115,9 +114,9 @@ class TestEncoderEmbeddingModels(CustomTestCase):
|
||||
# print("similarity diff", abs(similarity - 1))
|
||||
|
||||
if len(truncated_prompts[i]) <= 1000:
|
||||
assert torch.all(
|
||||
abs(similarity - 1) < prefill_tolerance
|
||||
), "embeddings are not all close"
|
||||
assert torch.all(abs(similarity - 1) < prefill_tolerance), (
|
||||
"embeddings are not all close"
|
||||
)
|
||||
|
||||
def test_prefill_logits(self):
|
||||
models_to_test = MODELS
|
||||
|
||||
@@ -46,9 +46,9 @@ def native_per_token_group_quant_fp8(
|
||||
quantized tensor along with the scaling factor used for quantization.
|
||||
Note that only `torch.float8_e4m3fn` is supported for now.
|
||||
"""
|
||||
assert (
|
||||
x.shape[-1] % group_size == 0
|
||||
), "the last dimension of `x` cannot be divisible by `group_size`"
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` cannot be divisible by `group_size`"
|
||||
)
|
||||
assert x.is_contiguous(), "`x` is not contiguous"
|
||||
|
||||
finfo = torch.finfo(dtype)
|
||||
@@ -343,7 +343,6 @@ def native_w8a8_block_fp8_matmul(A, B, As, Bs, block_size, output_dtype=torch.fl
|
||||
|
||||
|
||||
class TestW8A8BlockFP8Matmul(CustomTestCase):
|
||||
|
||||
if not _is_cuda:
|
||||
OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16]
|
||||
M = [1, 7, 83, 512, 2048]
|
||||
|
||||
@@ -171,7 +171,6 @@ def block_quant_dequant(
|
||||
|
||||
|
||||
class TestDeepGemmBlackwell(CustomTestCase):
|
||||
|
||||
if not _is_cuda:
|
||||
OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16]
|
||||
M = [1, 7, 83, 512, 2048]
|
||||
|
||||
@@ -67,7 +67,7 @@ class TestDeepseekV32FP4DP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v3-fp4)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
|
||||
self.assertGreater(metrics["score"], 0.93)
|
||||
@@ -135,7 +135,7 @@ class TestDeepseekV32FP4TP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v3-fp4)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
|
||||
self.assertGreater(metrics["score"], 0.93)
|
||||
|
||||
@@ -70,7 +70,7 @@ class TestDeepseekV3FP4(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v3-fp4)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
|
||||
self.assertGreater(metrics["score"], 0.93)
|
||||
@@ -83,7 +83,7 @@ class TestDeepseekV3FP4(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (deepseek-v3-fp4)\n" f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (deepseek-v3-fp4)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
|
||||
self.assertGreater(speed, 120)
|
||||
|
||||
@@ -46,7 +46,6 @@ class TestEvalFP8Accuracy(CustomTestCase):
|
||||
|
||||
|
||||
class TestEvalFP8DynamicQuantAccuracy(CustomTestCase):
|
||||
|
||||
def _run_test(self, model, other_args, expected_score):
|
||||
base_url = DEFAULT_URL_FOR_TEST
|
||||
other_args = other_args or []
|
||||
|
||||
@@ -9,7 +9,6 @@ from sglang.test.test_utils import (
|
||||
|
||||
|
||||
class TestNoChunkedPrefill(CustomTestCase):
|
||||
|
||||
def test_no_chunked_prefill(self):
|
||||
run_mmlu_test(
|
||||
disable_radix_cache=False, enable_mixed_chunk=False, chunked_prefill_size=-1
|
||||
|
||||
@@ -59,5 +59,4 @@ class TestEagle3Basic(EagleServerBase):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
unittest.main()
|
||||
|
||||
@@ -386,8 +386,8 @@ class TestPerformance(unittest.TestCase):
|
||||
speedup = t_ref / t_new
|
||||
dev_tag = "CUDA" if "cuda" in str(self.device) else "CPU"
|
||||
print(
|
||||
f" [{dev_tag}] B={batch_size:3d} S={seq_len:5d} img%={int(image_fraction*100):3d}%"
|
||||
f" ref={t_ref*1e3:.2f}ms new={t_new*1e3:.2f}ms speedup={speedup:.2f}x"
|
||||
f" [{dev_tag}] B={batch_size:3d} S={seq_len:5d} img%={int(image_fraction * 100):3d}%"
|
||||
f" ref={t_ref * 1e3:.2f}ms new={t_new * 1e3:.2f}ms speedup={speedup:.2f}x"
|
||||
)
|
||||
self.assertGreaterEqual(
|
||||
speedup,
|
||||
|
||||
@@ -57,9 +57,9 @@ class TestFimCompletion(CustomTestCase):
|
||||
assert response.id
|
||||
assert response.created
|
||||
assert response.object == "text_completion"
|
||||
assert (
|
||||
response.usage.prompt_tokens == num_prompt_tokens
|
||||
), f"{response.usage.prompt_tokens} vs {num_prompt_tokens}"
|
||||
assert response.usage.prompt_tokens == num_prompt_tokens, (
|
||||
f"{response.usage.prompt_tokens} vs {num_prompt_tokens}"
|
||||
)
|
||||
assert response.usage.completion_tokens > 0
|
||||
assert response.usage.total_tokens > 0
|
||||
|
||||
|
||||
@@ -30,7 +30,6 @@ def _process_return(ret):
|
||||
|
||||
|
||||
class TestGetWeightsByName(CustomTestCase):
|
||||
|
||||
def init_hf_model(self, model_name, tie_word_embeddings):
|
||||
self.hf_model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name, torch_dtype="bfloat16", tie_word_embeddings=tie_word_embeddings
|
||||
|
||||
@@ -104,15 +104,15 @@ def test_kda_target_verify_equivalence():
|
||||
state_diff = (cached_state - decode_state).abs().max().item()
|
||||
status = "OK" if state_diff < 1e-5 else "FAIL"
|
||||
print(f" step={step} req={req_idx}: diff={state_diff:.6e} [{status}]")
|
||||
assert (
|
||||
state_diff < 1e-5
|
||||
), f"Intermediate state mismatch at step={step}, req={req_idx}: {state_diff}"
|
||||
assert state_diff < 1e-5, (
|
||||
f"Intermediate state mismatch at step={step}, req={req_idx}: {state_diff}"
|
||||
)
|
||||
|
||||
ssm_unchanged_diff = (ssm_states_verify - ssm_states_base).abs().max().item()
|
||||
print(f"SSM state in-place change (should be 0): {ssm_unchanged_diff:.6e}")
|
||||
assert (
|
||||
ssm_unchanged_diff == 0.0
|
||||
), f"target_verify modified ssm_states in-place! diff: {ssm_unchanged_diff}"
|
||||
assert ssm_unchanged_diff == 0.0, (
|
||||
f"target_verify modified ssm_states in-place! diff: {ssm_unchanged_diff}"
|
||||
)
|
||||
|
||||
print("\nPASSED: KDA target_verify matches sequential decode!")
|
||||
|
||||
|
||||
@@ -218,7 +218,6 @@ def generate_baseline(
|
||||
|
||||
|
||||
class TestLogprobsDense(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Set up the test class - initialize the engine once for all tests."""
|
||||
|
||||
@@ -16,7 +16,6 @@ from sglang.test.test_utils import (
|
||||
|
||||
|
||||
class TestEvalFP8ModelOptQuantAccuracy(CustomTestCase):
|
||||
|
||||
def _run_test(self, model, other_args, expected_score):
|
||||
base_url = DEFAULT_URL_FOR_TEST
|
||||
other_args = other_args or []
|
||||
|
||||
@@ -9,7 +9,6 @@ from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class TestDownloadFromModelScope(CustomTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = "iic/nlp_lstmcrf_word-segmentation_chinese-news"
|
||||
|
||||
@@ -155,7 +155,6 @@ def _cleanup(actor, pg):
|
||||
@unittest.skipUnless(_has_ray, "ray is not installed")
|
||||
@unittest.skipUnless(_NUM_GPUS >= 1, "requires at least 1 GPU")
|
||||
class TestRayEngineOfflineTP1(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not ray.is_initialized():
|
||||
@@ -196,7 +195,6 @@ class TestRayEngineOfflineTP1(unittest.TestCase):
|
||||
@unittest.skipUnless(_has_ray, "ray is not installed")
|
||||
@unittest.skipUnless(_NUM_GPUS >= 2, "requires at least 2 GPUs")
|
||||
class TestRayEngineOfflineTP2(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not ray.is_initialized():
|
||||
@@ -231,7 +229,6 @@ class TestRayEngineOfflineTP2(unittest.TestCase):
|
||||
@unittest.skipUnless(_has_ray, "ray is not installed")
|
||||
@unittest.skipUnless(_NUM_GPUS >= 2, "requires at least 2 GPUs")
|
||||
class TestRayEngineOfflinePP2(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not ray.is_initialized():
|
||||
@@ -347,7 +344,6 @@ class TestRayEngineOfflineDPAttention(unittest.TestCase):
|
||||
@unittest.skipUnless(_has_ray, "ray is not installed")
|
||||
@unittest.skipUnless(_NUM_GPUS >= 1, "requires at least 1 GPU")
|
||||
class TestRayEngineErrors(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not ray.is_initialized():
|
||||
|
||||
@@ -69,9 +69,9 @@ class TestSageMakerServer(CustomTestCase):
|
||||
ret_num_top_logprobs = len(
|
||||
response["choices"][0]["logprobs"]["content"][0]["top_logprobs"]
|
||||
)
|
||||
assert (
|
||||
ret_num_top_logprobs == logprobs
|
||||
), f"{ret_num_top_logprobs} vs {logprobs}"
|
||||
assert ret_num_top_logprobs == logprobs, (
|
||||
f"{ret_num_top_logprobs} vs {logprobs}"
|
||||
)
|
||||
|
||||
assert len(response["choices"]) == parallel_sample_num
|
||||
assert response["choices"][0]["message"]["role"] == "assistant"
|
||||
@@ -155,18 +155,18 @@ class TestSageMakerServer(CustomTestCase):
|
||||
.get("content")[0]
|
||||
.get("top_logprobs")
|
||||
)
|
||||
assert (
|
||||
ret_num_top_logprobs == logprobs
|
||||
), f"{ret_num_top_logprobs} vs {logprobs}"
|
||||
assert ret_num_top_logprobs == logprobs, (
|
||||
f"{ret_num_top_logprobs} vs {logprobs}"
|
||||
)
|
||||
|
||||
assert isinstance(data["content"], str)
|
||||
assert line["id"]
|
||||
assert line["created"]
|
||||
|
||||
for index in [i for i in range(parallel_sample_num)]:
|
||||
assert not is_firsts.get(
|
||||
index, True
|
||||
), f"index {index} is not found in the response"
|
||||
assert not is_firsts.get(index, True), (
|
||||
f"index {index} is not found in the response"
|
||||
)
|
||||
|
||||
def test_chat_completion(self):
|
||||
for logprobs in [None, 5]:
|
||||
|
||||
@@ -25,7 +25,6 @@ def _make_req(rid, origin_input_text, origin_input_ids, sampling_params=None, **
|
||||
|
||||
|
||||
class TestSchedulePolicy(CustomTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tree_cache = RadixCache.create_simulated()
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTest
|
||||
|
||||
|
||||
class TestSRTEngineWithQuantArgs(CustomTestCase):
|
||||
|
||||
def test_1_quantization_args(self):
|
||||
|
||||
# we only test fp8 because other methods are currently dependent on vllm. We can add other methods back to test after vllm dependency is resolved.
|
||||
|
||||
@@ -38,7 +38,6 @@ class TestTokenizerBatchEncode(unittest.TestCase):
|
||||
"sglang.srt.utils.hf_transformers_utils.get_tokenizer"
|
||||
) as mock_tokenizer,
|
||||
):
|
||||
|
||||
mock_tokenizer.return_value = Mock(vocab_size=32000)
|
||||
publish(self.server_args, role="tokenizer")
|
||||
self.tokenizer_manager = TokenizerManager(self.server_args, self.port_args)
|
||||
|
||||
@@ -14,7 +14,6 @@ from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class TestTritonAttentionMLA(CustomTestCase):
|
||||
|
||||
def _set_all_seeds(self, seed):
|
||||
"""Set all random seeds for reproducibility."""
|
||||
random.seed(seed)
|
||||
|
||||
@@ -212,8 +212,7 @@ class TestMiniCPMV2_6Logits(VisionLLMLogitsBase):
|
||||
# per image
|
||||
if len(pixel_b) != len(tgt_b):
|
||||
raise ValueError(
|
||||
"Inconsistent N lengths, found: "
|
||||
f"{len(pixel_b)} vs {len(tgt_b)}"
|
||||
f"Inconsistent N lengths, found: {len(pixel_b)} vs {len(tgt_b)}"
|
||||
)
|
||||
for pixel_n, tgt_n in zip(pixel_b, tgt_b):
|
||||
pixel_values_flat += [pixel_n]
|
||||
|
||||
@@ -266,7 +266,7 @@ def main():
|
||||
|
||||
print(
|
||||
f"Daemon pp_rank={pp_rank} tp_rank={tp_rank} ready "
|
||||
f"({time.time()-start:.0f}s)"
|
||||
f"({time.time() - start:.0f}s)"
|
||||
)
|
||||
if error_found:
|
||||
break
|
||||
@@ -278,7 +278,7 @@ def main():
|
||||
sys.exit(1)
|
||||
|
||||
print(
|
||||
f"\nAll {total_ranks} daemons ready! Total load time: {time.time()-start:.1f}s"
|
||||
f"\nAll {total_ranks} daemons ready! Total load time: {time.time() - start:.1f}s"
|
||||
)
|
||||
|
||||
# Query config from daemon (pp_rank=0, tp_rank=0)
|
||||
|
||||
@@ -97,14 +97,14 @@ class TestWhisperCudaGraph(CustomTestCase):
|
||||
result = self._transcribe()
|
||||
self.assertIn("text", result)
|
||||
results.append(result["text"])
|
||||
print(f"Request {i+1}: {result['text'][:80]}...")
|
||||
print(f"Request {i + 1}: {result['text'][:80]}...")
|
||||
|
||||
# All transcriptions of the same audio should be identical
|
||||
for i in range(1, len(results)):
|
||||
self.assertEqual(
|
||||
results[0],
|
||||
results[i],
|
||||
f"Transcription {i+1} differs from first transcription",
|
||||
f"Transcription {i + 1} differs from first transcription",
|
||||
)
|
||||
|
||||
def test_transcription_quality(self):
|
||||
|
||||
@@ -76,9 +76,9 @@ def main():
|
||||
if rank == 0:
|
||||
print(
|
||||
f"world={world} owner={owner} shape=[{n_tok},{hidden}] "
|
||||
f"(~{n_tok*hidden*2/1e6:.0f}MB) | all_ranks_bitwise_ok={bool(flags.item())} "
|
||||
f"(~{n_tok * hidden * 2 / 1e6:.0f}MB) | all_ranks_bitwise_ok={bool(flags.item())} "
|
||||
f"(A==truth={eq_truth} A==B={eq_ab}) | all_gather {ta:.3f}ms "
|
||||
f"broadcast {tb:.3f}ms speedup {ta/tb:.2f}x",
|
||||
f"broadcast {tb:.3f}ms speedup {ta / tb:.2f}x",
|
||||
flush=True,
|
||||
)
|
||||
dist.destroy_process_group()
|
||||
|
||||
@@ -61,7 +61,7 @@ class TestDeepseekV32IndexTopkPattern(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v32)\n{metrics["accuracy"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], 0.935)
|
||||
|
||||
@@ -108,7 +108,7 @@ class TestDeepseekV32IndexFreq(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v32)\n{metrics["accuracy"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], 0.935)
|
||||
|
||||
|
||||
@@ -246,9 +246,9 @@ class TestDeepSeekR1EvalAMD(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -98,8 +98,7 @@ class TestDeepseekV32DP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v32 DP MI325)\n"
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v32 DP MI325)\n{metrics["accuracy"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
|
||||
|
||||
@@ -112,8 +111,7 @@ class TestDeepseekV32DP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (deepseek-v32 DP MI325)\n"
|
||||
f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (deepseek-v32 DP MI325)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(speed, 10)
|
||||
|
||||
|
||||
@@ -182,9 +182,9 @@ class TestDeepSeekV32EvalAMD(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -99,8 +99,7 @@ class TestDeepseekV32TC(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v32 TC MI325)\n"
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v32 TC MI325)\n{metrics["accuracy"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
|
||||
|
||||
@@ -113,8 +112,7 @@ class TestDeepseekV32TC(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (deepseek-v32 TC MI325)\n"
|
||||
f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (deepseek-v32 TC MI325)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(speed, 10)
|
||||
|
||||
|
||||
@@ -172,9 +172,9 @@ class TestGLM51EvalAMD(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -80,8 +80,7 @@ class TestGLM51HiSparseEvalAMD(unittest.TestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (glm-5.1 hisparse mi30x)\n"
|
||||
f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (glm-5.1 hisparse mi30x)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["score"], 0.93)
|
||||
|
||||
|
||||
@@ -182,9 +182,9 @@ class TestGLM5EvalAMD(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -184,9 +184,9 @@ class TestGptOssEvalAMD(unittest.TestCase):
|
||||
|
||||
for config in self.models:
|
||||
with self.subTest(model=config.model_path):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {config.model_path}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -222,9 +222,9 @@ class TestGrokEvalAMD(unittest.TestCase):
|
||||
|
||||
for config in self.models:
|
||||
with self.subTest(model=config.model_path):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {config.model_path}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -181,9 +181,9 @@ def check_model_scores(results):
|
||||
line = f"| {model} | {tp_size} | {score:.3f} | {threshold_str} | {startup_str} | {eval_str} | {total_str} | {status} |\n"
|
||||
summary += line
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print("SUMMARY - TP=2 Instruction Models (gsm8k)")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
print(summary)
|
||||
print(f"\n📊 Final Statistics:")
|
||||
print(f" Passed: {passed_count}")
|
||||
@@ -219,19 +219,19 @@ class TestNightlyGsm8KEval(unittest.TestCase):
|
||||
all_results = []
|
||||
total_test_start = time.time()
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print("AMD GSM8K Evaluation Test (TP=2 Instruction Models)")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"Benchmark: gsm8k (chat completions)")
|
||||
print(f"{'='*60}\n")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
for model_group, is_fp8, is_tp2 in self.model_groups:
|
||||
for model in model_group:
|
||||
with self.subTest(model=model):
|
||||
tp_size = 2 if is_tp2 else 1
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {model} (TP={tp_size}, FP8={is_fp8})")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
model_start = time.time()
|
||||
startup_time = None
|
||||
@@ -326,7 +326,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
|
||||
# Check all scores after collecting all results
|
||||
check_model_scores(all_results)
|
||||
print(
|
||||
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
|
||||
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time / 60:.1f} min)"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -179,9 +179,9 @@ class TestMiniMaxM25EvalAMD(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -179,9 +179,9 @@ class TestMiniMaxM27EvalAMD(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -172,14 +172,14 @@ class TestNightlyVLMMmmuEvalAMD(unittest.TestCase):
|
||||
all_results = []
|
||||
total_test_start = time.time()
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print("AMD VLM MMMU Evaluation Test")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"Benchmark: MMMU (100 samples)")
|
||||
print(f"Models to test: {len(self.models)}")
|
||||
for m in self.models:
|
||||
print(f" - {m['model_path']} (TP={m['tp_size']})")
|
||||
print(f"{'='*60}\n")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
for model_config in self.models:
|
||||
model_path = model_config["model_path"]
|
||||
@@ -189,9 +189,9 @@ class TestNightlyVLMMmmuEvalAMD(unittest.TestCase):
|
||||
error_message = None
|
||||
|
||||
with self.subTest(model=model_path):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {model_path} (TP={tp_size})")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
model_start = time.time()
|
||||
startup_time = None
|
||||
@@ -358,21 +358,21 @@ class TestNightlyVLMMmmuEvalAMD(unittest.TestCase):
|
||||
|
||||
summary += f"| {model} | {tp_size} | {score_str} | {threshold:.2f} | {startup_str} | {eval_str} | {total_str} | {status} |\n"
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print("SUMMARY - AMD VLM MMMU Evaluation")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
print(summary)
|
||||
print(f"\n📊 Final Statistics:")
|
||||
print(f" Passed: {passed_count}")
|
||||
print(f" Failed: {failed_count}")
|
||||
print(
|
||||
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
|
||||
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time / 60:.1f} min)"
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### TestNightlyVLMMmmuEvalAMD\n{summary}\n\n"
|
||||
f"**Total Runtime:** {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
|
||||
f"**Total Runtime:** {total_test_time:.1f}s ({total_test_time / 60:.1f} min)"
|
||||
)
|
||||
|
||||
if failed_models:
|
||||
|
||||
@@ -182,9 +182,9 @@ class TestDeepSeekR1EvalMI35x(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -180,9 +180,9 @@ class TestDeepSeekR1MXFP4ArFusionEvalMI35x(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -178,9 +178,9 @@ class TestDeepSeekR1MXFP4EvalMI35x(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -181,9 +181,9 @@ class TestDeepSeekR1MXFP4KvFp8EvalMI35x(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -89,8 +89,7 @@ class TestDeepseekV32DP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (deepseek-v32 DP MI35x)\n"
|
||||
f'{metrics["accuracy"]=:.3f}\n'
|
||||
f'### test_gsm8k (deepseek-v32 DP MI35x)\n{metrics["accuracy"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
|
||||
|
||||
@@ -103,8 +102,7 @@ class TestDeepseekV32DP(CustomTestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_bs_1_speed (deepseek-v32 DP MI35x)\n"
|
||||
f"{speed=:.2f} token/s\n"
|
||||
f"### test_bs_1_speed (deepseek-v32 DP MI35x)\n{speed=:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(speed, 10)
|
||||
|
||||
|
||||
@@ -183,9 +183,9 @@ class TestDeepSeekV32EvalMI35x(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -172,9 +172,9 @@ class TestGLM51EvalMI35x(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -80,8 +80,7 @@ class TestGLM51HiSparseEvalMI35x(unittest.TestCase):
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_gsm8k (glm-5.1 hisparse mi35x)\n"
|
||||
f'{metrics["score"]=:.3f}\n'
|
||||
f'### test_gsm8k (glm-5.1 hisparse mi35x)\n{metrics["score"]=:.3f}\n'
|
||||
)
|
||||
self.assertGreater(metrics["score"], 0.93)
|
||||
|
||||
|
||||
@@ -182,9 +182,9 @@ class TestGLM5EvalMI35x(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -185,9 +185,9 @@ class TestGLM5MXFP4EvalMI35x(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -196,9 +196,9 @@ class TestGptOssEvalMI35x(unittest.TestCase):
|
||||
|
||||
for config in self.models:
|
||||
with self.subTest(model=config.model_path):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {config.model_path}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -185,9 +185,9 @@ class TestGptOssW4A8Mxfp4EvalMI35x(unittest.TestCase):
|
||||
|
||||
for config in self.models:
|
||||
with self.subTest(model=config.model_path):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {config.model_path}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
@@ -132,9 +132,9 @@ class TestKimiK25AiterMlaEvalMI35x(unittest.TestCase):
|
||||
for config in self.models:
|
||||
display_name = config.get_display_name()
|
||||
with self.subTest(model=display_name):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Testing: {display_name}")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
env = os.environ.copy()
|
||||
for key, value in config.env_vars.items():
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user