[CI][RFC] Replace black-jupyter with ruff-format (#37210)

Co-authored-by: Alison Shao <a.shao@wustl.edu>
This commit is contained in:
Alex Nails
2026-09-02 19:46:08 -07:00
committed by GitHub
co-authored by Alison Shao
parent 2641e427be
commit 28262c20df
1411 changed files with 7766 additions and 8176 deletions
@@ -58,7 +58,7 @@ class TestDeepseekV3Basic(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (deepseek-v3)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.935)
@@ -70,7 +70,7 @@ class TestDeepseekV3Basic(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v3)\n" f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (deepseek-v3)\n{speed=:.2f} token/s\n"
)
if is_in_amd_ci():
self.assertGreater(speed, 12)
@@ -61,7 +61,7 @@ class TestDeepseekV32DP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (deepseek-v32)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.935)
@@ -73,7 +73,7 @@ class TestDeepseekV32DP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32)\n" f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (deepseek-v32)\n{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 50)
@@ -119,7 +119,7 @@ class TestDeepseekV32TP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (deepseek-v32)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.935)
@@ -131,7 +131,7 @@ class TestDeepseekV32TP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32)\n" f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (deepseek-v32)\n{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 80)
@@ -180,7 +180,7 @@ class TestGLM5DP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (glm-5)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (glm-5)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.935)
@@ -192,7 +192,7 @@ class TestGLM5DP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (glm-5)\n" f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (glm-5)\n{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 40)
@@ -238,7 +238,7 @@ class TestGLM5TP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (glm-5)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (glm-5)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.935)
@@ -250,7 +250,7 @@ class TestGLM5TP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (glm-5)\n" f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (glm-5)\n{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 60)
+1 -1
View File
@@ -138,7 +138,7 @@ def get_rdma_devices_args():
if not (base_rdma_group <= gpu_idx < base_rdma_group + 4):
warnings.warn(
f"GPU index {gpu_idx} is outside expected group "
f"{base_rdma_group}-{base_rdma_group+3}"
f"{base_rdma_group}-{base_rdma_group + 3}"
)
# 3. Generate RDMA device names
@@ -21,7 +21,6 @@ TEST_MODEL_MATRIX = {
class TestAscendDeepSeekMTP(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.models = TEST_MODEL_MATRIX.keys()
@@ -274,12 +274,12 @@ def compare_outputs(trtllm_out, reference_out, tolerance=1e-2):
"""Compare outputs with detailed analysis."""
# Basic checks
assert (
trtllm_out.shape == reference_out.shape
), f"Shape mismatch: {trtllm_out.shape} vs {reference_out.shape}"
assert (
trtllm_out.dtype == reference_out.dtype
), f"Dtype mismatch: {trtllm_out.dtype} vs {reference_out.dtype}"
assert trtllm_out.shape == reference_out.shape, (
f"Shape mismatch: {trtllm_out.shape} vs {reference_out.shape}"
)
assert trtllm_out.dtype == reference_out.dtype, (
f"Dtype mismatch: {trtllm_out.dtype} vs {reference_out.dtype}"
)
# Check for NaN/Inf
assert not torch.isnan(trtllm_out).any(), "TRTLLM output contains NaN"
@@ -310,7 +310,7 @@ def compare_outputs(trtllm_out, reference_out, tolerance=1e-2):
trt_val = trtllm_out[idx_tuple].item()
ref_val = reference_out[idx_tuple].item()
print(
f" [{idx_tuple}]: TRTLLM={trt_val:.6f}, Reference={ref_val:.6f}, diff={abs(trt_val-ref_val):.6f}"
f" [{idx_tuple}]: TRTLLM={trt_val:.6f}, Reference={ref_val:.6f}, diff={abs(trt_val - ref_val):.6f}"
)
return all_close
@@ -72,9 +72,9 @@ class _BeamSweepBase(CustomTestCase):
cls.prompts = [r.prompt for r in rows if r.prompt_len < MAX_PROMPT_LEN][
:NUM_PROMPTS
]
assert (
len(cls.prompts) == NUM_PROMPTS
), f"only {len(cls.prompts)} short prompts sampled"
assert len(cls.prompts) == NUM_PROMPTS, (
f"only {len(cls.prompts)} short prompts sampled"
)
@classmethod
def tearDownClass(cls):
@@ -53,15 +53,15 @@ class TestAbortBasic(ScriptedTestCase):
"finished",
"unknown",
), f"after abort r should be finished/unknown, got {r.status}"
assert (
r.kv_pages == 0
), f"abort must release KV; r.kv_pages={r.kv_pages} after abort"
assert (
r.req is None or r.req.kv.req_pool_idx is None
), f"abort must release row; r.req={r.req} after abort"
assert (
r.lock_refs == 0
), f"abort must release lock_refs; r.lock_refs={r.lock_refs}"
assert r.kv_pages == 0, (
f"abort must release KV; r.kv_pages={r.kv_pages} after abort"
)
assert r.req is None or r.req.kv.req_pool_idx is None, (
f"abort must release row; r.req={r.req} after abort"
)
assert r.lock_refs == 0, (
f"abort must release lock_refs; r.lock_refs={r.lock_refs}"
)
def test_abort_at_chunk_0(self):
self.server.execute_script(self._script_abort_at_chunk_0)
@@ -436,9 +436,9 @@ class TestAbortBasic(ScriptedTestCase):
assert not r.is_chunking, "aborted gap req must stay out of chunking"
if r.req is not None:
assert (
r.req.inflight_middle_chunks == 0
), f"inflight_middle_chunks not cleared; got {r.req.inflight_middle_chunks}"
assert r.req.inflight_middle_chunks == 0, (
f"inflight_middle_chunks not cleared; got {r.req.inflight_middle_chunks}"
)
def test_abort_when_chunked_only_then_idle(self):
self.server.execute_script(self._script_abort_when_chunked_only_then_idle)
@@ -509,10 +509,10 @@ class TestAbortBasic(ScriptedTestCase):
yield from _drain_until_released(t, r1)
assert r1.kv_pages == 0, (
f"force_retract + abort same yield must release KV; got " f"{r1.kv_pages}"
f"force_retract + abort same yield must release KV; got {r1.kv_pages}"
)
assert r1.req is None or r1.req.kv.req_pool_idx is None, (
f"force_retract + abort same yield must release row; got " f"{r1.req}"
f"force_retract + abort same yield must release row; got {r1.req}"
)
assert r1.lock_refs == 0, (
f"force_retract + abort same yield must release lock_refs; "
@@ -73,9 +73,9 @@ class TestChunkSizeDefault(ScriptedTestCase):
r = t.start_req(prompt_len=1, max_new_tokens=2)
yield from run_until_finished(r)
assert r.finished
assert (
r.chunks_done == 0
), f"single-token prompt should not chunk, got chunks_done={r.chunks_done}"
assert r.chunks_done == 0, (
f"single-token prompt should not chunk, got chunks_done={r.chunks_done}"
)
def test_chunk_size_256_prompt_100x(self):
self.server.execute_script(self._script_chunk_size_256_prompt_100x)
@@ -26,9 +26,9 @@ class TestScriptedHttpSmoke(ScriptedTestCase):
break
yield
assert r.finished
assert (
saw_chunking
), "expected the req to hold the chunked_req slot at least once"
assert saw_chunking, (
"expected the req to hold the chunked_req slot at least once"
)
def test_two_reqs_finish(self):
self.server.execute_script(self._script_two_reqs_finish)
@@ -123,9 +123,9 @@ class TestSWAHalfWindowChunk(ScriptedTestCase):
r = t.start_req(prompt_len=2 * _SWA_WINDOW, max_new_tokens=4)
yield from run_until_finished(r, max_steps=800)
assert r.finished
assert (
r.chunks_done >= 4
), f"expected >=4 chunks for 2*window / (window/2), got {r.chunks_done}"
assert r.chunks_done >= 4, (
f"expected >=4 chunks for 2*window / (window/2), got {r.chunks_done}"
)
assert len(r.req.output_ids) == 4
@@ -171,9 +171,9 @@ class TestSWARadix(ScriptedTestCase):
)
yield from run_until_finished(r2, max_steps=800)
assert r2.finished
assert (
r2.req.cached_tokens > 0
), f"r2 must hit the radix prefix, got cached_tokens={r2.req.cached_tokens}"
assert r2.req.cached_tokens > 0, (
f"r2 must hit the radix prefix, got cached_tokens={r2.req.cached_tokens}"
)
if __name__ == "__main__":
@@ -38,9 +38,9 @@ class TestInvariantsBasic(ScriptedTestCase):
for _ in range(DEFAULT_MAX_STEPS):
if r.is_chunking:
observed_chunking = True
assert (
r.kv_pages > 0
), f"kv_pages must be > 0 while is_chunking; got {r.kv_pages}"
assert r.kv_pages > 0, (
f"kv_pages must be > 0 while is_chunking; got {r.kv_pages}"
)
if r.finished:
break
yield
@@ -63,9 +63,9 @@ class TestInvariantsBasic(ScriptedTestCase):
+ comp.get("decode", [])
+ comp.get("chunked", [])
)
assert (
r.rid in all_rids
), f"running but not in batch_composition: {comp}"
assert r.rid in all_rids, (
f"running but not in batch_composition: {comp}"
)
if r.finished:
return
yield
@@ -157,9 +157,9 @@ class TestInvariantsBasic(ScriptedTestCase):
t.flush_cache()
yield
final = t.engine_stats()
assert (
final["kv_pool_free"] >= baseline["kv_pool_free"]
), f"KV leak: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
assert final["kv_pool_free"] >= baseline["kv_pool_free"], (
f"KV leak: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
)
assert final["req_pool_free"] >= baseline["req_pool_free"]
def test_long_lived_engine_reps_chunked(self):
@@ -212,9 +212,9 @@ class TestInvariantsBasic(ScriptedTestCase):
t.flush_cache()
yield
final_kv = t.engine_stats()["kv_pool_free"]
assert (
final_kv >= baseline_kv
), f"KV leak after sustained chunked load: {baseline_kv} -> {final_kv}"
assert final_kv >= baseline_kv, (
f"KV leak after sustained chunked load: {baseline_kv} -> {final_kv}"
)
def test_round_robin_short_and_chunked(self):
self.server.execute_script(self._script_round_robin_short_and_chunked)
@@ -356,9 +356,9 @@ class TestInvariantsBasic(ScriptedTestCase):
)
yield from run_until_finished(r)
assert r.finished
assert (
r.chunks_done >= 2
), f"VERY_LONG_PROMPT_LEN should chunk; got chunks_done={r.chunks_done}"
assert r.chunks_done >= 2, (
f"VERY_LONG_PROMPT_LEN should chunk; got chunks_done={r.chunks_done}"
)
assert len(r.req.output_ids) == n, (
f"ignore_eos=True + max_new_tokens={n} must produce exactly "
f"{n} output tokens; got len(output_tokens)={len(r.req.output_ids)}"
@@ -62,9 +62,9 @@ class TestKVPressureBasic(ScriptedTestCase):
f"long req must really chunk under pinned cache; got chunks_done="
f"{r_long.chunks_done}"
)
assert (
r_long.lock_refs == 0
), f"req {r_long.rid} leaked {r_long.lock_refs} lock_refs after finish"
assert r_long.lock_refs == 0, (
f"req {r_long.rid} leaked {r_long.lock_refs} lock_refs after finish"
)
t._release_exhausted_pools()
final_lock_refs = t.get_all_node_lock_refs()
@@ -171,9 +171,9 @@ class TestLifecycleBasic(ScriptedTestCase):
assert "running" in seen, f"never observed running status; seen={seen}"
assert seen[-1] == "finished", f"final status must be finished; seen={seen}"
finished_idx = seen.index("finished")
assert all(
s in ("finished",) for s in seen[finished_idx:]
), f"status regressed after finish; seen={seen}"
assert all(s in ("finished",) for s in seen[finished_idx:]), (
f"status regressed after finish; seen={seen}"
)
def test_long_prompt_only_one_decode(self):
self.server.execute_script(self._script_long_prompt_only_one_decode)
@@ -279,8 +279,7 @@ class TestLifecycleBasic(ScriptedTestCase):
assert r1.finished and r2.finished
assert r2.chunks_done == 0
assert r2.req.cached_tokens > 0, (
f"r2 must hit r1's radix prefix; got cached_tokens="
f"{r2.req.cached_tokens}"
f"r2 must hit r1's radix prefix; got cached_tokens={r2.req.cached_tokens}"
)
assert len(r2.req.output_ids) == 2
@@ -408,9 +407,9 @@ class TestLifecycleBasic(ScriptedTestCase):
t.flush_cache()
yield
final = t.engine_stats()["kv_pool_free"]
assert (
final >= baseline - 1
), f"KV pool drift: baseline={baseline}, final={final}"
assert final >= baseline - 1, (
f"KV pool drift: baseline={baseline}, final={final}"
)
def test_abort_all_during_chunked(self):
self.server.execute_script(self._script_abort_all_during_chunked)
@@ -143,14 +143,13 @@ class TestMaxNewTokensFirstDecodeAdjacent(ScriptedTestCase):
decode_records = _decode_records(batch_log, r.rid)
assert len(decode_records) == max_new_tokens, (
f"expected {max_new_tokens} decode forwards, got " f"{len(decode_records)}"
f"expected {max_new_tokens} decode forwards, got {len(decode_records)}"
)
rid_modes = [rec.mode for rec in rid_records]
first_decode_pos = rid_modes.index("decode")
assert first_decode_pos >= 1, (
f"first decode must be preceded by an extend chunk; rid_modes="
f"{rid_modes}"
f"first decode must be preceded by an extend chunk; rid_modes={rid_modes}"
)
assert rid_modes[first_decode_pos - 1] == "extend", (
f"record immediately before the first decode (in this rid's "
@@ -20,9 +20,9 @@ def _drain_flush_then_assert_no_kv_leak(t: ScriptedContext, baseline: dict):
t.flush_cache()
yield
final = t.engine_stats()
assert (
final["kv_pool_free"] >= baseline["kv_pool_free"]
), f"KV leak: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
assert final["kv_pool_free"] >= baseline["kv_pool_free"], (
f"KV leak: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
)
class TestMultiReqBasic(ScriptedTestCase):
@@ -39,9 +39,9 @@ class TestMultiReqBasic(ScriptedTestCase):
yield
assert r1.is_chunking, "r1 should still be chunking"
assert (
not r2.is_chunking
), "r2 must wait for r1's chunk loop to clear before chunking"
assert not r2.is_chunking, (
"r2 must wait for r1's chunk loop to clear before chunking"
)
yield from run_until_all_finished([r1, r2])
assert r1.finished and r2.finished
@@ -144,9 +144,9 @@ class TestMultiReqBasic(ScriptedTestCase):
r2 = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN + 8, max_new_tokens=2)
yield from run_until_finished(r2)
assert r1.finished and r2.finished
assert (
r2.chunks_done < r1.chunks_done
), "r2 reuses r1's cached prefix, so it should chunk fewer times"
assert r2.chunks_done < r1.chunks_done, (
"r2 reuses r1's cached prefix, so it should chunk fewer times"
)
def test_trickle_per_yield_50(self):
self.server.execute_script(self._script_trickle_per_yield_50)
@@ -175,9 +175,9 @@ class TestPPPdmux(ScriptedTestCase):
def _script_pp_split_prefill_chunked_no_merge_assert(t: ScriptedContext):
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
yield from run_until_finished(r, max_steps=800)
assert (
r.finished
), "engine died before req finished — merge_batch assert may have tripped"
assert r.finished, (
"engine died before req finished — merge_batch assert may have tripped"
)
assert r.chunks_done >= 2, (
f"pdmux + chunked path must produce >=2 chunks to exercise "
f"split_prefill_batch filter; got chunks_done={r.chunks_done}"
@@ -30,9 +30,9 @@ class TestPriorityBasic(ScriptedTestCase):
t.pause_generation(mode="retract")
yield
assert (
r.status == "waiting"
), f"force-retracted chunked req must be back in waiting; got {r.status}"
assert r.status == "waiting", (
f"force-retracted chunked req must be back in waiting; got {r.status}"
)
assert r.kv_pages == 0, f"retract must release KV; got {r.kv_pages}"
t.continue_generation()
@@ -76,9 +76,9 @@ class TestRadixBasic(ScriptedTestCase):
f"after eviction r2 must re-chunk from scratch; "
f"chunks_done={r2.chunks_done} cached_tokens={r2.req.cached_tokens}"
)
assert (
r2.req.cached_tokens == 0
), f"eviction must clear r1's prefix; cached_tokens={r2.req.cached_tokens}"
assert r2.req.cached_tokens == 0, (
f"eviction must clear r1's prefix; cached_tokens={r2.req.cached_tokens}"
)
assert r2.kv_pages == 0
assert r2.lock_refs == 0
@@ -219,9 +219,9 @@ class TestRadixBasic(ScriptedTestCase):
r = t.start_req(prompt_len=prompt_len, max_new_tokens=2)
yield from run_until_finished(r, max_steps=400)
assert r.finished
assert (
r.chunks_done == 0
), f"full prefix hit must skip chunked path; got chunks_done={r.chunks_done}"
assert r.chunks_done == 0, (
f"full prefix hit must skip chunked path; got chunks_done={r.chunks_done}"
)
def test_radix_evict_race_concurrent_chunked_admit(self):
self.server.execute_script(
@@ -319,9 +319,9 @@ class TestRadixNoTailChunked(ScriptedTestCase):
"test must observe r as the in-flight chunked_req at least once; the "
"no-tail else branch was never exercised"
)
assert (
r.kv_pages == 0
), f"finished chunked req must release KV; got {r.kv_pages}"
assert r.kv_pages == 0, (
f"finished chunked req must release KV; got {r.kv_pages}"
)
class TestRadixHitCountInvariant(ScriptedTestCase):
@@ -100,9 +100,9 @@ class TestRegressionBasic(ScriptedTestCase):
f"observed max={observed_max} (pre-fix bug would bump to 2 "
f"at the last-chunk admit boundary)"
)
assert (
cleared_inflight
), "inflight_middle_chunks should be 0 once the chunk loop clears"
assert cleared_inflight, (
"inflight_middle_chunks should be 0 once the chunk loop clears"
)
yield from run_until_finished(r)
assert r.finished
@@ -247,15 +247,15 @@ class TestRegressionBasic(ScriptedTestCase):
t.abort(r)
yield from _drain_until_released(t, r)
assert (
r.req.kv.req_pool_idx is None
), f"96d4749094: abort must release row; got row_idx={r.req.kv.req_pool_idx!r}"
assert (
r.kv_pages == 0
), f"96d4749094: abort must release KV; got kv_pages={r.kv_pages}"
assert (
r.lock_refs == 0
), f"96d4749094: abort must release lock_ref; got lock_refs={r.lock_refs}"
assert r.req.kv.req_pool_idx is None, (
f"96d4749094: abort must release row; got row_idx={r.req.kv.req_pool_idx!r}"
)
assert r.kv_pages == 0, (
f"96d4749094: abort must release KV; got kv_pages={r.kv_pages}"
)
assert r.lock_refs == 0, (
f"96d4749094: abort must release lock_ref; got lock_refs={r.lock_refs}"
)
assert not r.is_chunking
assert r.req.inflight_middle_chunks == 0
assert sum(t.get_all_node_lock_refs().values()) == baseline_refs
@@ -34,8 +34,7 @@ class TestSamplingBasic(ScriptedTestCase):
if r.rid in rec.rids and rec.mode == "decode"
]
assert len(decode_records) == 0, (
f"max_new_tokens=0 must run zero decode forwards; got "
f"{len(decode_records)}"
f"max_new_tokens=0 must run zero decode forwards; got {len(decode_records)}"
)
def test_max_new_tokens_one_long_chunked(self):
@@ -100,8 +99,7 @@ class TestSamplingBasic(ScriptedTestCase):
assert r.chunks_done >= 2
assert len(r.req.output_ids) == 16
assert isinstance(r.req.finished_reason, FINISH_LENGTH), (
f"ignore_eos=True must finish via length cap; got "
f"{r.req.finished_reason!r}"
f"ignore_eos=True must finish via length cap; got {r.req.finished_reason!r}"
)
def test_return_logprob_top_logprobs_chunked(self):
@@ -176,9 +174,9 @@ class TestSamplingBasic(ScriptedTestCase):
)
yield from run_until_finished(r)
assert r.finished
assert (
r.chunks_done >= 2
), f"prompt should span multiple chunks, got chunks_done={r.chunks_done}"
assert r.chunks_done >= 2, (
f"prompt should span multiple chunks, got chunks_done={r.chunks_done}"
)
assert r.req.logprob is not None
input_lp = r.req.logprob.input_token_logprobs_val
assert len(input_lp) == prompt_len, (
@@ -201,9 +199,9 @@ class TestSamplingBasic(ScriptedTestCase):
)
yield from run_until_finished(r)
assert r.finished
assert (
r.chunks_done >= 3
), f"prompt should span 3+ chunks, got chunks_done={r.chunks_done}"
assert r.chunks_done >= 3, (
f"prompt should span 3+ chunks, got chunks_done={r.chunks_done}"
)
assert r.req.logprob is not None
input_lp = r.req.logprob.input_token_logprobs_val
assert len(input_lp) == prompt_len - start_len, (
@@ -246,9 +244,9 @@ class TestSamplingBasic(ScriptedTestCase):
)
yield from run_until_finished(r_eos, max_steps=2000)
assert r_eos.finished
assert (
r_eos.chunks_done >= 2
), f"scenario 1 should chunk; got chunks_done={r_eos.chunks_done}"
assert r_eos.chunks_done >= 2, (
f"scenario 1 should chunk; got chunks_done={r_eos.chunks_done}"
)
assert isinstance(r_eos.req.finished_reason, FINISH_MATCHED_TOKEN), (
f"a stop token the model deterministically produces under greedy must "
f"finish via the matched-token path; got {r_eos.req.finished_reason!r}"
@@ -261,9 +259,9 @@ class TestSamplingBasic(ScriptedTestCase):
)
yield from run_until_finished(r_length)
assert r_length.finished
assert (
r_length.chunks_done >= 2
), f"scenario 2 should chunk; got chunks_done={r_length.chunks_done}"
assert r_length.chunks_done >= 2, (
f"scenario 2 should chunk; got chunks_done={r_length.chunks_done}"
)
assert isinstance(r_length.req.finished_reason, FINISH_LENGTH), (
f"ignore_eos=True + max_new_tokens=4 chunked must finish via "
f"length cap; got {r_length.req.finished_reason!r}"
@@ -45,9 +45,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
for _ in range(DEFAULT_MAX_STEPS):
if r.is_chunking:
saw_chunking = True
assert (
not t.is_idle
), "scheduler must not idle while chunked_req is in flight"
assert not t.is_idle, (
"scheduler must not idle while chunked_req is in flight"
)
if r.finished:
break
yield
@@ -102,9 +102,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
break
yield
assert (
t.scheduler.chunked_req is None
), f"abort must clear the chunked slot; got {t.scheduler.chunked_req!r}"
assert t.scheduler.chunked_req is None, (
f"abort must clear the chunked slot; got {t.scheduler.chunked_req!r}"
)
assert r.kv_pages == 0
assert r.lock_refs == 0
@@ -193,19 +193,19 @@ class TestSpecialCaseBasic(ScriptedTestCase):
if r1.is_chunking:
saw_r1_chunking = True
comp = t.batch_composition()
assert r1.rid in comp.get(
"chunked", []
), f"mid-chunk r1 must occupy the chunked role; got {comp!r}"
assert r1.rid not in comp.get(
"running", []
), f"chunked r1 must be excluded from the running role; got {comp!r}"
assert r1.rid in comp.get("chunked", []), (
f"mid-chunk r1 must occupy the chunked role; got {comp!r}"
)
assert r1.rid not in comp.get("running", []), (
f"chunked r1 must be excluded from the running role; got {comp!r}"
)
if r1.finished and r2.finished:
break
yield
assert r1.finished and r2.finished
assert (
saw_r1_chunking
), "r1 must have chunked at some point to exercise the exclude branch"
assert saw_r1_chunking, (
"r1 must have chunked at some point to exercise the exclude branch"
)
@unittest.skip(
"pdmux split_prefill_batch requires the pdmux topology — "
@@ -257,9 +257,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
t.pause_generation(mode="retract")
yield
assert (
t.scheduler.chunked_req is None
), f"pause(retract) must clear chunked_req; got {t.scheduler.chunked_req!r}"
assert t.scheduler.chunked_req is None, (
f"pause(retract) must clear chunked_req; got {t.scheduler.chunked_req!r}"
)
assert not r.finished, "retract must re-queue r, not finish or abort it"
assert r.status == "waiting", (
f"retracted chunked req must return to the waiting queue; "
@@ -268,9 +268,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
t.continue_generation()
yield from run_until_finished(r)
assert (
r.finished
), "continue_generation must drive the re-queued req to completion"
assert r.finished, (
"continue_generation must drive the re-queued req to completion"
)
def test_retract_during_gap_inflight_middle_chunks_positive(self):
self.server.execute_script(
@@ -304,9 +304,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
t.continue_generation()
yield from run_until_finished(r, max_steps=2000)
assert (
r.finished
), "continue_generation must drive the re-queued req to completion"
assert r.finished, (
"continue_generation must drive the re-queued req to completion"
)
assert r.kv_pages == 0
assert len(r.req.output_ids) == 2
@@ -344,9 +344,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
break
yield
assert r.finished
assert (
saw_chunking
), "test must observe the dual-queue chunked state at least once"
assert saw_chunking, (
"test must observe the dual-queue chunked state at least once"
)
assert saw_dedup, (
"test must observe the chunked req with a committed prefix so the "
"dedup subtraction is actually exercised"
@@ -491,9 +491,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
break
yield
assert r.finished
assert (
saw_mid_chunk
), "test must observe the fill_ids reset boundary at least once"
assert saw_mid_chunk, (
"test must observe the fill_ids reset boundary at least once"
)
assert r.finished
def test_chunked_req_slot_cleared_when_chunk_completes(self):
@@ -516,9 +516,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
yield
assert r.finished
assert saw_chunking, "req should have occupied the chunked_req slot mid-chunk"
assert (
s.chunked_req is None
), f"chunked_req slot must clear after last chunk; got {s.chunked_req!r}"
assert s.chunked_req is None, (
f"chunked_req slot must clear after last chunk; got {s.chunked_req!r}"
)
def test_second_chunked_admit_blocked_when_chunked_req_set(self):
self.server.execute_script(
@@ -571,9 +571,9 @@ class TestSpecialCaseBasic(ScriptedTestCase):
progressed = False
for _ in range(DEFAULT_MAX_STEPS):
if r.is_chunking:
assert (
not t.is_idle
), "scheduler must not go idle while a chunked req is in flight"
assert not t.is_idle, (
"scheduler must not go idle while a chunked req is in flight"
)
cur_chunks_done = r.chunks_done
if cur_chunks_done > prev_chunks_done:
progressed = True
@@ -681,9 +681,9 @@ class TestSpecialCaseMixedChunk(ScriptedTestCase):
prompt_token=310,
)
yield from run_until(r, lambda h: h.is_chunking)
assert (
t.last_batch_forward_mode != "MIXED"
), f"return_logprob must disable mixed-chunk path; got {t.last_batch_forward_mode!r}"
assert t.last_batch_forward_mode != "MIXED", (
f"return_logprob must disable mixed-chunk path; got {t.last_batch_forward_mode!r}"
)
yield from run_until_finished(r)
def test_mixed_chunk_with_running_batch(self):
@@ -700,9 +700,9 @@ class TestSpecialCaseMixedChunk(ScriptedTestCase):
yield
yield from run_until(r_chunk, lambda h: h.is_chunking)
assert (
t.last_batch_forward_mode == "MIXED"
), f"chunked admission with running batch must enter MIXED; got {t.last_batch_forward_mode!r}"
assert t.last_batch_forward_mode == "MIXED", (
f"chunked admission with running batch must enter MIXED; got {t.last_batch_forward_mode!r}"
)
for _ in range(DEFAULT_MAX_STEPS * 2):
if r_chunk.finished and r_dec.finished:
break
@@ -743,9 +743,9 @@ class TestSpecialCaseNoChunking(ScriptedTestCase):
def _script_chunk_size_negative_disables_chunking(t: ScriptedContext):
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
for _ in range(DEFAULT_MAX_STEPS):
assert (
not r.is_chunking
), "chunked_prefill_size=-1 should disable chunked path"
assert not r.is_chunking, (
"chunked_prefill_size=-1 should disable chunked path"
)
if r.finished:
return
yield
@@ -825,9 +825,9 @@ class TestSpecialCaseHiCache(ScriptedTestCase):
yield
assert r.finished
assert saw_chunking, "test must observe r mid-chunk at least once"
assert (
first_chunk_snap is not None
), "test must snapshot cached_tokens at the first chunk boundary"
assert first_chunk_snap is not None, (
"test must snapshot cached_tokens at the first chunk boundary"
)
def test_hicache_cached_tokens_set_once_invariant(self):
self.server.execute_script(
@@ -862,9 +862,9 @@ class TestSpecialCaseHiCache(ScriptedTestCase):
break
yield
assert r.finished
assert (
saw_chunking
), "test must observe the req mid-chunk (chunks_done >= 1) at least once"
assert saw_chunking, (
"test must observe the req mid-chunk (chunks_done >= 1) at least once"
)
assert snap is not None, "test must snapshot the cached_tokens_* breakdown"
@@ -1011,24 +1011,23 @@ class TestSpecialCaseRetractMerge(ScriptedTestCase):
t.pause_generation(mode="retract")
yield
assert (
s.last_batch is None
), "retract must clear last_batch after merging the extend chunk batch"
assert s.last_batch is None, (
"retract must clear last_batch after merging the extend chunk batch"
)
assert len(s.running_batch.reqs) == 0, (
"the merged extend chunk batch must be retracted out of running_batch, "
f"not stranded; got {len(s.running_batch.reqs)} reqs"
)
assert (
r.status == "waiting"
), f"retracted chunked req must return to the waiting queue; got {r.status!r}"
assert r.status == "waiting", (
f"retracted chunked req must return to the waiting queue; got {r.status!r}"
)
assert r.kv_pages == 0
t.continue_generation()
yield from run_until_finished(r)
assert r.finished
assert len(r.req.output_ids) == 2, (
f"resumed req must emit exactly max_new_tokens; got "
f"{len(r.req.output_ids)}"
f"resumed req must emit exactly max_new_tokens; got {len(r.req.output_ids)}"
)
@@ -1183,9 +1182,9 @@ class TestSpecialCaseRetractedStain(ScriptedTestCase):
assert r.finished
req = r.req
assert (
req.retracted_stain is True
), "retract must set retracted_stain so the cached-token recount is suppressed"
assert req.retracted_stain is True, (
"retract must set retracted_stain so the cached-token recount is suppressed"
)
assert req.cached_tokens == cached_before, (
f"retracted_stain must suppress re-adding pre_len-already_computed on "
f"resume; cached_tokens grew from {cached_before} to {req.cached_tokens}"
@@ -1237,17 +1236,17 @@ class TestSpecialCaseMiddleChunkNoToken(ScriptedTestCase):
f"middle chunk must not append an output token; got "
f"output_ids len {len(r.req.output_ids)}"
)
assert (
r.status != "finished"
), "middle chunk must not finish the req (skip_stream_req)"
assert r.status != "finished", (
"middle chunk must not finish the req (skip_stream_req)"
)
if r.finished:
break
yield
assert r.finished
assert saw_middle_chunk, "test must observe r mid-chunk at least once"
assert (
len(r.req.output_ids) >= 1
), "output tokens must appear only after the chunked prefill completes"
assert len(r.req.output_ids) >= 1, (
"output tokens must appear only after the chunked prefill completes"
)
if __name__ == "__main__":
@@ -264,7 +264,6 @@ def _ref_compress(
class TestFusedCompressAttn(unittest.TestCase):
def _run_test(
self,
ratio: int,
@@ -397,7 +396,6 @@ class TestFusedCompressAttn(unittest.TestCase):
class TestStateOrdering(unittest.TestCase):
def test_write_then_compress(self):
"""Verify write-first, compress-second matches reference."""
device = torch.device("cuda")
+19 -13
View File
@@ -149,7 +149,7 @@ def test_main(
for with_topk in (False, True):
if local_rank == 0:
print(
f'[testing] Running with {"FP8" if isinstance(current_x, tuple) else "BF16"}, {"with" if with_topk else "without"} top-k (async={async_mode}, previous={previous_mode}) ...',
f"[testing] Running with {'FP8' if isinstance(current_x, tuple) else 'BF16'}, {'with' if with_topk else 'without'} top-k (async={async_mode}, previous={previous_mode}) ...",
flush=True,
end="",
)
@@ -192,9 +192,9 @@ def test_main(
# Checks
recv_gbl_rank_prefix_sum = handle[-4]
assert gbl_num_tokens_per_rank[rank].item() == recv_x.size(
0
), f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}"
assert gbl_num_tokens_per_rank[rank].item() == recv_x.size(0), (
f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}"
)
assert (
gbl_num_tokens_per_expert.view(num_ranks, -1)[rank].tolist()
== recv_num_tokens_per_expert_list
@@ -316,10 +316,13 @@ def test_main(
tune_args = {"x": current_x, "handle": handle, "config": config}
t = bench(lambda: buffer.dispatch(**tune_args))[0]
if t < best_time:
best_time, best_results = t, (
num_sms,
nvl_chunk_size,
rdma_chunk_size,
best_time, best_results = (
t,
(
num_sms,
nvl_chunk_size,
rdma_chunk_size,
),
)
if local_rank == 0:
print(
@@ -328,7 +331,7 @@ def test_main(
)
if local_rank == 0:
print(
f'[tuning] Best dispatch ({"FP8" if isinstance(current_x, tuple) else "BF16"}): SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {rdma_send_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)',
f"[tuning] Best dispatch ({'FP8' if isinstance(current_x, tuple) else 'BF16'}): SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {rdma_send_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)",
flush=True,
)
print("", flush=True)
@@ -385,10 +388,13 @@ def test_main(
flush=True,
)
if t < best_time:
best_time, best_results = t, (
num_sms,
nvl_chunk_size,
rdma_chunk_size,
best_time, best_results = (
t,
(
num_sms,
nvl_chunk_size,
rdma_chunk_size,
),
)
if local_rank == 0:
+5 -5
View File
@@ -120,7 +120,7 @@ def test_main(
for with_topk in (False, True):
if local_rank == 0:
print(
f'[testing] Running with {"FP8" if isinstance(current_x, tuple) else "BF16"}, {"with" if with_topk else "without"} top-k (async={async_mode}, previous={previous_mode}) ...',
f"[testing] Running with {'FP8' if isinstance(current_x, tuple) else 'BF16'}, {'with' if with_topk else 'without'} top-k (async={async_mode}, previous={previous_mode}) ...",
flush=True,
end="",
)
@@ -162,9 +162,9 @@ def test_main(
# Checks
rank_prefix_matrix = handle[0]
assert gbl_num_tokens_per_rank[rank].item() == recv_x.size(
0
), f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}"
assert gbl_num_tokens_per_rank[rank].item() == recv_x.size(0), (
f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}"
)
assert (
gbl_num_tokens_per_expert.view(num_ranks, -1)[rank].tolist()
== recv_num_tokens_per_expert_list
@@ -280,7 +280,7 @@ def test_main(
)
if local_rank == 0:
print(
f'[tuning] Best dispatch ({"FP8" if isinstance(current_x, tuple) else "BF16"}): SMs {best_results[0]}, NVL chunk {best_results[1]}, {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)',
f"[tuning] Best dispatch ({'FP8' if isinstance(current_x, tuple) else 'BF16'}): SMs {best_results[0]}, NVL chunk {best_results[1]}, {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)",
flush=True,
)
print("", flush=True)
+19 -18
View File
@@ -36,9 +36,9 @@ def test_main(
# NOTES: the integers greater than 256 exceeds the BF16 precision limit
rank_offset = 128
assert (
num_ranks - rank_offset < 257
), "Too many ranks (exceeding test precision limit)"
assert num_ranks - rank_offset < 257, (
"Too many ranks (exceeding test precision limit)"
)
x = torch.ones((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") * (
rank - rank_offset
@@ -55,9 +55,9 @@ def test_main(
# Randomly mask some positions
for i in range(10):
topk_idx[random.randint(0, num_tokens - 1), random.randint(0, num_topk - 1)] = (
-1
)
topk_idx[
random.randint(0, num_tokens - 1), random.randint(0, num_topk - 1)
] = -1
# Check dispatch correctness
do_check = True
@@ -114,9 +114,9 @@ def test_main(
assert (
num_valid_tokens == (recv_layout_range & int_mask).sum().item()
), f"{num_valid_tokens} != {recv_layout_range & int_mask}.sum().item()"
assert (
num_valid_tokens == (all_topk_idx == expert_id).sum().item()
), f"{num_valid_tokens} != {(all_topk_idx == expert_id).sum().item()}"
assert num_valid_tokens == (all_topk_idx == expert_id).sum().item(), (
f"{num_valid_tokens} != {(all_topk_idx == expert_id).sum().item()}"
)
# Check received data
recv_x = recv_x[:num_valid_tokens]
@@ -127,9 +127,10 @@ def test_main(
recv_x[:, -128:] - recv_src_info.view(-1, 1) % num_tokens
).sum().item() == 0
for j in range(num_ranks):
begin_idx, count = (recv_layout_range[j] >> 32).item(), (
recv_layout_range[j] & int_mask
).item()
begin_idx, count = (
(recv_layout_range[j] >> 32).item(),
(recv_layout_range[j] & int_mask).item(),
)
assert (recv_x_amin == j - rank_offset).sum().item() == (
all_topk_idx[j] == expert_id
).sum().item()
@@ -145,9 +146,9 @@ def test_main(
# Check combine correctness
for zero_copy in (False, True):
if zero_copy:
buffer.get_next_low_latency_combine_buffer(handle)[
:, :, :
] = simulated_gemm_x
buffer.get_next_low_latency_combine_buffer(handle)[:, :, :] = (
simulated_gemm_x
)
out = torch.empty(
(num_tokens, hidden), dtype=torch.bfloat16, device="cuda"
)
@@ -203,9 +204,9 @@ def test_main(
)
large_gemm_with_hook(hook) if return_recv_hook else None
if zero_copy:
buffer.get_next_low_latency_combine_buffer(handle)[
:, :, :
] = simulated_gemm_x
buffer.get_next_low_latency_combine_buffer(handle)[:, :, :] = (
simulated_gemm_x
)
combined_x, event, hook = buffer.low_latency_combine(
simulated_gemm_x,
topk_idx,
+1 -2
View File
@@ -238,8 +238,7 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase):
join_end = rank_offset + join_tp
if join_end > len(visible_devices):
raise RuntimeError(
f"Scale-up requires {join_end} visible GPUs, got "
f"{len(visible_devices)}"
f"Scale-up requires {join_end} visible GPUs, got {len(visible_devices)}"
)
env["CUDA_VISIBLE_DEVICES"] = ",".join(visible_devices[rank_offset:join_end])
base_joining_log = os.environ.get(
+2 -5
View File
@@ -16,7 +16,6 @@ from sglang.test.test_utils import CustomTestCase
class TestFlashinferDispatcher(CustomTestCase):
@classmethod
def setUpClass(cls):
server_args = ServerArgs(model_path="dummy")
@@ -125,8 +124,7 @@ class TestFlashinferDispatcher(CustomTestCase):
self.assertTrue(
torch.all(
received_hidden_states[
expected_source_rank
* num_tokens : (expected_source_rank + 1)
expected_source_rank * num_tokens : (expected_source_rank + 1)
* num_tokens
]
== 100.0 + expected_source_rank
@@ -229,8 +227,7 @@ class TestFlashinferDispatcher(CustomTestCase):
self.assertTrue(
torch.all(
received_hidden_states[
expected_source_rank
* num_tokens : (expected_source_rank + 1)
expected_source_rank * num_tokens : (expected_source_rank + 1)
* num_tokens
]
== 100.0 + expected_source_rank
@@ -242,7 +242,7 @@ class TestDisaggregationDecodeWithHiCache(DisaggregationHiCacheBase):
self.assertGreater(
cached_tokens,
previous_cached_tokens,
f"Turn {turn} should have more cached tokens than turn {turn-1}",
f"Turn {turn} should have more cached tokens than turn {turn - 1}",
)
# Update context and cached tokens for next iteration
@@ -380,14 +380,14 @@ def test_plan_then_io_cuda_graph_replay() -> None:
graph.replay()
torch.cuda.synchronize()
# Anchor slot table matches the synchronous layer-0 result.
assert torch.equal(
out.cpu(), ref_slots[s][0].cpu()
), f"slots differ at step {s}"
assert torch.equal(out.cpu(), ref_slots[s][0].cpu()), (
f"slots differ at step {s}"
)
# Every layer's device buffer stays bit-identical to synchronous swap-in.
for layer in range(_PIO_LAYERS):
assert torch.equal(
buf[layer].cpu(), ref_snap[s][layer].cpu()
), f"buffer differs at step {s}, layer {layer}"
assert torch.equal(buf[layer].cpu(), ref_snap[s][layer].cpu()), (
f"buffer differs at step {s}, layer {layer}"
)
if __name__ == "__main__":
@@ -53,7 +53,6 @@ MOCK_CHOICES_INPUT_DATA = {
class TestChoices(CustomTestCase):
def test_token_length_normalized(self):
"""Confirm 'antidisestablishmentarianism' is selected due to high confidences for
its later tokens resulting in highest token length normalized prompt logprob."""
@@ -47,6 +47,7 @@ json_jump_forward = (
+ r"""\}\n"""
)
# fmt: off
@sgl.function
def json_gen(s):
@@ -175,7 +175,7 @@ def get_k_and_s_triton():
end_time = time.perf_counter()
print(
f"_get_k_and_s_triton_kernel triton kernel infer time is {((end_time-start_time)*1000):.4f} ms\n"
f"_get_k_and_s_triton_kernel triton kernel infer time is {((end_time - start_time) * 1000):.4f} ms\n"
)
-1
View File
@@ -30,7 +30,6 @@ from sglang.test.test_utils import CustomTestCase, is_in_ci
class TestLoRABackend(CustomTestCase):
def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]):
for model_case in model_cases:
# If skip_long_prompt is True, filter out prompts longer than 1000 characters
-1
View File
@@ -44,7 +44,6 @@ TEST_CUDA_GRAPH_PADDING_PROMPTS = [
class TestLoRACudaGraph(CustomTestCase):
def _run_without_cuda_graph_on_model_cases(self, model_cases: List[LoRAModelCase]):
# Since we have already enabled CUDA graph by default in other lora tests,
# we only need to run lora tests without CUDA graph here.
-1
View File
@@ -9,7 +9,6 @@ from sglang.test.test_utils import CustomTestCase
class TestTorchNativeLoRABackend(CustomTestCase):
device = "cpu"
# set duplicate weights to test merging during prepare_lora_batch
@@ -74,7 +74,9 @@ class _FakeMHATokenToKVPool(_FakeKVCache):
self.v_head_dim = (
swa_v_head_dim
if swa_v_head_dim is not None
else v_head_dim if v_head_dim is not None else head_dim
else v_head_dim
if v_head_dim is not None
else head_dim
)
self._create_buffers()
+6 -7
View File
@@ -29,7 +29,6 @@ TORCH_DTYPES = [torch.float16]
class TestClipModels(unittest.TestCase):
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
@@ -61,12 +60,12 @@ class TestClipModels(unittest.TestCase):
)
print("text similarity diff", abs(text_similarity - 1))
print("image similarity diff", abs(image_similarity - 1))
assert torch.all(
abs(text_similarity - 1) < prefill_tolerance
), "embeddings are not all close"
assert torch.all(
abs(image_similarity - 1) < prefill_tolerance
), "embeddings are not all close"
assert torch.all(abs(text_similarity - 1) < prefill_tolerance), (
"embeddings are not all close"
)
assert torch.all(abs(image_similarity - 1) < prefill_tolerance), (
"embeddings are not all close"
)
def test_accuracy(self):
for model, prefill_tolerance in MODELS:
+6 -6
View File
@@ -64,16 +64,16 @@ class TestQmeQwenModels(CustomTestCase):
hf_text_embeddings.embed_logits[0], srt_text_embeddings.embed_logits[0]
)
print("texts similarity diff", abs(similarity - 1))
assert torch.all(
abs(similarity - 1) < prefill_tolerance
), "embeddings are not all close"
assert torch.all(abs(similarity - 1) < prefill_tolerance), (
"embeddings are not all close"
)
similarity = get_similarities(
hf_image_embeddings.embed_logits[0], srt_image_embeddings.embed_logits[0]
)
print("images similarity diff", abs(similarity - 1))
assert torch.all(
abs(similarity - 1) < prefill_tolerance
), "embeddings are not all close"
assert torch.all(abs(similarity - 1) < prefill_tolerance), (
"embeddings are not all close"
)
def test_accuracy(self):
for model, prefill_tolerance in MODELS:
+1 -1
View File
@@ -61,7 +61,7 @@ class TestKimiK2Thinking(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (Kimi-K2-Thinking)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (Kimi-K2-Thinking)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.95)
@@ -67,7 +67,7 @@ class TestMistralLarge3Basic(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (mistral-large-3)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (mistral-large-3)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.90)
@@ -79,7 +79,7 @@ class TestMistralLarge3Basic(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (mistral-large-3)\n" f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (mistral-large-3)\n{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 50)
+7 -7
View File
@@ -156,9 +156,9 @@ async def _stream_websocket_async(
async with websockets.connect(websocket_url) as websocket:
created = json.loads(await websocket.recv())
assert (
created.get("type") == "session.created"
), f"expected session.created, got {created!r}"
assert created.get("type") == "session.created", (
f"expected session.created, got {created!r}"
)
session_id = created["session"]["id"]
transcription_cfg = {"model": "qwen3-asr"}
@@ -200,9 +200,9 @@ async def _stream_websocket_async(
if t == "conversation.item.input_audio_transcription.delta":
deltas.append(resp["delta"])
elif t == "conversation.item.input_audio_transcription.completed":
assert (
"usage" in resp
), f"transcription.completed missing required usage field: {resp!r}"
assert "usage" in resp, (
f"transcription.completed missing required usage field: {resp!r}"
)
assert resp["usage"].get("type") == "duration", resp["usage"]
completed_msg.update(resp)
return
@@ -317,7 +317,7 @@ class TestQwen3ASRTranscription(CustomTestCase):
self.assertEqual(
results[0],
results[i],
f"Request {i+1} differs from first request",
f"Request {i + 1} differs from first request",
)
print(f"[Consistency] All 3 requests match: {results[0][:80]}...")
@@ -66,7 +66,7 @@ class TestVLMPiecewiseCudaGraph(CustomTestCase):
os.makedirs(output_path, exist_ok=True)
# -------- compose --model_args --------
model_args = f'model_version="{model_version}",' f"tp={tp}"
model_args = f'model_version="{model_version}",tp={tp}'
# -------- build command list --------
cmd = [
@@ -69,7 +69,7 @@ class TestVLMViTCudaGraph(CustomTestCase):
os.makedirs(output_path, exist_ok=True)
# -------- compose --model_args --------
model_args = f'model_version="{model_version}",' f"tp={tp}"
model_args = f'model_version="{model_version}",tp={tp}'
# -------- build command list --------
cmd = [
@@ -66,7 +66,7 @@ class TestVLMViTFlashinferCudnn(CustomTestCase):
os.makedirs(output_path, exist_ok=True)
# -------- compose --model_args --------
model_args = f'model_version="{model_version}",' f"tp={tp}"
model_args = f'model_version="{model_version}",tp={tp}'
# -------- build command list --------
cmd = [
@@ -250,12 +250,12 @@ class TestCacheReport(CustomTestCase):
)
# Verify cache hit for same salt
assert (
cached_tokens_1_second > cached_tokens_1_first
), "Should have cache hit with same cache_salt"
assert (
cached_tokens_1_second == prompt_tokens_1 - 1
), "Should cache all prompt tokens except the last one"
assert cached_tokens_1_second > cached_tokens_1_first, (
"Should have cache hit with same cache_salt"
)
assert cached_tokens_1_second == prompt_tokens_1 - 1, (
"Should cache all prompt tokens except the last one"
)
# Third request with different cache_salt "salt2" - should not get cache hit
response3 = self.client.chat.completions.create(
@@ -269,9 +269,9 @@ class TestCacheReport(CustomTestCase):
print(f"First request with salt2 - cached_tokens: {cached_tokens_2_first}")
# Verify no cache hit for different salt (should be similar to first request with salt1)
assert (
cached_tokens_2_first <= cached_tokens_1_first + self.min_cached
), "Different cache_salt should not share cache"
assert cached_tokens_2_first <= cached_tokens_1_first + self.min_cached, (
"Different cache_salt should not share cache"
)
# Fourth request with same cache_salt "salt2" - should now get cache hit
response4 = self.client.chat.completions.create(
@@ -285,9 +285,9 @@ class TestCacheReport(CustomTestCase):
print(f"Second request with salt2 - cached_tokens: {cached_tokens_2_second}")
# Verify cache hit for salt2
assert (
cached_tokens_2_second > cached_tokens_2_first
), "Should have cache hit with same cache_salt for salt2"
assert cached_tokens_2_second > cached_tokens_2_first, (
"Should have cache hit with same cache_salt for salt2"
)
if __name__ == "__main__":
@@ -23,7 +23,6 @@ from sglang.test.test_utils import (
class TestToolChoiceLlama32(CustomTestCase):
@classmethod
def setUpClass(cls):
# Mark flaky tests for this model
@@ -379,15 +378,15 @@ class TestToolChoiceLlama32(CustomTestCase):
# Update function name if present (first chunk)
if tool_call_delta.function and tool_call_delta.function.name:
tool_calls_by_index[tool_index]["function"][
"name"
] = tool_call_delta.function.name
tool_calls_by_index[tool_index]["function"]["name"] = (
tool_call_delta.function.name
)
# Accumulate arguments (all chunks)
if tool_call_delta.function and tool_call_delta.function.arguments:
tool_calls_by_index[tool_index]["function"][
"arguments"
] += tool_call_delta.function.arguments
tool_calls_by_index[tool_index]["function"]["arguments"] += (
tool_call_delta.function.arguments
)
self.assertGreater(len(tool_calls_by_index), 0)
@@ -17,7 +17,6 @@ from sglang.test.test_utils import (
class TestBenchOneBatch1GPU(CustomTestCase):
def test_bs1_small(self):
_, output_throughput, _ = run_bench_one_batch(
DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs-decode", "2"]
@@ -20,7 +20,6 @@ TORCH_DTYPES = [torch.float32]
class TestCrossEncoderModels(CustomTestCase):
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
@@ -55,9 +54,9 @@ class TestCrossEncoderModels(CustomTestCase):
for i in range(len(srt_scores)):
score_difference = abs(hf_scores[i] - srt_scores[i])
assert (
score_difference < score_tolerance
), "cross encoder scores are not all close"
assert score_difference < score_tolerance, (
"cross encoder scores are not all close"
)
def preprocess_prompts(self, prompt):
processed_prompts = []
@@ -37,7 +37,6 @@ sgl_to_st_ratio = []
class TestEncoderEmbeddingModels(CustomTestCase):
@classmethod
def setUpClass(cls):
mp.set_start_method("spawn", force=True)
@@ -115,9 +114,9 @@ class TestEncoderEmbeddingModels(CustomTestCase):
# print("similarity diff", abs(similarity - 1))
if len(truncated_prompts[i]) <= 1000:
assert torch.all(
abs(similarity - 1) < prefill_tolerance
), "embeddings are not all close"
assert torch.all(abs(similarity - 1) < prefill_tolerance), (
"embeddings are not all close"
)
def test_prefill_logits(self):
models_to_test = MODELS
+3 -4
View File
@@ -46,9 +46,9 @@ def native_per_token_group_quant_fp8(
quantized tensor along with the scaling factor used for quantization.
Note that only `torch.float8_e4m3fn` is supported for now.
"""
assert (
x.shape[-1] % group_size == 0
), "the last dimension of `x` cannot be divisible by `group_size`"
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` cannot be divisible by `group_size`"
)
assert x.is_contiguous(), "`x` is not contiguous"
finfo = torch.finfo(dtype)
@@ -343,7 +343,6 @@ def native_w8a8_block_fp8_matmul(A, B, As, Bs, block_size, output_dtype=torch.fl
class TestW8A8BlockFP8Matmul(CustomTestCase):
if not _is_cuda:
OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16]
M = [1, 7, 83, 512, 2048]
@@ -171,7 +171,6 @@ def block_quant_dequant(
class TestDeepGemmBlackwell(CustomTestCase):
if not _is_cuda:
OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16]
M = [1, 7, 83, 512, 2048]
@@ -67,7 +67,7 @@ class TestDeepseekV32FP4DP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (deepseek-v3-fp4)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
@@ -135,7 +135,7 @@ class TestDeepseekV32FP4TP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (deepseek-v3-fp4)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
@@ -70,7 +70,7 @@ class TestDeepseekV3FP4(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (deepseek-v3-fp4)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
@@ -83,7 +83,7 @@ class TestDeepseekV3FP4(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v3-fp4)\n" f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (deepseek-v3-fp4)\n{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 120)
@@ -46,7 +46,6 @@ class TestEvalFP8Accuracy(CustomTestCase):
class TestEvalFP8DynamicQuantAccuracy(CustomTestCase):
def _run_test(self, model, other_args, expected_score):
base_url = DEFAULT_URL_FOR_TEST
other_args = other_args or []
@@ -9,7 +9,6 @@ from sglang.test.test_utils import (
class TestNoChunkedPrefill(CustomTestCase):
def test_no_chunked_prefill(self):
run_mmlu_test(
disable_radix_cache=False, enable_mixed_chunk=False, chunked_prefill_size=-1
@@ -59,5 +59,4 @@ class TestEagle3Basic(EagleServerBase):
if __name__ == "__main__":
unittest.main()
+2 -2
View File
@@ -386,8 +386,8 @@ class TestPerformance(unittest.TestCase):
speedup = t_ref / t_new
dev_tag = "CUDA" if "cuda" in str(self.device) else "CPU"
print(
f" [{dev_tag}] B={batch_size:3d} S={seq_len:5d} img%={int(image_fraction*100):3d}%"
f" ref={t_ref*1e3:.2f}ms new={t_new*1e3:.2f}ms speedup={speedup:.2f}x"
f" [{dev_tag}] B={batch_size:3d} S={seq_len:5d} img%={int(image_fraction * 100):3d}%"
f" ref={t_ref * 1e3:.2f}ms new={t_new * 1e3:.2f}ms speedup={speedup:.2f}x"
)
self.assertGreaterEqual(
speedup,
+3 -3
View File
@@ -57,9 +57,9 @@ class TestFimCompletion(CustomTestCase):
assert response.id
assert response.created
assert response.object == "text_completion"
assert (
response.usage.prompt_tokens == num_prompt_tokens
), f"{response.usage.prompt_tokens} vs {num_prompt_tokens}"
assert response.usage.prompt_tokens == num_prompt_tokens, (
f"{response.usage.prompt_tokens} vs {num_prompt_tokens}"
)
assert response.usage.completion_tokens > 0
assert response.usage.total_tokens > 0
-1
View File
@@ -30,7 +30,6 @@ def _process_return(ret):
class TestGetWeightsByName(CustomTestCase):
def init_hf_model(self, model_name, tie_word_embeddings):
self.hf_model = AutoModelForCausalLM.from_pretrained(
model_name, torch_dtype="bfloat16", tie_word_embeddings=tie_word_embeddings
+6 -6
View File
@@ -104,15 +104,15 @@ def test_kda_target_verify_equivalence():
state_diff = (cached_state - decode_state).abs().max().item()
status = "OK" if state_diff < 1e-5 else "FAIL"
print(f" step={step} req={req_idx}: diff={state_diff:.6e} [{status}]")
assert (
state_diff < 1e-5
), f"Intermediate state mismatch at step={step}, req={req_idx}: {state_diff}"
assert state_diff < 1e-5, (
f"Intermediate state mismatch at step={step}, req={req_idx}: {state_diff}"
)
ssm_unchanged_diff = (ssm_states_verify - ssm_states_base).abs().max().item()
print(f"SSM state in-place change (should be 0): {ssm_unchanged_diff:.6e}")
assert (
ssm_unchanged_diff == 0.0
), f"target_verify modified ssm_states in-place! diff: {ssm_unchanged_diff}"
assert ssm_unchanged_diff == 0.0, (
f"target_verify modified ssm_states in-place! diff: {ssm_unchanged_diff}"
)
print("\nPASSED: KDA target_verify matches sequential decode!")
-1
View File
@@ -218,7 +218,6 @@ def generate_baseline(
class TestLogprobsDense(unittest.TestCase):
@classmethod
def setUpClass(cls):
"""Set up the test class - initialize the engine once for all tests."""
-1
View File
@@ -16,7 +16,6 @@ from sglang.test.test_utils import (
class TestEvalFP8ModelOptQuantAccuracy(CustomTestCase):
def _run_test(self, model, other_args, expected_score):
base_url = DEFAULT_URL_FOR_TEST
other_args = other_args or []
@@ -9,7 +9,6 @@ from sglang.test.test_utils import CustomTestCase
class TestDownloadFromModelScope(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = "iic/nlp_lstmcrf_word-segmentation_chinese-news"
-4
View File
@@ -155,7 +155,6 @@ def _cleanup(actor, pg):
@unittest.skipUnless(_has_ray, "ray is not installed")
@unittest.skipUnless(_NUM_GPUS >= 1, "requires at least 1 GPU")
class TestRayEngineOfflineTP1(unittest.TestCase):
@classmethod
def setUpClass(cls):
if not ray.is_initialized():
@@ -196,7 +195,6 @@ class TestRayEngineOfflineTP1(unittest.TestCase):
@unittest.skipUnless(_has_ray, "ray is not installed")
@unittest.skipUnless(_NUM_GPUS >= 2, "requires at least 2 GPUs")
class TestRayEngineOfflineTP2(unittest.TestCase):
@classmethod
def setUpClass(cls):
if not ray.is_initialized():
@@ -231,7 +229,6 @@ class TestRayEngineOfflineTP2(unittest.TestCase):
@unittest.skipUnless(_has_ray, "ray is not installed")
@unittest.skipUnless(_NUM_GPUS >= 2, "requires at least 2 GPUs")
class TestRayEngineOfflinePP2(unittest.TestCase):
@classmethod
def setUpClass(cls):
if not ray.is_initialized():
@@ -347,7 +344,6 @@ class TestRayEngineOfflineDPAttention(unittest.TestCase):
@unittest.skipUnless(_has_ray, "ray is not installed")
@unittest.skipUnless(_NUM_GPUS >= 1, "requires at least 1 GPU")
class TestRayEngineErrors(unittest.TestCase):
@classmethod
def setUpClass(cls):
if not ray.is_initialized():
+9 -9
View File
@@ -69,9 +69,9 @@ class TestSageMakerServer(CustomTestCase):
ret_num_top_logprobs = len(
response["choices"][0]["logprobs"]["content"][0]["top_logprobs"]
)
assert (
ret_num_top_logprobs == logprobs
), f"{ret_num_top_logprobs} vs {logprobs}"
assert ret_num_top_logprobs == logprobs, (
f"{ret_num_top_logprobs} vs {logprobs}"
)
assert len(response["choices"]) == parallel_sample_num
assert response["choices"][0]["message"]["role"] == "assistant"
@@ -155,18 +155,18 @@ class TestSageMakerServer(CustomTestCase):
.get("content")[0]
.get("top_logprobs")
)
assert (
ret_num_top_logprobs == logprobs
), f"{ret_num_top_logprobs} vs {logprobs}"
assert ret_num_top_logprobs == logprobs, (
f"{ret_num_top_logprobs} vs {logprobs}"
)
assert isinstance(data["content"], str)
assert line["id"]
assert line["created"]
for index in [i for i in range(parallel_sample_num)]:
assert not is_firsts.get(
index, True
), f"index {index} is not found in the response"
assert not is_firsts.get(index, True), (
f"index {index} is not found in the response"
)
def test_chat_completion(self):
for logprobs in [None, 5]:
-1
View File
@@ -25,7 +25,6 @@ def _make_req(rid, origin_input_text, origin_input_ids, sampling_params=None, **
class TestSchedulePolicy(CustomTestCase):
def setUp(self):
self.tree_cache = RadixCache.create_simulated()
@@ -5,7 +5,6 @@ from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTest
class TestSRTEngineWithQuantArgs(CustomTestCase):
def test_1_quantization_args(self):
# we only test fp8 because other methods are currently dependent on vllm. We can add other methods back to test after vllm dependency is resolved.
@@ -38,7 +38,6 @@ class TestTokenizerBatchEncode(unittest.TestCase):
"sglang.srt.utils.hf_transformers_utils.get_tokenizer"
) as mock_tokenizer,
):
mock_tokenizer.return_value = Mock(vocab_size=32000)
publish(self.server_args, role="tokenizer")
self.tokenizer_manager = TokenizerManager(self.server_args, self.port_args)
@@ -14,7 +14,6 @@ from sglang.test.test_utils import CustomTestCase
class TestTritonAttentionMLA(CustomTestCase):
def _set_all_seeds(self, seed):
"""Set all random seeds for reproducibility."""
random.seed(seed)
+1 -2
View File
@@ -212,8 +212,7 @@ class TestMiniCPMV2_6Logits(VisionLLMLogitsBase):
# per image
if len(pixel_b) != len(tgt_b):
raise ValueError(
"Inconsistent N lengths, found: "
f"{len(pixel_b)} vs {len(tgt_b)}"
f"Inconsistent N lengths, found: {len(pixel_b)} vs {len(tgt_b)}"
)
for pixel_n, tgt_n in zip(pixel_b, tgt_b):
pixel_values_flat += [pixel_n]
+2 -2
View File
@@ -266,7 +266,7 @@ def main():
print(
f"Daemon pp_rank={pp_rank} tp_rank={tp_rank} ready "
f"({time.time()-start:.0f}s)"
f"({time.time() - start:.0f}s)"
)
if error_found:
break
@@ -278,7 +278,7 @@ def main():
sys.exit(1)
print(
f"\nAll {total_ranks} daemons ready! Total load time: {time.time()-start:.1f}s"
f"\nAll {total_ranks} daemons ready! Total load time: {time.time() - start:.1f}s"
)
# Query config from daemon (pp_rank=0, tp_rank=0)
+2 -2
View File
@@ -97,14 +97,14 @@ class TestWhisperCudaGraph(CustomTestCase):
result = self._transcribe()
self.assertIn("text", result)
results.append(result["text"])
print(f"Request {i+1}: {result['text'][:80]}...")
print(f"Request {i + 1}: {result['text'][:80]}...")
# All transcriptions of the same audio should be identical
for i in range(1, len(results)):
self.assertEqual(
results[0],
results[i],
f"Transcription {i+1} differs from first transcription",
f"Transcription {i + 1} differs from first transcription",
)
def test_transcription_quality(self):
@@ -76,9 +76,9 @@ def main():
if rank == 0:
print(
f"world={world} owner={owner} shape=[{n_tok},{hidden}] "
f"(~{n_tok*hidden*2/1e6:.0f}MB) | all_ranks_bitwise_ok={bool(flags.item())} "
f"(~{n_tok * hidden * 2 / 1e6:.0f}MB) | all_ranks_bitwise_ok={bool(flags.item())} "
f"(A==truth={eq_truth} A==B={eq_ab}) | all_gather {ta:.3f}ms "
f"broadcast {tb:.3f}ms speedup {ta/tb:.2f}x",
f"broadcast {tb:.3f}ms speedup {ta / tb:.2f}x",
flush=True,
)
dist.destroy_process_group()
@@ -61,7 +61,7 @@ class TestDeepseekV32IndexTopkPattern(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n'
f'### test_gsm8k (deepseek-v32)\n{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.935)
@@ -108,7 +108,7 @@ class TestDeepseekV32IndexFreq(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n'
f'### test_gsm8k (deepseek-v32)\n{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.935)
@@ -246,9 +246,9 @@ class TestDeepSeekR1EvalAMD(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -98,8 +98,7 @@ class TestDeepseekV32DP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 DP MI325)\n"
f'{metrics["accuracy"]=:.3f}\n'
f'### test_gsm8k (deepseek-v32 DP MI325)\n{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
@@ -112,8 +111,7 @@ class TestDeepseekV32DP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 DP MI325)\n"
f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (deepseek-v32 DP MI325)\n{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 10)
@@ -182,9 +182,9 @@ class TestDeepSeekV32EvalAMD(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -99,8 +99,7 @@ class TestDeepseekV32TC(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 TC MI325)\n"
f'{metrics["accuracy"]=:.3f}\n'
f'### test_gsm8k (deepseek-v32 TC MI325)\n{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
@@ -113,8 +112,7 @@ class TestDeepseekV32TC(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 TC MI325)\n"
f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (deepseek-v32 TC MI325)\n{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 10)
@@ -172,9 +172,9 @@ class TestGLM51EvalAMD(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -80,8 +80,7 @@ class TestGLM51HiSparseEvalAMD(unittest.TestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (glm-5.1 hisparse mi30x)\n"
f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (glm-5.1 hisparse mi30x)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
@@ -182,9 +182,9 @@ class TestGLM5EvalAMD(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -184,9 +184,9 @@ class TestGptOssEvalAMD(unittest.TestCase):
for config in self.models:
with self.subTest(model=config.model_path):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {config.model_path}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -222,9 +222,9 @@ class TestGrokEvalAMD(unittest.TestCase):
for config in self.models:
with self.subTest(model=config.model_path):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {config.model_path}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -181,9 +181,9 @@ def check_model_scores(results):
line = f"| {model} | {tp_size} | {score:.3f} | {threshold_str} | {startup_str} | {eval_str} | {total_str} | {status} |\n"
summary += line
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print("SUMMARY - TP=2 Instruction Models (gsm8k)")
print(f"{'='*60}")
print(f"{'=' * 60}")
print(summary)
print(f"\n📊 Final Statistics:")
print(f" Passed: {passed_count}")
@@ -219,19 +219,19 @@ class TestNightlyGsm8KEval(unittest.TestCase):
all_results = []
total_test_start = time.time()
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print("AMD GSM8K Evaluation Test (TP=2 Instruction Models)")
print(f"{'='*60}")
print(f"{'=' * 60}")
print(f"Benchmark: gsm8k (chat completions)")
print(f"{'='*60}\n")
print(f"{'=' * 60}\n")
for model_group, is_fp8, is_tp2 in self.model_groups:
for model in model_group:
with self.subTest(model=model):
tp_size = 2 if is_tp2 else 1
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {model} (TP={tp_size}, FP8={is_fp8})")
print(f"{'='*60}")
print(f"{'=' * 60}")
model_start = time.time()
startup_time = None
@@ -326,7 +326,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
# Check all scores after collecting all results
check_model_scores(all_results)
print(
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time / 60:.1f} min)"
)
@@ -179,9 +179,9 @@ class TestMiniMaxM25EvalAMD(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -179,9 +179,9 @@ class TestMiniMaxM27EvalAMD(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -172,14 +172,14 @@ class TestNightlyVLMMmmuEvalAMD(unittest.TestCase):
all_results = []
total_test_start = time.time()
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print("AMD VLM MMMU Evaluation Test")
print(f"{'='*60}")
print(f"{'=' * 60}")
print(f"Benchmark: MMMU (100 samples)")
print(f"Models to test: {len(self.models)}")
for m in self.models:
print(f" - {m['model_path']} (TP={m['tp_size']})")
print(f"{'='*60}\n")
print(f"{'=' * 60}\n")
for model_config in self.models:
model_path = model_config["model_path"]
@@ -189,9 +189,9 @@ class TestNightlyVLMMmmuEvalAMD(unittest.TestCase):
error_message = None
with self.subTest(model=model_path):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {model_path} (TP={tp_size})")
print(f"{'='*60}")
print(f"{'=' * 60}")
model_start = time.time()
startup_time = None
@@ -358,21 +358,21 @@ class TestNightlyVLMMmmuEvalAMD(unittest.TestCase):
summary += f"| {model} | {tp_size} | {score_str} | {threshold:.2f} | {startup_str} | {eval_str} | {total_str} | {status} |\n"
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print("SUMMARY - AMD VLM MMMU Evaluation")
print(f"{'='*60}")
print(f"{'=' * 60}")
print(summary)
print(f"\n📊 Final Statistics:")
print(f" Passed: {passed_count}")
print(f" Failed: {failed_count}")
print(
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
f"\n⏱️ Total test runtime: {total_test_time:.1f}s ({total_test_time / 60:.1f} min)"
)
if is_in_ci():
write_github_step_summary(
f"### TestNightlyVLMMmmuEvalAMD\n{summary}\n\n"
f"**Total Runtime:** {total_test_time:.1f}s ({total_test_time/60:.1f} min)"
f"**Total Runtime:** {total_test_time:.1f}s ({total_test_time / 60:.1f} min)"
)
if failed_models:
@@ -182,9 +182,9 @@ class TestDeepSeekR1EvalMI35x(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -180,9 +180,9 @@ class TestDeepSeekR1MXFP4ArFusionEvalMI35x(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -178,9 +178,9 @@ class TestDeepSeekR1MXFP4EvalMI35x(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -181,9 +181,9 @@ class TestDeepSeekR1MXFP4KvFp8EvalMI35x(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -89,8 +89,7 @@ class TestDeepseekV32DP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 DP MI35x)\n"
f'{metrics["accuracy"]=:.3f}\n'
f'### test_gsm8k (deepseek-v32 DP MI35x)\n{metrics["accuracy"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], GSM8K_ACCURACY_THRESHOLD)
@@ -103,8 +102,7 @@ class TestDeepseekV32DP(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (deepseek-v32 DP MI35x)\n"
f"{speed=:.2f} token/s\n"
f"### test_bs_1_speed (deepseek-v32 DP MI35x)\n{speed=:.2f} token/s\n"
)
self.assertGreater(speed, 10)
@@ -183,9 +183,9 @@ class TestDeepSeekV32EvalMI35x(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -172,9 +172,9 @@ class TestGLM51EvalMI35x(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -80,8 +80,7 @@ class TestGLM51HiSparseEvalMI35x(unittest.TestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (glm-5.1 hisparse mi35x)\n"
f'{metrics["score"]=:.3f}\n'
f'### test_gsm8k (glm-5.1 hisparse mi35x)\n{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["score"], 0.93)
@@ -182,9 +182,9 @@ class TestGLM5EvalMI35x(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -185,9 +185,9 @@ class TestGLM5MXFP4EvalMI35x(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -196,9 +196,9 @@ class TestGptOssEvalMI35x(unittest.TestCase):
for config in self.models:
with self.subTest(model=config.model_path):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {config.model_path}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -185,9 +185,9 @@ class TestGptOssW4A8Mxfp4EvalMI35x(unittest.TestCase):
for config in self.models:
with self.subTest(model=config.model_path):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {config.model_path}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
@@ -132,9 +132,9 @@ class TestKimiK25AiterMlaEvalMI35x(unittest.TestCase):
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"\n{'=' * 60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
print(f"{'=' * 60}")
env = os.environ.copy()
for key, value in config.env_vars.items():

Some files were not shown because too many files have changed in this diff Show More