[diffusion] nightly: measure every framework with one client end-to-end methodology (#41689)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
4b3f63ec70
commit
bd66ce343e
@@ -1,6 +1,9 @@
|
||||
import importlib.util
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[5]
|
||||
|
||||
|
||||
@@ -22,7 +25,9 @@ dashboard = _load_script(
|
||||
)
|
||||
|
||||
|
||||
def test_sglang_server_warmup_matches_measured_shape():
|
||||
def test_sglang_launch_is_exactly_the_recipe():
|
||||
# An outside benchmark launches the published recipe as-is, so the harness
|
||||
# must not add shape-specific warmup flags of its own.
|
||||
case = {
|
||||
"model": "example/model",
|
||||
"num_gpus": 2,
|
||||
@@ -36,10 +41,75 @@ def test_sglang_server_warmup_matches_measured_shape():
|
||||
30000,
|
||||
)
|
||||
|
||||
resolution_index = command.index("--warmup-resolutions")
|
||||
frame_index = command.index("--warmup-num-frames")
|
||||
assert command[resolution_index + 1] == "768x512"
|
||||
assert command[frame_index + 1] == "121"
|
||||
assert "--warmup-resolutions" not in command
|
||||
assert "--warmup-num-frames" not in command
|
||||
assert command[-4:] == ["--warmup-mode", "server", "--tp-size", "2"]
|
||||
|
||||
|
||||
def test_requests_of_a_run_never_repeat_an_input(monkeypatch):
|
||||
# sglang reuses the encodings of inputs it has seen, so a repeated prompt
|
||||
# or image would time that reuse instead of the encoders
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (1024, 704), (200, 100, 50)).save(buffer, format="PNG")
|
||||
monkeypatch.setattr(runner, "_cached_ref_image", buffer.getvalue())
|
||||
monkeypatch.setattr(runner, "_ref_image_variants", {})
|
||||
case = {"prompt": "Make the cat wear a red hat"}
|
||||
|
||||
requests = [runner._request_case(case, index) for index in range(10)]
|
||||
prompts = [request["prompt"] for request in requests]
|
||||
downscaled = [
|
||||
Image.open(
|
||||
io.BytesIO(runner._get_ref_image_variant({}, request["input_variant"]))
|
||||
)
|
||||
.resize((256, 176), Image.BICUBIC)
|
||||
.tobytes()
|
||||
for request in requests
|
||||
]
|
||||
|
||||
assert len(set(prompts)) == len(set(downscaled)) == 10
|
||||
assert len({len(prompt) for prompt in prompts}) == 1
|
||||
assert case == {"prompt": "Make the cat wear a red hat"}
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload=None, content=b""):
|
||||
self._payload = payload
|
||||
self.content = content
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
def test_sglang_video_latency_includes_polling_and_download(monkeypatch):
|
||||
statuses = iter(["queued", "in_progress", "completed"])
|
||||
fetched = []
|
||||
|
||||
class FakeRequests:
|
||||
@staticmethod
|
||||
def post(url, json=None, timeout=None):
|
||||
return _FakeResponse({"id": "job-1"})
|
||||
|
||||
@staticmethod
|
||||
def get(url, timeout=None):
|
||||
fetched.append(url)
|
||||
if url.endswith("/content"):
|
||||
return _FakeResponse(content=b"mp4-bytes")
|
||||
return _FakeResponse({"status": next(statuses)})
|
||||
|
||||
sleeps = []
|
||||
monkeypatch.setattr(runner, "requests", FakeRequests)
|
||||
monkeypatch.setattr(runner.time, "sleep", sleeps.append)
|
||||
case = {"model": "example/model", "prompt": "p", "width": 64, "height": 64}
|
||||
|
||||
latency = runner.send_video_request_sglang("http://host", case)
|
||||
|
||||
assert latency >= 0
|
||||
assert fetched[-1] == "http://host/v1/videos/job-1/content"
|
||||
assert fetched.count("http://host/v1/videos/job-1") == 3
|
||||
assert sleeps == [runner.POLL_INTERVAL_S] * 2
|
||||
|
||||
|
||||
def test_explicit_server_warmup_shape_is_preserved():
|
||||
@@ -67,6 +137,26 @@ def test_explicit_server_warmup_shape_is_preserved():
|
||||
assert command[command.index("--warmup-num-frames") + 1] == "25"
|
||||
|
||||
|
||||
def test_health_wait_fails_fast_when_the_server_exits(monkeypatch):
|
||||
class ExitedProc:
|
||||
returncode = 1
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
def unreachable(*args, **kwargs):
|
||||
raise runner.requests.exceptions.ConnectionError("refused")
|
||||
|
||||
monkeypatch.setattr(runner.requests, "get", unreachable)
|
||||
monkeypatch.setattr(runner.time, "sleep", lambda _: None)
|
||||
try:
|
||||
runner.wait_for_health("http://host:1", proc=ExitedProc(), timeout=3600)
|
||||
except RuntimeError as e:
|
||||
assert "exited with code 1" in str(e)
|
||||
else:
|
||||
raise AssertionError("wait_for_health should fail once the server exits")
|
||||
|
||||
|
||||
def test_perf_dump_summary_uses_medians():
|
||||
perf_dumps = [
|
||||
{
|
||||
@@ -109,11 +199,14 @@ def test_dashboard_uses_historical_median_and_shows_server_breakdown():
|
||||
current = {
|
||||
"timestamp": "2026-09-04T00:00:00+00:00",
|
||||
"commit_sha": "abcdef123456",
|
||||
"methodology": "client-e2e-v1",
|
||||
"warmup_requests": 1,
|
||||
"results": [
|
||||
{
|
||||
"case_id": "example",
|
||||
"framework": "sglang",
|
||||
"model": "example/model",
|
||||
"first_request_latency_s": 25.0,
|
||||
"latency_s": 10.4,
|
||||
"latency_samples_s": [10.3, 10.4, 10.5],
|
||||
"measurement_count": 3,
|
||||
@@ -152,6 +245,7 @@ def test_dashboard_uses_historical_median_and_shows_server_breakdown():
|
||||
assert alerts == []
|
||||
assert "Incomplete Server Telemetry" in markdown
|
||||
assert "**model**: 2/3 server samples available" in markdown
|
||||
assert "| 3 | 2/3 | **10.40** |" in markdown
|
||||
assert "| 3 | 2/3 | 25.00 | **10.40** |" in markdown
|
||||
assert "Methodology `client-e2e-v1`" in markdown
|
||||
assert "## SGLang Server-Side Breakdown" in markdown
|
||||
assert "| model | 10.00 | 0.10 | 9.80 | 0.10 | 196.00 |" in markdown
|
||||
|
||||
Reference in New Issue
Block a user