[diffusion] fix: fix concurrent server perf dump writes across ranks (#41541)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -67,6 +67,59 @@ def _perf_record(memory_snapshots: dict[str, dict]) -> RequestPerfRecord:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_output_rank", [False, True])
|
||||
@pytest.mark.parametrize("is_warmup", [False, True])
|
||||
def test_worker_perf_dump_has_one_writer_per_replica(
|
||||
is_output_rank, is_warmup, monkeypatch, tmp_path
|
||||
):
|
||||
worker = GPUWorker.__new__(GPUWorker)
|
||||
worker.is_output_rank = is_output_rank
|
||||
worker.server_args = SimpleNamespace(model_path="test-model")
|
||||
worker._realtime_sessions = SimpleNamespace(attach=Mock())
|
||||
worker._release_warmup_pool = Mock()
|
||||
worker._materialize_output_transport = Mock()
|
||||
worker._record_output_peak_memory = Mock()
|
||||
worker._record_replica_peak_memory = Mock()
|
||||
monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
|
||||
monkeypatch.setattr(perf_logger_module, "get_git_commit_hash", lambda: "test")
|
||||
monkeypatch.setattr(PerformanceLogger, "log_request_summary", Mock())
|
||||
|
||||
path = tmp_path / "perf.json"
|
||||
path.write_text('{"writer": "output-rank"}')
|
||||
metrics = RequestMetrics("request")
|
||||
output = OutputBatch(metrics=metrics)
|
||||
req = SimpleNamespace(
|
||||
request_id="request",
|
||||
is_warmup=is_warmup,
|
||||
extra={},
|
||||
suppress_logs=True,
|
||||
perf_dump_path=str(path),
|
||||
)
|
||||
result = worker._execute_forward_common(
|
||||
req,
|
||||
forward_fn=lambda: output,
|
||||
log_reqs=[],
|
||||
return_req=False,
|
||||
save_output_paths=Mock(),
|
||||
error_context="test",
|
||||
)
|
||||
|
||||
assert result is output
|
||||
assert result.error is None
|
||||
report = json.loads(path.read_text())
|
||||
if is_output_rank and not is_warmup:
|
||||
assert report["request_id"] == "request"
|
||||
assert report["tag"] == "server_perf_dump"
|
||||
assert report["meta"] == {"model": "test-model"}
|
||||
else:
|
||||
assert report == {"writer": "output-rank"}
|
||||
# non-output ranks must still participate in the replica's memory reduction
|
||||
if is_warmup:
|
||||
worker._record_replica_peak_memory.assert_not_called()
|
||||
else:
|
||||
worker._record_replica_peak_memory.assert_called_once_with([metrics])
|
||||
|
||||
|
||||
def test_request_metrics_attributes_steps_and_iterations_to_active_stage():
|
||||
metrics = RequestMetrics("request")
|
||||
metrics.active_stage_name = "ShapeStage"
|
||||
|
||||
Reference in New Issue
Block a user