[diffusion] fix: fix concurrent server perf dump writes across ranks (#41541)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-06 09:29:51 +08:00
committed by GitHub
co-authored by Mick Qian
parent 36b216e2ee
commit 21c9bbdf2c
@@ -67,6 +67,59 @@ def _perf_record(memory_snapshots: dict[str, dict]) -> RequestPerfRecord:
)
@pytest.mark.parametrize("is_output_rank", [False, True])
@pytest.mark.parametrize("is_warmup", [False, True])
def test_worker_perf_dump_has_one_writer_per_replica(
is_output_rank, is_warmup, monkeypatch, tmp_path
):
worker = GPUWorker.__new__(GPUWorker)
worker.is_output_rank = is_output_rank
worker.server_args = SimpleNamespace(model_path="test-model")
worker._realtime_sessions = SimpleNamespace(attach=Mock())
worker._release_warmup_pool = Mock()
worker._materialize_output_transport = Mock()
worker._record_output_peak_memory = Mock()
worker._record_replica_peak_memory = Mock()
monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
monkeypatch.setattr(perf_logger_module, "get_git_commit_hash", lambda: "test")
monkeypatch.setattr(PerformanceLogger, "log_request_summary", Mock())
path = tmp_path / "perf.json"
path.write_text('{"writer": "output-rank"}')
metrics = RequestMetrics("request")
output = OutputBatch(metrics=metrics)
req = SimpleNamespace(
request_id="request",
is_warmup=is_warmup,
extra={},
suppress_logs=True,
perf_dump_path=str(path),
)
result = worker._execute_forward_common(
req,
forward_fn=lambda: output,
log_reqs=[],
return_req=False,
save_output_paths=Mock(),
error_context="test",
)
assert result is output
assert result.error is None
report = json.loads(path.read_text())
if is_output_rank and not is_warmup:
assert report["request_id"] == "request"
assert report["tag"] == "server_perf_dump"
assert report["meta"] == {"model": "test-model"}
else:
assert report == {"writer": "output-rank"}
# non-output ranks must still participate in the replica's memory reduction
if is_warmup:
worker._record_replica_peak_memory.assert_not_called()
else:
worker._record_replica_peak_memory.assert_called_once_with([metrics])
def test_request_metrics_attributes_steps_and_iterations_to_active_stage():
metrics = RequestMetrics("request")
metrics.active_stage_name = "ShapeStage"