From 21c9bbdf2c69d2fac0195c8dd305f4f6623d4af1 Mon Sep 17 00:00:00 2001 From: Mick Date: Tue, 6 Oct 2026 09:29:51 +0800 Subject: [PATCH] [diffusion] fix: fix concurrent server perf dump writes across ranks (#41541) Co-authored-by: Mick Qian --- .../test/unit/test_performance_metrics.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py b/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py index e88358b04a..6ebd03de24 100644 --- a/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py +++ b/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py @@ -67,6 +67,59 @@ def _perf_record(memory_snapshots: dict[str, dict]) -> RequestPerfRecord: ) +@pytest.mark.parametrize("is_output_rank", [False, True]) +@pytest.mark.parametrize("is_warmup", [False, True]) +def test_worker_perf_dump_has_one_writer_per_replica( + is_output_rank, is_warmup, monkeypatch, tmp_path +): + worker = GPUWorker.__new__(GPUWorker) + worker.is_output_rank = is_output_rank + worker.server_args = SimpleNamespace(model_path="test-model") + worker._realtime_sessions = SimpleNamespace(attach=Mock()) + worker._release_warmup_pool = Mock() + worker._materialize_output_transport = Mock() + worker._record_output_peak_memory = Mock() + worker._record_replica_peak_memory = Mock() + monkeypatch.setattr(current_platform, "is_cpu", lambda: True) + monkeypatch.setattr(perf_logger_module, "get_git_commit_hash", lambda: "test") + monkeypatch.setattr(PerformanceLogger, "log_request_summary", Mock()) + + path = tmp_path / "perf.json" + path.write_text('{"writer": "output-rank"}') + metrics = RequestMetrics("request") + output = OutputBatch(metrics=metrics) + req = SimpleNamespace( + request_id="request", + is_warmup=is_warmup, + extra={}, + suppress_logs=True, + perf_dump_path=str(path), + ) + result = worker._execute_forward_common( + req, + forward_fn=lambda: output, + log_reqs=[], + return_req=False, + save_output_paths=Mock(), + error_context="test", + ) + + assert result is output + assert result.error is None + report = json.loads(path.read_text()) + if is_output_rank and not is_warmup: + assert report["request_id"] == "request" + assert report["tag"] == "server_perf_dump" + assert report["meta"] == {"model": "test-model"} + else: + assert report == {"writer": "output-rank"} + # non-output ranks must still participate in the replica's memory reduction + if is_warmup: + worker._record_replica_peak_memory.assert_not_called() + else: + worker._record_replica_peak_memory.assert_called_once_with([metrics]) + + def test_request_metrics_attributes_steps_and_iterations_to_active_stage(): metrics = RequestMetrics("request") metrics.active_stage_name = "ShapeStage"