From bb9a820f09f97480bfc6d07564fb2b691b8857a3 Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 2 Oct 2026 21:44:10 +0800 Subject: [PATCH] [diffusion] chore: make perf dump one writer per replica with no first-request NCCL setup (#41711) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../runtime/managers/gpu_worker.py | 40 +++++++++----- .../runtime/utils/perf_logger.py | 5 +- .../test/unit/test_perf_dump_write.py | 55 +++++++++++++++++++ .../test/unit/test_performance_metrics.py | 21 +++++-- 4 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_perf_dump_write.py diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index 0ce719e3ab..cce60a69c9 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -758,18 +758,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): if not req.is_warmup: PerformanceLogger.log_request_summary(metrics=output_batch.metrics) - # dump per-request perf report to the server-mode file path. - if ( - req.perf_dump_path is not None - and not req.is_warmup - and output_batch.metrics is not None - ): - PerformanceLogger.dump_benchmark_report( - file_path=req.perf_dump_path, - metrics=output_batch.metrics, - meta={"model": self.server_args.model_path}, - tag="server_perf_dump", - ) + self._dump_perf_report(req, output_batch) except Exception as e: if propagate_forward_errors and forward_failed: if isinstance(e, StopIteration): @@ -1104,6 +1093,23 @@ class GPUWorker(GPUWorkerPostTrainingMixin): if self.is_output_rank: output_batch.peak_memory_mb = snapshot.peak_reserved_mb + def _dump_perf_report(self, req: Req, output_batch: OutputBatch) -> None: + """Write the per-request perf report to the server-mode file path.""" + # one writer per replica, or ranks sharing the path clobber each other + if ( + req.perf_dump_path is None + or req.is_warmup + or output_batch.metrics is None + or not self.is_output_rank + ): + return + PerformanceLogger.dump_benchmark_report( + file_path=req.perf_dump_path, + metrics=output_batch.metrics, + meta={"model": self.server_args.model_path}, + tag="server_perf_dump", + ) + def _record_replica_peak_memory(self, output_metrics: list[Any]) -> None: """Record replica-wide loading and runtime allocator peaks.""" if not current_platform.is_cuda(): @@ -1118,9 +1124,15 @@ class GPUWorker(GPUWorkerPostTrainingMixin): self._runtime_peak_allocated_mb, ], dtype=torch.float64, - device=current_platform.get_device(self.local_rank), ) - peaks = get_replica_group().all_reduce(peaks, op=torch.distributed.ReduceOp.MAX) + replica = get_replica_group() + if replica.world_size > 1: + # Host counters over gloo: the device group's first collective + # sets up NCCL connections, which cost the first dumped request + # 0.15 s of latency. + torch.distributed.all_reduce( + peaks, op=torch.distributed.ReduceOp.MAX, group=replica.cpu_group + ) if not self.is_output_rank: return diff --git a/python/sglang/multimodal_gen/runtime/utils/perf_logger.py b/python/sglang/multimodal_gen/runtime/utils/perf_logger.py index 3739bda967..2c103897a4 100644 --- a/python/sglang/multimodal_gen/runtime/utils/perf_logger.py +++ b/python/sglang/multimodal_gen/runtime/utils/perf_logger.py @@ -445,8 +445,11 @@ class PerformanceLogger: try: abs_path = os.path.abspath(file_path) os.makedirs(os.path.dirname(abs_path), exist_ok=True) - with open(abs_path, "w", encoding="utf-8") as f: + # readers poll for this file: never let them see a partial write + tmp_path = f"{abs_path}.{os.getpid()}.tmp" + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(report, f, indent=2) + os.replace(tmp_path, abs_path) logger.info(f"Metrics dumped to: {CYAN}{abs_path}{RESET}") except IOError as e: logger.error(f"Failed to dump metrics to {abs_path}: {e}") diff --git a/python/sglang/multimodal_gen/test/unit/test_perf_dump_write.py b/python/sglang/multimodal_gen/test/unit/test_perf_dump_write.py new file mode 100644 index 0000000000..d7b2e62cb5 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_perf_dump_write.py @@ -0,0 +1,55 @@ +import json +from types import SimpleNamespace + +import pytest + +from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker +from sglang.multimodal_gen.runtime.utils import perf_logger as perf_logger_module +from sglang.multimodal_gen.runtime.utils.perf_logger import PerformanceLogger + + +def _metrics(request_id: str) -> SimpleNamespace: + return SimpleNamespace( + request_id=request_id, + stages={"DenoisingStage": 1.0}, + steps=[1.0], + memory_snapshots={}, + total_duration_ms=1.0, + ) + + +def test_reader_during_a_dump_sees_the_previous_complete_report(tmp_path, monkeypatch): + # clients poll this path; writing it in place exposed a truncated file + path = tmp_path / "perf.json" + monkeypatch.setattr(perf_logger_module, "get_git_commit_hash", lambda: "test") + PerformanceLogger.dump_benchmark_report(str(path), _metrics("first")) + + seen_mid_write = [] + real_dump = json.dump + + def dump_while_reading(obj, fp, **kwargs): + seen_mid_write.append(path.read_text()) + real_dump(obj, fp, **kwargs) + + monkeypatch.setattr(perf_logger_module.json, "dump", dump_while_reading) + PerformanceLogger.dump_benchmark_report(str(path), _metrics("second")) + + assert [json.loads(text)["request_id"] for text in seen_mid_write] == ["first"] + assert json.loads(path.read_text())["request_id"] == "second" + assert sorted(p.name for p in tmp_path.iterdir()) == ["perf.json"] + + +@pytest.mark.parametrize("is_output_rank", [True, False]) +def test_only_the_output_rank_writes_the_report(tmp_path, monkeypatch, is_output_rank): + monkeypatch.setattr(perf_logger_module, "get_git_commit_hash", lambda: "test") + worker = GPUWorker.__new__(GPUWorker) + worker.is_output_rank = is_output_rank + worker.server_args = SimpleNamespace(model_path="model") + path = tmp_path / "perf.json" + + worker._dump_perf_report( + SimpleNamespace(perf_dump_path=str(path), is_warmup=False), + SimpleNamespace(metrics=_metrics("request")), + ) + + assert path.exists() == is_output_rank diff --git a/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py b/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py index 2996546918..249ea9d81f 100644 --- a/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py +++ b/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py @@ -186,10 +186,19 @@ def test_worker_records_replica_load_and_runtime_peaks(): worker._runtime_peak_allocated_mb = 0.0 output = OutputBatch() metrics = RequestMetrics("request") - replica_group = Mock() - replica_group.all_reduce.return_value = torch.tensor( - [5120.0, 3584.0, 6144.0, 3500.0, 2560.0], dtype=torch.float64 - ) + replica_group = SimpleNamespace(world_size=2, cpu_group=object()) + reduced = [] + + def all_reduce(tensor, op, group): + # five host counters over the gloo group, never a device collective + assert tensor.device.type == "cpu" + assert op == torch.distributed.ReduceOp.MAX + assert group is replica_group.cpu_group + reduced.append(tensor.tolist()) + tensor.copy_( + torch.tensor([5120.0, 3584.0, 6144.0, 3500.0, 2560.0], dtype=torch.float64) + ) + snapshots = [ MemorySnapshot(0.0, 0.0, 2048.0, 3072.0), MemorySnapshot(0.0, 0.0, 2048.0, 3072.0), @@ -205,6 +214,9 @@ def test_worker_records_replica_load_and_runtime_peaks(): patch.object( gpu_worker_module, "get_replica_group", return_value=replica_group ), + patch.object( + gpu_worker_module.torch.distributed, "all_reduce", side_effect=all_reduce + ), ): worker._record_output_peak_memory(output) worker._record_replica_peak_memory([metrics]) @@ -218,6 +230,7 @@ def test_worker_records_replica_load_and_runtime_peaks(): assert metrics.memory_snapshots["load_peak"].peak_allocated_mb == 3500.0 assert metrics.memory_snapshots["runtime_peak"].peak_allocated_mb == 2560.0 assert metrics.memory_snapshots["warmup_peak"].peak_reserved_mb == 6144.0 + assert reduced == [[4096.0, 3072.0, 0.0, 3000.0, 2048.0]] def test_server_warmup_preserves_peak_after_managed_stage_timeline():