[diffusion] chore: make perf dump one writer per replica with no first-request NCCL setup (#41711)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-02 21:44:10 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 2cd92d10b9
commit bb9a820f09
4 changed files with 102 additions and 19 deletions
@@ -758,18 +758,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
if not req.is_warmup:
PerformanceLogger.log_request_summary(metrics=output_batch.metrics)
# dump per-request perf report to the server-mode file path.
if (
req.perf_dump_path is not None
and not req.is_warmup
and output_batch.metrics is not None
):
PerformanceLogger.dump_benchmark_report(
file_path=req.perf_dump_path,
metrics=output_batch.metrics,
meta={"model": self.server_args.model_path},
tag="server_perf_dump",
)
self._dump_perf_report(req, output_batch)
except Exception as e:
if propagate_forward_errors and forward_failed:
if isinstance(e, StopIteration):
@@ -1104,6 +1093,23 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
if self.is_output_rank:
output_batch.peak_memory_mb = snapshot.peak_reserved_mb
def _dump_perf_report(self, req: Req, output_batch: OutputBatch) -> None:
"""Write the per-request perf report to the server-mode file path."""
# one writer per replica, or ranks sharing the path clobber each other
if (
req.perf_dump_path is None
or req.is_warmup
or output_batch.metrics is None
or not self.is_output_rank
):
return
PerformanceLogger.dump_benchmark_report(
file_path=req.perf_dump_path,
metrics=output_batch.metrics,
meta={"model": self.server_args.model_path},
tag="server_perf_dump",
)
def _record_replica_peak_memory(self, output_metrics: list[Any]) -> None:
"""Record replica-wide loading and runtime allocator peaks."""
if not current_platform.is_cuda():
@@ -1118,9 +1124,15 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
self._runtime_peak_allocated_mb,
],
dtype=torch.float64,
device=current_platform.get_device(self.local_rank),
)
peaks = get_replica_group().all_reduce(peaks, op=torch.distributed.ReduceOp.MAX)
replica = get_replica_group()
if replica.world_size > 1:
# Host counters over gloo: the device group's first collective
# sets up NCCL connections, which cost the first dumped request
# 0.15 s of latency.
torch.distributed.all_reduce(
peaks, op=torch.distributed.ReduceOp.MAX, group=replica.cpu_group
)
if not self.is_output_rank:
return
@@ -445,8 +445,11 @@ class PerformanceLogger:
try:
abs_path = os.path.abspath(file_path)
os.makedirs(os.path.dirname(abs_path), exist_ok=True)
with open(abs_path, "w", encoding="utf-8") as f:
# readers poll for this file: never let them see a partial write
tmp_path = f"{abs_path}.{os.getpid()}.tmp"
with open(tmp_path, "w", encoding="utf-8") as f:
json.dump(report, f, indent=2)
os.replace(tmp_path, abs_path)
logger.info(f"Metrics dumped to: {CYAN}{abs_path}{RESET}")
except IOError as e:
logger.error(f"Failed to dump metrics to {abs_path}: {e}")
@@ -0,0 +1,55 @@
import json
from types import SimpleNamespace
import pytest
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
from sglang.multimodal_gen.runtime.utils import perf_logger as perf_logger_module
from sglang.multimodal_gen.runtime.utils.perf_logger import PerformanceLogger
def _metrics(request_id: str) -> SimpleNamespace:
return SimpleNamespace(
request_id=request_id,
stages={"DenoisingStage": 1.0},
steps=[1.0],
memory_snapshots={},
total_duration_ms=1.0,
)
def test_reader_during_a_dump_sees_the_previous_complete_report(tmp_path, monkeypatch):
# clients poll this path; writing it in place exposed a truncated file
path = tmp_path / "perf.json"
monkeypatch.setattr(perf_logger_module, "get_git_commit_hash", lambda: "test")
PerformanceLogger.dump_benchmark_report(str(path), _metrics("first"))
seen_mid_write = []
real_dump = json.dump
def dump_while_reading(obj, fp, **kwargs):
seen_mid_write.append(path.read_text())
real_dump(obj, fp, **kwargs)
monkeypatch.setattr(perf_logger_module.json, "dump", dump_while_reading)
PerformanceLogger.dump_benchmark_report(str(path), _metrics("second"))
assert [json.loads(text)["request_id"] for text in seen_mid_write] == ["first"]
assert json.loads(path.read_text())["request_id"] == "second"
assert sorted(p.name for p in tmp_path.iterdir()) == ["perf.json"]
@pytest.mark.parametrize("is_output_rank", [True, False])
def test_only_the_output_rank_writes_the_report(tmp_path, monkeypatch, is_output_rank):
monkeypatch.setattr(perf_logger_module, "get_git_commit_hash", lambda: "test")
worker = GPUWorker.__new__(GPUWorker)
worker.is_output_rank = is_output_rank
worker.server_args = SimpleNamespace(model_path="model")
path = tmp_path / "perf.json"
worker._dump_perf_report(
SimpleNamespace(perf_dump_path=str(path), is_warmup=False),
SimpleNamespace(metrics=_metrics("request")),
)
assert path.exists() == is_output_rank
@@ -186,10 +186,19 @@ def test_worker_records_replica_load_and_runtime_peaks():
worker._runtime_peak_allocated_mb = 0.0
output = OutputBatch()
metrics = RequestMetrics("request")
replica_group = Mock()
replica_group.all_reduce.return_value = torch.tensor(
[5120.0, 3584.0, 6144.0, 3500.0, 2560.0], dtype=torch.float64
)
replica_group = SimpleNamespace(world_size=2, cpu_group=object())
reduced = []
def all_reduce(tensor, op, group):
# five host counters over the gloo group, never a device collective
assert tensor.device.type == "cpu"
assert op == torch.distributed.ReduceOp.MAX
assert group is replica_group.cpu_group
reduced.append(tensor.tolist())
tensor.copy_(
torch.tensor([5120.0, 3584.0, 6144.0, 3500.0, 2560.0], dtype=torch.float64)
)
snapshots = [
MemorySnapshot(0.0, 0.0, 2048.0, 3072.0),
MemorySnapshot(0.0, 0.0, 2048.0, 3072.0),
@@ -205,6 +214,9 @@ def test_worker_records_replica_load_and_runtime_peaks():
patch.object(
gpu_worker_module, "get_replica_group", return_value=replica_group
),
patch.object(
gpu_worker_module.torch.distributed, "all_reduce", side_effect=all_reduce
),
):
worker._record_output_peak_memory(output)
worker._record_replica_peak_memory([metrics])
@@ -218,6 +230,7 @@ def test_worker_records_replica_load_and_runtime_peaks():
assert metrics.memory_snapshots["load_peak"].peak_allocated_mb == 3500.0
assert metrics.memory_snapshots["runtime_peak"].peak_allocated_mb == 2560.0
assert metrics.memory_snapshots["warmup_peak"].peak_reserved_mb == 6144.0
assert reduced == [[4096.0, 3072.0, 0.0, 3000.0, 2048.0]]
def test_server_warmup_preserves_peak_after_managed_stage_timeline():