[diffusion] chore: make perf dump one writer per replica with no first-request NCCL setup (#41711)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
2cd92d10b9
commit
bb9a820f09
@@ -758,18 +758,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
if not req.is_warmup:
|
||||
PerformanceLogger.log_request_summary(metrics=output_batch.metrics)
|
||||
|
||||
# dump per-request perf report to the server-mode file path.
|
||||
if (
|
||||
req.perf_dump_path is not None
|
||||
and not req.is_warmup
|
||||
and output_batch.metrics is not None
|
||||
):
|
||||
PerformanceLogger.dump_benchmark_report(
|
||||
file_path=req.perf_dump_path,
|
||||
metrics=output_batch.metrics,
|
||||
meta={"model": self.server_args.model_path},
|
||||
tag="server_perf_dump",
|
||||
)
|
||||
self._dump_perf_report(req, output_batch)
|
||||
except Exception as e:
|
||||
if propagate_forward_errors and forward_failed:
|
||||
if isinstance(e, StopIteration):
|
||||
@@ -1104,6 +1093,23 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
if self.is_output_rank:
|
||||
output_batch.peak_memory_mb = snapshot.peak_reserved_mb
|
||||
|
||||
def _dump_perf_report(self, req: Req, output_batch: OutputBatch) -> None:
|
||||
"""Write the per-request perf report to the server-mode file path."""
|
||||
# one writer per replica, or ranks sharing the path clobber each other
|
||||
if (
|
||||
req.perf_dump_path is None
|
||||
or req.is_warmup
|
||||
or output_batch.metrics is None
|
||||
or not self.is_output_rank
|
||||
):
|
||||
return
|
||||
PerformanceLogger.dump_benchmark_report(
|
||||
file_path=req.perf_dump_path,
|
||||
metrics=output_batch.metrics,
|
||||
meta={"model": self.server_args.model_path},
|
||||
tag="server_perf_dump",
|
||||
)
|
||||
|
||||
def _record_replica_peak_memory(self, output_metrics: list[Any]) -> None:
|
||||
"""Record replica-wide loading and runtime allocator peaks."""
|
||||
if not current_platform.is_cuda():
|
||||
@@ -1118,9 +1124,15 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
self._runtime_peak_allocated_mb,
|
||||
],
|
||||
dtype=torch.float64,
|
||||
device=current_platform.get_device(self.local_rank),
|
||||
)
|
||||
peaks = get_replica_group().all_reduce(peaks, op=torch.distributed.ReduceOp.MAX)
|
||||
replica = get_replica_group()
|
||||
if replica.world_size > 1:
|
||||
# Host counters over gloo: the device group's first collective
|
||||
# sets up NCCL connections, which cost the first dumped request
|
||||
# 0.15 s of latency.
|
||||
torch.distributed.all_reduce(
|
||||
peaks, op=torch.distributed.ReduceOp.MAX, group=replica.cpu_group
|
||||
)
|
||||
if not self.is_output_rank:
|
||||
return
|
||||
|
||||
|
||||
@@ -445,8 +445,11 @@ class PerformanceLogger:
|
||||
try:
|
||||
abs_path = os.path.abspath(file_path)
|
||||
os.makedirs(os.path.dirname(abs_path), exist_ok=True)
|
||||
with open(abs_path, "w", encoding="utf-8") as f:
|
||||
# readers poll for this file: never let them see a partial write
|
||||
tmp_path = f"{abs_path}.{os.getpid()}.tmp"
|
||||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(report, f, indent=2)
|
||||
os.replace(tmp_path, abs_path)
|
||||
logger.info(f"Metrics dumped to: {CYAN}{abs_path}{RESET}")
|
||||
except IOError as e:
|
||||
logger.error(f"Failed to dump metrics to {abs_path}: {e}")
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
|
||||
from sglang.multimodal_gen.runtime.utils import perf_logger as perf_logger_module
|
||||
from sglang.multimodal_gen.runtime.utils.perf_logger import PerformanceLogger
|
||||
|
||||
|
||||
def _metrics(request_id: str) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
request_id=request_id,
|
||||
stages={"DenoisingStage": 1.0},
|
||||
steps=[1.0],
|
||||
memory_snapshots={},
|
||||
total_duration_ms=1.0,
|
||||
)
|
||||
|
||||
|
||||
def test_reader_during_a_dump_sees_the_previous_complete_report(tmp_path, monkeypatch):
|
||||
# clients poll this path; writing it in place exposed a truncated file
|
||||
path = tmp_path / "perf.json"
|
||||
monkeypatch.setattr(perf_logger_module, "get_git_commit_hash", lambda: "test")
|
||||
PerformanceLogger.dump_benchmark_report(str(path), _metrics("first"))
|
||||
|
||||
seen_mid_write = []
|
||||
real_dump = json.dump
|
||||
|
||||
def dump_while_reading(obj, fp, **kwargs):
|
||||
seen_mid_write.append(path.read_text())
|
||||
real_dump(obj, fp, **kwargs)
|
||||
|
||||
monkeypatch.setattr(perf_logger_module.json, "dump", dump_while_reading)
|
||||
PerformanceLogger.dump_benchmark_report(str(path), _metrics("second"))
|
||||
|
||||
assert [json.loads(text)["request_id"] for text in seen_mid_write] == ["first"]
|
||||
assert json.loads(path.read_text())["request_id"] == "second"
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == ["perf.json"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_output_rank", [True, False])
|
||||
def test_only_the_output_rank_writes_the_report(tmp_path, monkeypatch, is_output_rank):
|
||||
monkeypatch.setattr(perf_logger_module, "get_git_commit_hash", lambda: "test")
|
||||
worker = GPUWorker.__new__(GPUWorker)
|
||||
worker.is_output_rank = is_output_rank
|
||||
worker.server_args = SimpleNamespace(model_path="model")
|
||||
path = tmp_path / "perf.json"
|
||||
|
||||
worker._dump_perf_report(
|
||||
SimpleNamespace(perf_dump_path=str(path), is_warmup=False),
|
||||
SimpleNamespace(metrics=_metrics("request")),
|
||||
)
|
||||
|
||||
assert path.exists() == is_output_rank
|
||||
@@ -186,10 +186,19 @@ def test_worker_records_replica_load_and_runtime_peaks():
|
||||
worker._runtime_peak_allocated_mb = 0.0
|
||||
output = OutputBatch()
|
||||
metrics = RequestMetrics("request")
|
||||
replica_group = Mock()
|
||||
replica_group.all_reduce.return_value = torch.tensor(
|
||||
[5120.0, 3584.0, 6144.0, 3500.0, 2560.0], dtype=torch.float64
|
||||
)
|
||||
replica_group = SimpleNamespace(world_size=2, cpu_group=object())
|
||||
reduced = []
|
||||
|
||||
def all_reduce(tensor, op, group):
|
||||
# five host counters over the gloo group, never a device collective
|
||||
assert tensor.device.type == "cpu"
|
||||
assert op == torch.distributed.ReduceOp.MAX
|
||||
assert group is replica_group.cpu_group
|
||||
reduced.append(tensor.tolist())
|
||||
tensor.copy_(
|
||||
torch.tensor([5120.0, 3584.0, 6144.0, 3500.0, 2560.0], dtype=torch.float64)
|
||||
)
|
||||
|
||||
snapshots = [
|
||||
MemorySnapshot(0.0, 0.0, 2048.0, 3072.0),
|
||||
MemorySnapshot(0.0, 0.0, 2048.0, 3072.0),
|
||||
@@ -205,6 +214,9 @@ def test_worker_records_replica_load_and_runtime_peaks():
|
||||
patch.object(
|
||||
gpu_worker_module, "get_replica_group", return_value=replica_group
|
||||
),
|
||||
patch.object(
|
||||
gpu_worker_module.torch.distributed, "all_reduce", side_effect=all_reduce
|
||||
),
|
||||
):
|
||||
worker._record_output_peak_memory(output)
|
||||
worker._record_replica_peak_memory([metrics])
|
||||
@@ -218,6 +230,7 @@ def test_worker_records_replica_load_and_runtime_peaks():
|
||||
assert metrics.memory_snapshots["load_peak"].peak_allocated_mb == 3500.0
|
||||
assert metrics.memory_snapshots["runtime_peak"].peak_allocated_mb == 2560.0
|
||||
assert metrics.memory_snapshots["warmup_peak"].peak_reserved_mb == 6144.0
|
||||
assert reduced == [[4096.0, 3072.0, 0.0, 3000.0, 2048.0]]
|
||||
|
||||
|
||||
def test_server_warmup_preserves_peak_after_managed_stage_timeline():
|
||||
|
||||
Reference in New Issue
Block a user