[MLX] Add Metal profiling hooks to server profiler (#28122)
This commit is contained in:
Lijuan Tang
@@ -0,0 +1,261 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.io_struct import ProfileReqOutput
|
||||
from sglang.srt.utils.tensor_bridge import use_mlx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetalCaptureProfiler:
|
||||
label: str
|
||||
trace_path: Path
|
||||
stop_capture: Callable[[], None]
|
||||
standalone: bool
|
||||
|
||||
@classmethod
|
||||
def start_mlx(cls, trace_path: Path):
|
||||
trace_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
import mlx.core as mx
|
||||
|
||||
mx.metal.start_capture(str(trace_path))
|
||||
except RuntimeError as e:
|
||||
return None, _capture_error("MLX", e)
|
||||
|
||||
return cls._started(
|
||||
label="MLX",
|
||||
trace_path=trace_path,
|
||||
stop_capture=mx.metal.stop_capture,
|
||||
standalone=True,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def start_mps(cls, trace_path: Path):
|
||||
trace_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
if not hasattr(torch, "mps") or not hasattr(torch.mps, "profiler"):
|
||||
raise RuntimeError("torch.mps.profiler is not available")
|
||||
context = torch.mps.profiler.metal_capture(str(trace_path))
|
||||
context.__enter__()
|
||||
except RuntimeError as e:
|
||||
return None, _capture_error("MPS", e)
|
||||
|
||||
return cls._started(
|
||||
label="MPS",
|
||||
trace_path=trace_path,
|
||||
stop_capture=lambda: context.__exit__(None, None, None),
|
||||
standalone=False,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _started(
|
||||
cls,
|
||||
*,
|
||||
label: str,
|
||||
trace_path: Path,
|
||||
stop_capture: Callable[[], None],
|
||||
standalone: bool,
|
||||
):
|
||||
profiler = cls(
|
||||
label=label,
|
||||
trace_path=trace_path,
|
||||
stop_capture=stop_capture,
|
||||
standalone=standalone,
|
||||
)
|
||||
logger.info("%s Metal capture started, saving to %s", label, trace_path)
|
||||
return profiler, ProfileReqOutput(success=True, message="Succeeded")
|
||||
|
||||
def stop(self) -> str:
|
||||
self.stop_capture()
|
||||
|
||||
logger.info(
|
||||
"%s Metal capture stopped. Trace saved to: %s",
|
||||
self.label,
|
||||
self.trace_path,
|
||||
)
|
||||
return f" Metal trace: {self.trace_path}"
|
||||
|
||||
|
||||
def _capture_error(label: str, error: RuntimeError) -> ProfileReqOutput:
|
||||
return ProfileReqOutput(
|
||||
success=False,
|
||||
message=(
|
||||
f"Failed to start {label} Metal capture: {error}. "
|
||||
"Set MTL_CAPTURE_ENABLED=1 in the server's environment "
|
||||
"before launching to enable GPU trace capture."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class MetalTorchProfiler:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
start_metal_capture: Callable[[Path], tuple[Any, ProfileReqOutput]],
|
||||
torch_profiler: Optional[Any] = None,
|
||||
):
|
||||
self.start_metal_capture = start_metal_capture
|
||||
self.torch_profiler = torch_profiler
|
||||
self.metal_profiler = None
|
||||
|
||||
def start(self):
|
||||
trace_path = _new_temp_gputrace_path()
|
||||
self.metal_profiler, result = self.start_metal_capture(trace_path)
|
||||
if not result.success:
|
||||
raise RuntimeError(result.message)
|
||||
if self.torch_profiler is not None:
|
||||
try:
|
||||
self.torch_profiler.start()
|
||||
except Exception:
|
||||
self.metal_profiler.stop()
|
||||
raise
|
||||
|
||||
def stop(self):
|
||||
try:
|
||||
if self.torch_profiler is not None:
|
||||
self.torch_profiler.stop()
|
||||
finally:
|
||||
if self.metal_profiler is not None:
|
||||
self.metal_profiler.stop()
|
||||
|
||||
def export_chrome_trace(self, path: str):
|
||||
if self.torch_profiler is not None:
|
||||
self.torch_profiler.export_chrome_trace(path)
|
||||
else:
|
||||
_write_empty_chrome_trace(path)
|
||||
|
||||
if self.metal_profiler is None:
|
||||
return
|
||||
|
||||
final_path = _unique_gputrace_path_for_chrome_trace(path)
|
||||
final_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if self.metal_profiler.trace_path.exists():
|
||||
shutil.move(str(self.metal_profiler.trace_path), str(final_path))
|
||||
logger.info("Metal trace saved to: %s", final_path)
|
||||
|
||||
|
||||
def apply_metal_profiler_patches() -> None:
|
||||
if getattr(torch.profiler.profile, "_sglang_metal_patched", False):
|
||||
return
|
||||
|
||||
original_profile = torch.profiler.profile
|
||||
|
||||
def profile(*args, **kwargs):
|
||||
activities = _get_activities(args, kwargs)
|
||||
if not _has_cuda_activity(activities):
|
||||
return original_profile(*args, **kwargs)
|
||||
|
||||
if use_mlx():
|
||||
return MetalTorchProfiler(
|
||||
start_metal_capture=MetalCaptureProfiler.start_mlx
|
||||
)
|
||||
|
||||
torch_activities = [
|
||||
activity for activity in activities if not _is_cuda_activity(activity)
|
||||
]
|
||||
torch_profiler = None
|
||||
if torch_activities:
|
||||
patched_args, patched_kwargs = _replace_activities(
|
||||
args, kwargs, torch_activities
|
||||
)
|
||||
torch_profiler = original_profile(*patched_args, **patched_kwargs)
|
||||
|
||||
return MetalTorchProfiler(
|
||||
start_metal_capture=MetalCaptureProfiler.start_mps,
|
||||
torch_profiler=torch_profiler,
|
||||
)
|
||||
|
||||
profile._sglang_metal_patched = True
|
||||
profile._sglang_original_profile = original_profile
|
||||
torch.profiler.profile = profile
|
||||
|
||||
|
||||
def _get_activities(args, kwargs):
|
||||
if "activities" in kwargs:
|
||||
return kwargs["activities"]
|
||||
if args:
|
||||
return args[0]
|
||||
return None
|
||||
|
||||
|
||||
def _replace_activities(args, kwargs, activities):
|
||||
kwargs = dict(kwargs)
|
||||
if "activities" in kwargs:
|
||||
kwargs["activities"] = activities
|
||||
return args, kwargs
|
||||
|
||||
if args:
|
||||
args = list(args)
|
||||
args[0] = activities
|
||||
return tuple(args), kwargs
|
||||
|
||||
kwargs["activities"] = activities
|
||||
return args, kwargs
|
||||
|
||||
|
||||
def _has_cuda_activity(activities) -> bool:
|
||||
if activities is None:
|
||||
return False
|
||||
return any(_is_cuda_activity(activity) for activity in activities)
|
||||
|
||||
|
||||
def _is_cuda_activity(activity) -> bool:
|
||||
return activity == torch.profiler.ProfilerActivity.CUDA
|
||||
|
||||
|
||||
def _new_temp_gputrace_path() -> Path:
|
||||
output_dir = Path(os.getenv("SGLANG_TORCH_PROFILER_DIR", "/tmp")).expanduser()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
for i in range(100):
|
||||
candidate = (
|
||||
output_dir / f"sglang-metal-{os.getpid()}-{time.time_ns()}-{i}.gputrace"
|
||||
)
|
||||
if not candidate.exists():
|
||||
return candidate
|
||||
raise RuntimeError(f"Cannot find an unused Metal trace path in {output_dir}")
|
||||
|
||||
|
||||
def _unique_gputrace_path_for_chrome_trace(path: str) -> Path:
|
||||
chrome_path = Path(path).expanduser()
|
||||
name = chrome_path.name
|
||||
if name.endswith(".trace.json.gz"):
|
||||
name = name[: -len(".trace.json.gz")] + ".gputrace"
|
||||
else:
|
||||
name = chrome_path.stem + ".gputrace"
|
||||
|
||||
base = chrome_path.with_name(name)
|
||||
if not base.exists():
|
||||
return base
|
||||
|
||||
stem = base.name[: -len(".gputrace")]
|
||||
for i in range(100):
|
||||
candidate = base.with_name(f"{stem}-{time.time_ns()}-{i}.gputrace")
|
||||
if not candidate.exists():
|
||||
return candidate
|
||||
raise RuntimeError(f"Cannot find an unused Metal trace path for {base}")
|
||||
|
||||
|
||||
def _write_empty_chrome_trace(path: str):
|
||||
trace = {"traceEvents": []}
|
||||
Path(path).expanduser().parent.mkdir(parents=True, exist_ok=True)
|
||||
if str(path).endswith(".gz"):
|
||||
with gzip.open(path, "wt") as f:
|
||||
json.dump(trace, f)
|
||||
else:
|
||||
with open(path, "w") as f:
|
||||
json.dump(trace, f)
|
||||
@@ -19,14 +19,16 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.io_struct import ProfileReq, ProfileReqOutput, ProfileReqType
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.server_args import get_global_server_args
|
||||
from sglang.srt.utils import is_npu
|
||||
from sglang.srt.utils import is_mps, is_npu
|
||||
from sglang.srt.utils.profile_merger import ProfileMerger
|
||||
from sglang.srt.utils.profile_utils import ProfileManager
|
||||
from sglang.srt.utils.torch_npu_patch_utils import apply_torch_npu_patches
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
|
||||
_is_npu = is_npu()
|
||||
_is_mps = is_mps()
|
||||
if _is_npu:
|
||||
import torch_npu
|
||||
|
||||
@@ -36,13 +38,14 @@ if _is_npu:
|
||||
["profiler.ProfilerActivity.CPU", torch_npu.profiler.ProfilerActivity.CPU],
|
||||
]
|
||||
apply_torch_npu_patches(torch_npu, patches)
|
||||
elif _is_mps:
|
||||
from sglang.srt.hardware_backend.mlx.profiler import apply_metal_profiler_patches
|
||||
|
||||
apply_metal_profiler_patches()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
from sglang.srt.utils.profile_utils import ProfileManager
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class SchedulerProfilerManager:
|
||||
ps: Any
|
||||
@@ -230,7 +233,11 @@ class SchedulerProfilerManager:
|
||||
)
|
||||
),
|
||||
)
|
||||
self.torch_profiler.start()
|
||||
try:
|
||||
self.torch_profiler.start()
|
||||
except RuntimeError as e:
|
||||
self.torch_profiler = None
|
||||
return ProfileReqOutput(success=False, message=str(e))
|
||||
self.profile_in_progress = True
|
||||
|
||||
if "MEM" in activities:
|
||||
|
||||
Reference in New Issue
Block a user