[SKILL] Sync SGLang skill docs (#23921)
This commit is contained in:
Xiaoyu Zhang
@@ -0,0 +1,806 @@
|
||||
"""Compact triage entrypoint for unified LLM torch-profiler analysis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
import triage_kernel_helpers as kernel_helpers
|
||||
import triage_overlap_helpers as overlap_helpers
|
||||
from profile_common import (
|
||||
discover_trace_targets,
|
||||
framework_display_name,
|
||||
load_server_args,
|
||||
load_trace_json,
|
||||
parse_stage,
|
||||
resolve_framework,
|
||||
run_profiler,
|
||||
)
|
||||
|
||||
MIN_RENDER_SHARE_PCT = 1.0
|
||||
MAPPING_KERNEL_SAMPLE_LIMIT_PER_NAME = 16
|
||||
|
||||
|
||||
def build_triage_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="analyze_llm_torch_profile.py",
|
||||
description=(
|
||||
"Compact LLM torch-profiler triage entrypoint for SGLang, vLLM, and "
|
||||
"TensorRT-LLM. "
|
||||
"This prints three tables: kernel mapping, overlap opportunities, "
|
||||
"and fuse opportunities. "
|
||||
"Use either a single trace/profile input or a mapping+formal two-trace pair."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--framework",
|
||||
type=str,
|
||||
default="auto",
|
||||
choices=["auto", "sglang", "vllm", "trtllm", "tllm", "tensorrt-llm"],
|
||||
help=(
|
||||
"Serving framework. Use auto to detect from trace contents, path hints, "
|
||||
"or URL features."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Single trace file or profile directory to triage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--url",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Running server URL for single-trace triage. SGLang supports direct "
|
||||
"capture through its profiler HTTP API. vLLM and TensorRT-LLM require "
|
||||
"a server-side torch-profiler output path exposed via --output-dir."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Trace output dir when using --url. For vLLM this should match the "
|
||||
"server's torch_profiler_dir. For TensorRT-LLM it should match the "
|
||||
"directory or file path configured by TLLM_TORCH_PROFILE_TRACE."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile-prefix",
|
||||
type=str,
|
||||
default="triage-trace",
|
||||
help=(
|
||||
"Profile prefix when generating a trace from --url. SGLang uses it "
|
||||
"directly; vLLM and TensorRT-LLM may ignore it on the HTTP profiler path."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mapping-input",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Graph-off mapping trace file or directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mapping-url",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Running graph-off server URL for the mapping trace.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--formal-input",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Formal graph-on trace file or directory.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--formal-url",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Running graph-on server URL for the formal trace.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mapping-output-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Trace output dir when using --mapping-url.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--formal-output-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Trace output dir when using --formal-url.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mapping-profile-prefix",
|
||||
type=str,
|
||||
default="mapping-trace",
|
||||
help="Profile prefix for the mapping trace.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--formal-profile-prefix",
|
||||
type=str,
|
||||
default="formal-trace",
|
||||
help="Profile prefix for the formal trace.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-steps",
|
||||
type=int,
|
||||
default=5,
|
||||
help="Profiler steps when generating traces from URLs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile-by-stage", action=argparse.BooleanOptionalAction, default=True
|
||||
)
|
||||
parser.add_argument(
|
||||
"--merge-profiles", action=argparse.BooleanOptionalAction, default=False
|
||||
)
|
||||
parser.add_argument("--probe-requests", type=int, default=1)
|
||||
parser.add_argument(
|
||||
"--probe-prompt",
|
||||
type=str,
|
||||
default=(
|
||||
"Repeat the word profiler many times with spaces so the server performs several decode steps. "
|
||||
"Do not add explanations."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--probe-max-new-tokens", type=int, default=None)
|
||||
parser.add_argument("--probe-delay", type=float, default=0.5)
|
||||
parser.add_argument(
|
||||
"--start-step",
|
||||
type=int,
|
||||
default=None,
|
||||
help="SGLang-only profiler start step when generating traces from URLs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pid-substring",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Restrict overlap analysis to PIDs containing this substring.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--kernel-table-limit",
|
||||
type=int,
|
||||
default=0,
|
||||
help="How many kernel rows to print per stage. Use 0 for all kernels.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--overlap-table-limit",
|
||||
type=int,
|
||||
default=0,
|
||||
help="How many overlap rows to print per stage. Use 0 for all kernels.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_triage_args(argv: Sequence[str]) -> argparse.Namespace:
|
||||
parser = build_triage_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
single_trace_mode = bool(args.input) or bool(args.url)
|
||||
dual_trace_mode = any(
|
||||
[
|
||||
args.mapping_input,
|
||||
args.mapping_url,
|
||||
args.formal_input,
|
||||
args.formal_url,
|
||||
]
|
||||
)
|
||||
|
||||
if single_trace_mode and dual_trace_mode:
|
||||
parser.error(
|
||||
"Use either single-trace mode (--input/--url) or two-trace mode "
|
||||
"(--mapping-* plus --formal-*), not both."
|
||||
)
|
||||
|
||||
if single_trace_mode:
|
||||
if bool(args.input) == bool(args.url):
|
||||
parser.error("Provide exactly one of --input or --url.")
|
||||
return args
|
||||
|
||||
if bool(args.mapping_input) == bool(args.mapping_url):
|
||||
parser.error("Provide exactly one of --mapping-input or --mapping-url.")
|
||||
if bool(args.formal_input) == bool(args.formal_url):
|
||||
parser.error("Provide exactly one of --formal-input or --formal-url.")
|
||||
return args
|
||||
|
||||
|
||||
def resolve_profile_targets(
|
||||
*,
|
||||
label: str,
|
||||
input_path: Optional[str],
|
||||
url: Optional[str],
|
||||
output_dir: Optional[str],
|
||||
profile_prefix: Optional[str],
|
||||
args: argparse.Namespace,
|
||||
) -> Tuple[List[Path], Optional[dict], str]:
|
||||
if bool(input_path) == bool(url):
|
||||
raise ValueError(f"{label} trace requires exactly one of input path or URL.")
|
||||
|
||||
if url:
|
||||
framework = resolve_framework(
|
||||
args.framework,
|
||||
input_path=Path(output_dir).resolve() if output_dir else None,
|
||||
url=url,
|
||||
)
|
||||
target_dir = run_profiler(
|
||||
url=url,
|
||||
output_dir=output_dir,
|
||||
num_steps=args.num_steps,
|
||||
profile_by_stage=args.profile_by_stage,
|
||||
merge_profiles=args.merge_profiles,
|
||||
profile_prefix=profile_prefix,
|
||||
probe_requests=max(0, args.probe_requests),
|
||||
probe_prompt=args.probe_prompt,
|
||||
probe_max_new_tokens=args.probe_max_new_tokens,
|
||||
probe_delay=args.probe_delay,
|
||||
start_step=args.start_step,
|
||||
framework=framework,
|
||||
framework_hint_path=output_dir,
|
||||
)
|
||||
traces, server_args = discover_trace_targets(target_dir, all_traces=False)
|
||||
resolved_framework = resolve_framework(
|
||||
args.framework,
|
||||
input_path=target_dir,
|
||||
url=url,
|
||||
server_args=server_args,
|
||||
)
|
||||
return traces, server_args, resolved_framework
|
||||
|
||||
resolved = Path(input_path).resolve()
|
||||
traces, server_args = discover_trace_targets(resolved, all_traces=False)
|
||||
if server_args is None:
|
||||
server_args = load_server_args(resolved)
|
||||
framework = resolve_framework(
|
||||
args.framework, input_path=resolved, server_args=server_args
|
||||
)
|
||||
return traces, server_args, framework
|
||||
|
||||
|
||||
def build_mapping_kernel_map(trace_paths: Sequence[Path], framework: str) -> dict:
|
||||
stage_site_stats = defaultdict(
|
||||
lambda: defaultdict(lambda: defaultdict(kernel_helpers.MappingSiteAggregate))
|
||||
)
|
||||
stage_kernel_categories: Dict[str, Dict[str, str]] = defaultdict(dict)
|
||||
global_site_stats = defaultdict(
|
||||
lambda: defaultdict(kernel_helpers.MappingSiteAggregate)
|
||||
)
|
||||
global_kernel_categories: Dict[str, str] = {}
|
||||
|
||||
for trace_path in trace_paths:
|
||||
trace = load_trace_json(trace_path)
|
||||
kernels, cpu_ops, python_frames, launch_events, _, _ = (
|
||||
kernel_helpers.extract_trace_data(trace)
|
||||
)
|
||||
if not kernels:
|
||||
continue
|
||||
cpu_ops_by_external_id = kernel_helpers.build_cpu_op_index(cpu_ops)
|
||||
launches_by_correlation = kernel_helpers.build_launch_index(launch_events)
|
||||
site_context_cache = {}
|
||||
default_stage = parse_stage(trace_path)
|
||||
for stage, stage_kernels in kernel_helpers.group_kernels_by_stage(
|
||||
kernels, default_stage
|
||||
).items():
|
||||
sampled_stage_kernels = (
|
||||
stage_kernels
|
||||
if framework == "sglang"
|
||||
else sample_kernels_for_mapping(stage_kernels)
|
||||
)
|
||||
local_site_stats = kernel_helpers.aggregate_kernel_sites(
|
||||
sampled_stage_kernels,
|
||||
cpu_ops_by_external_id,
|
||||
python_frames,
|
||||
launches_by_correlation=launches_by_correlation,
|
||||
site_context_cache=site_context_cache,
|
||||
)
|
||||
kernel_categories = {
|
||||
kernel.canonical_name: kernel.category for kernel in stage_kernels
|
||||
}
|
||||
kernel_helpers.merge_site_stats(stage_site_stats[stage], local_site_stats)
|
||||
kernel_helpers.merge_site_stats(global_site_stats, local_site_stats)
|
||||
stage_kernel_categories[stage].update(kernel_categories)
|
||||
global_kernel_categories.update(kernel_categories)
|
||||
|
||||
stage_payloads = {
|
||||
stage: kernel_helpers.build_stage_payload(
|
||||
dict(site_stats), stage_kernel_categories.get(stage, {})
|
||||
)
|
||||
for stage, site_stats in stage_site_stats.items()
|
||||
}
|
||||
global_payload = kernel_helpers.build_stage_payload(
|
||||
dict(global_site_stats), global_kernel_categories
|
||||
)
|
||||
return {"stages": stage_payloads, "global": global_payload}
|
||||
|
||||
|
||||
def stage_index(stage: str) -> int:
|
||||
return {"extend": 0, "prefill": 0, "decode": 1, "all": 2}.get(stage, 99)
|
||||
|
||||
|
||||
def sample_kernels_for_mapping(
|
||||
kernels: Sequence[kernel_helpers.KernelEvent],
|
||||
per_name_limit: int = MAPPING_KERNEL_SAMPLE_LIMIT_PER_NAME,
|
||||
) -> List[kernel_helpers.KernelEvent]:
|
||||
if per_name_limit <= 0:
|
||||
return list(kernels)
|
||||
|
||||
grouped: Dict[str, List[kernel_helpers.KernelEvent]] = defaultdict(list)
|
||||
for kernel in kernels:
|
||||
grouped[kernel.canonical_name].append(kernel)
|
||||
|
||||
sampled: List[kernel_helpers.KernelEvent] = []
|
||||
for kernel_name in sorted(grouped):
|
||||
items = grouped[kernel_name]
|
||||
if len(items) <= per_name_limit:
|
||||
sampled.extend(items)
|
||||
continue
|
||||
for sample_idx in range(per_name_limit):
|
||||
pos = round(sample_idx * (len(items) - 1) / (per_name_limit - 1))
|
||||
sampled.append(items[pos])
|
||||
sampled.sort(key=lambda kernel: (kernel.ts, kernel.name))
|
||||
return sampled
|
||||
|
||||
|
||||
def stage_display(stage: str) -> str:
|
||||
return kernel_helpers.stage_label(stage)
|
||||
|
||||
|
||||
def pick_stage_value(stage_to_value: Dict[str, object], stage: str) -> Optional[object]:
|
||||
if stage in stage_to_value:
|
||||
return stage_to_value[stage]
|
||||
if "all" in stage_to_value:
|
||||
return stage_to_value["all"]
|
||||
if len(stage_to_value) == 1:
|
||||
return next(iter(stage_to_value.values()))
|
||||
return None
|
||||
|
||||
|
||||
def render_stages(stage_to_value: Dict[str, object]) -> List[str]:
|
||||
stages = set(stage_to_value)
|
||||
if any(stage != "all" for stage in stages):
|
||||
stages.discard("all")
|
||||
return sorted(stages, key=stage_index)
|
||||
|
||||
|
||||
def build_overlap_stage_bundle_map(
|
||||
trace_paths: Sequence[Path],
|
||||
*,
|
||||
label_prefix: str,
|
||||
server_args: Optional[dict],
|
||||
pid_substring: Optional[str],
|
||||
) -> Dict[str, overlap_helpers.TraceBundle]:
|
||||
stage_bundles: Dict[str, overlap_helpers.TraceBundle] = {}
|
||||
for trace_path in sorted(
|
||||
trace_paths, key=lambda item: (stage_index(parse_stage(item)), item.name)
|
||||
):
|
||||
trace_json = load_trace_json(trace_path)
|
||||
raw_events = trace_json.get(
|
||||
"traceEvents",
|
||||
trace_json if isinstance(trace_json, list) else [],
|
||||
)
|
||||
events, pid = overlap_helpers.extract_kernel_events(trace_json, pid_substring)
|
||||
if not events:
|
||||
continue
|
||||
default_stage = parse_stage(trace_path)
|
||||
stage_groups = overlap_helpers.group_events_by_stage(events, default_stage)
|
||||
for stage in render_stages(stage_groups):
|
||||
if stage in stage_bundles:
|
||||
continue
|
||||
stage_bundles[stage] = overlap_helpers.TraceBundle(
|
||||
label=f"{label_prefix}-{stage}",
|
||||
trace_path=trace_path,
|
||||
server_args=server_args,
|
||||
raw_events=raw_events,
|
||||
events=stage_groups[stage],
|
||||
pid=pid,
|
||||
)
|
||||
if "all" in stage_groups and not stage_bundles:
|
||||
stage_bundles["all"] = overlap_helpers.TraceBundle(
|
||||
label=f"{label_prefix}-all",
|
||||
trace_path=trace_path,
|
||||
server_args=server_args,
|
||||
raw_events=raw_events,
|
||||
events=stage_groups["all"],
|
||||
pid=pid,
|
||||
)
|
||||
return stage_bundles
|
||||
|
||||
|
||||
def group_rows_by_stage(rows: Sequence[dict]) -> List[Tuple[str, List[dict]]]:
|
||||
grouped: Dict[str, List[dict]] = defaultdict(list)
|
||||
for row in rows:
|
||||
grouped[str(row.get("stage") or "all")].append(row)
|
||||
return [
|
||||
(stage, grouped[stage]) for stage in sorted(grouped.keys(), key=stage_index)
|
||||
]
|
||||
|
||||
|
||||
def render_kernel_table_for_stage(rows: Sequence[dict]) -> List[str]:
|
||||
lines = [
|
||||
"| Kernel | Category | GPU time | Share | Launches | Python location (site share) | CPU op |",
|
||||
"| --- | --- | ---: | ---: | ---: | --- | --- |",
|
||||
]
|
||||
if not rows:
|
||||
lines.append(
|
||||
"| No kernel rows at or above 1.0% share. | - | - | - | - | - | - |"
|
||||
)
|
||||
return lines
|
||||
for row in rows:
|
||||
lines.append(
|
||||
"| {kernel} | {category} | {gpu_time} | {share:.1f}% | {launches} | {location} | {cpu_op} |".format(
|
||||
kernel=kernel_helpers.escape_md_cell(row["kernel"]),
|
||||
category=kernel_helpers.escape_md_cell(row["category"]),
|
||||
gpu_time=kernel_helpers.format_ms(row["total_us"]),
|
||||
share=row["share_pct"],
|
||||
launches=row["launches"],
|
||||
location=kernel_helpers.escape_md_cell(row["location"]),
|
||||
cpu_op=kernel_helpers.escape_md_cell(row["cpu_op"]),
|
||||
)
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def render_stage_section_tables(
|
||||
rows: Sequence[dict],
|
||||
*,
|
||||
render_stage_fn,
|
||||
stage_label_prefix: str = "#####",
|
||||
) -> List[str]:
|
||||
if not rows:
|
||||
return render_stage_fn([])
|
||||
stage_groups = group_rows_by_stage(rows)
|
||||
if len(stage_groups) == 1 and stage_groups[0][0] == "all":
|
||||
return render_stage_fn(stage_groups[0][1])
|
||||
|
||||
lines: List[str] = []
|
||||
for index, (stage, stage_rows) in enumerate(stage_groups):
|
||||
lines.append(f"{stage_label_prefix} {stage_display(stage)}")
|
||||
lines.extend(render_stage_fn(stage_rows))
|
||||
if index != len(stage_groups) - 1:
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def render_kernel_tables(rows: Sequence[dict]) -> List[str]:
|
||||
return render_stage_section_tables(
|
||||
rows, render_stage_fn=render_kernel_table_for_stage
|
||||
)
|
||||
|
||||
|
||||
def render_overlap_table_for_stage(rows: Sequence[dict]) -> List[str]:
|
||||
lines = [
|
||||
"| Priority | Verdict | Kernel | Python scope | Formal signal | Dep risk | Recommendation |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
if not rows:
|
||||
lines.append(
|
||||
"| - | - | No rows cleared the 1.0% reporting bar. Use mapping/formal mode for overlap attribution. | - | - | - | - |"
|
||||
)
|
||||
return lines
|
||||
for row in rows:
|
||||
formal_signal = (
|
||||
f"{row['total_us']:.1f} us, share {row['share_pct']:.1f}%, "
|
||||
f"excl {row['exclusive_ratio'] * 100:.1f}% / hid {row['hidden_ratio'] * 100:.1f}%"
|
||||
)
|
||||
lines.append(
|
||||
"| "
|
||||
+ " | ".join(
|
||||
[
|
||||
row["priority"],
|
||||
row["verdict"],
|
||||
kernel_helpers.escape_md_cell(row["kernel"]),
|
||||
kernel_helpers.escape_md_cell(row["python_scope"]),
|
||||
kernel_helpers.escape_md_cell(formal_signal),
|
||||
overlap_helpers.dependency_risk_label(row["dependency_signal"]),
|
||||
row["recommendation"],
|
||||
]
|
||||
)
|
||||
+ " |"
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def render_overlap_tables(rows: Sequence[dict]) -> List[str]:
|
||||
return render_stage_section_tables(
|
||||
rows,
|
||||
render_stage_fn=render_overlap_table_for_stage,
|
||||
)
|
||||
|
||||
|
||||
def render_fuse_table_for_stage(rows: Sequence[dict]) -> List[str]:
|
||||
lines = [
|
||||
"| Pattern | Confidence | Related GPU time | Share | Evidence kernels | Current kernel Python location | Candidate fused Python path | Rationale |",
|
||||
"| --- | --- | ---: | ---: | --- | --- | --- | --- |",
|
||||
]
|
||||
if not rows:
|
||||
lines.append(
|
||||
"| No medium-confidence source-backed fusion opportunity matched this trace. | - | - | - | - | - | - | - |"
|
||||
)
|
||||
return lines
|
||||
for row in rows:
|
||||
lines.append(
|
||||
"| {pattern} | {confidence} | {gpu_time} | {share:.1f}% | {evidence} | {current_locations} | {candidate_path} | {rationale} |".format(
|
||||
pattern=kernel_helpers.escape_md_cell(row["pattern"]),
|
||||
confidence=kernel_helpers.escape_md_cell(row["confidence"]),
|
||||
gpu_time=kernel_helpers.format_ms(row["related_us"]),
|
||||
share=row["share_pct"],
|
||||
evidence=kernel_helpers.escape_md_cell(row["evidence"]),
|
||||
current_locations=kernel_helpers.escape_md_cell(
|
||||
row["current_locations"]
|
||||
),
|
||||
candidate_path=kernel_helpers.escape_md_cell(row["candidate_path"]),
|
||||
rationale=kernel_helpers.escape_md_cell(row["rationale"]),
|
||||
)
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def render_fuse_tables(rows: Sequence[dict]) -> List[str]:
|
||||
return render_stage_section_tables(
|
||||
rows,
|
||||
render_stage_fn=render_fuse_table_for_stage,
|
||||
)
|
||||
|
||||
|
||||
def run_triage(args: argparse.Namespace) -> int:
|
||||
single_trace_mode = bool(args.input) or bool(args.url)
|
||||
if single_trace_mode:
|
||||
formal_traces, formal_server_args, formal_framework = resolve_profile_targets(
|
||||
label="input",
|
||||
input_path=args.input,
|
||||
url=args.url,
|
||||
output_dir=args.output_dir,
|
||||
profile_prefix=args.profile_prefix,
|
||||
args=args,
|
||||
)
|
||||
mapping_traces = formal_traces
|
||||
mapping_server_args = formal_server_args
|
||||
mapping_framework = formal_framework
|
||||
else:
|
||||
mapping_traces, mapping_server_args, mapping_framework = (
|
||||
resolve_profile_targets(
|
||||
label="mapping",
|
||||
input_path=args.mapping_input,
|
||||
url=args.mapping_url,
|
||||
output_dir=args.mapping_output_dir,
|
||||
profile_prefix=args.mapping_profile_prefix,
|
||||
args=args,
|
||||
)
|
||||
)
|
||||
formal_traces, formal_server_args, formal_framework = resolve_profile_targets(
|
||||
label="formal",
|
||||
input_path=args.formal_input,
|
||||
url=args.formal_url,
|
||||
output_dir=args.formal_output_dir,
|
||||
profile_prefix=args.formal_profile_prefix,
|
||||
args=args,
|
||||
)
|
||||
|
||||
mapping_kernel_map = build_mapping_kernel_map(mapping_traces, mapping_framework)
|
||||
|
||||
kernel_rows_rendered: List[dict] = []
|
||||
fuse_rows_rendered: List[dict] = []
|
||||
formal_stage_payloads: Dict[str, dict] = {}
|
||||
|
||||
for formal_trace in formal_traces:
|
||||
trace = load_trace_json(formal_trace)
|
||||
kernels, cpu_ops, python_frames, launch_events, _, _ = (
|
||||
kernel_helpers.extract_trace_data(trace)
|
||||
)
|
||||
if not kernels:
|
||||
continue
|
||||
default_stage = parse_stage(formal_trace)
|
||||
stage_groups = kernel_helpers.group_kernels_by_stage(kernels, default_stage)
|
||||
formal_cpu_ops_by_external_id = kernel_helpers.build_cpu_op_index(cpu_ops)
|
||||
formal_launches_by_correlation = kernel_helpers.build_launch_index(
|
||||
launch_events
|
||||
)
|
||||
formal_site_context_cache = {}
|
||||
for stage_name, stage_kernels in stage_groups.items():
|
||||
local_site_stats = kernel_helpers.aggregate_kernel_sites(
|
||||
stage_kernels,
|
||||
formal_cpu_ops_by_external_id,
|
||||
python_frames,
|
||||
launches_by_correlation=formal_launches_by_correlation,
|
||||
site_context_cache=formal_site_context_cache,
|
||||
)
|
||||
formal_stage_payloads[stage_name] = kernel_helpers.build_stage_payload(
|
||||
local_site_stats,
|
||||
{kernel.canonical_name: kernel.category for kernel in stage_kernels},
|
||||
)
|
||||
trace_total_us = sum(kernel.dur for kernel in kernels)
|
||||
for stage in sorted(stage_groups, key=stage_index):
|
||||
stage_kernels = stage_groups[stage]
|
||||
if not stage_kernels:
|
||||
continue
|
||||
total_us = sum(kernel.dur for kernel in stage_kernels)
|
||||
if (
|
||||
stage == "all"
|
||||
and default_stage == "all"
|
||||
and kernel_helpers.pct(total_us, trace_total_us) < MIN_RENDER_SHARE_PCT
|
||||
):
|
||||
continue
|
||||
kernel_stats = kernel_helpers.aggregate(
|
||||
stage_kernels, key_fn=lambda item: item.canonical_name
|
||||
)
|
||||
kernel_categories = {
|
||||
kernel.canonical_name: kernel.category for kernel in stage_kernels
|
||||
}
|
||||
full_kernel_rows = kernel_helpers.build_kernel_rows(
|
||||
stage=stage,
|
||||
kernel_stats=kernel_stats,
|
||||
kernel_categories=kernel_categories,
|
||||
local_stage_payload=formal_stage_payloads.get(stage, {"kernels": {}}),
|
||||
external_kernel_map=mapping_kernel_map,
|
||||
)
|
||||
visible_kernel_rows = kernel_helpers.limit_kernel_rows(
|
||||
full_kernel_rows, args.kernel_table_limit
|
||||
)
|
||||
for row in visible_kernel_rows:
|
||||
share_pct = kernel_helpers.pct(row.total_us, total_us)
|
||||
if share_pct < MIN_RENDER_SHARE_PCT:
|
||||
continue
|
||||
kernel_rows_rendered.append(
|
||||
{
|
||||
"stage": stage,
|
||||
"kernel": row.name,
|
||||
"category": row.category,
|
||||
"total_us": row.total_us,
|
||||
"share_pct": share_pct,
|
||||
"launches": row.aggregate.count,
|
||||
"location": row.location,
|
||||
"cpu_op": row.cpu_op,
|
||||
}
|
||||
)
|
||||
for item in kernel_helpers.detect_fusion_opportunities(
|
||||
kernel_rows=full_kernel_rows,
|
||||
total_us=total_us,
|
||||
server_args=formal_server_args or mapping_server_args,
|
||||
framework=formal_framework,
|
||||
):
|
||||
share_pct = kernel_helpers.pct(item.related_us, total_us)
|
||||
if share_pct < MIN_RENDER_SHARE_PCT:
|
||||
continue
|
||||
fuse_rows_rendered.append(
|
||||
{
|
||||
"stage": stage,
|
||||
"pattern": item.pattern,
|
||||
"confidence": item.confidence,
|
||||
"related_us": item.related_us,
|
||||
"share_pct": share_pct,
|
||||
"evidence": item.evidence,
|
||||
"current_locations": item.current_locations,
|
||||
"candidate_path": item.candidate_path,
|
||||
"rationale": item.rationale,
|
||||
}
|
||||
)
|
||||
|
||||
overlap_rows_rendered: List[dict] = []
|
||||
if not single_trace_mode:
|
||||
mapping_overlap_bundles = build_overlap_stage_bundle_map(
|
||||
mapping_traces,
|
||||
label_prefix="mapping",
|
||||
server_args=mapping_server_args,
|
||||
pid_substring=args.pid_substring,
|
||||
)
|
||||
formal_overlap_bundles = build_overlap_stage_bundle_map(
|
||||
formal_traces,
|
||||
label_prefix="formal",
|
||||
server_args=formal_server_args,
|
||||
pid_substring=args.pid_substring,
|
||||
)
|
||||
for stage in render_stages(formal_overlap_bundles):
|
||||
formal_bundle = pick_stage_value(formal_overlap_bundles, stage)
|
||||
mapping_bundle = pick_stage_value(mapping_overlap_bundles, stage)
|
||||
if formal_bundle is None or mapping_bundle is None:
|
||||
continue
|
||||
formal_bundle.overlap_stats = overlap_helpers.analyze_overlap(
|
||||
formal_bundle.events
|
||||
)
|
||||
aggregates = overlap_helpers.aggregate_events(formal_bundle.events)
|
||||
source_map = overlap_helpers.build_kernel_source_map(
|
||||
mapping_bundle,
|
||||
kernel_map_entry_lookup=lambda stage_name, kernel_name: (
|
||||
kernel_helpers.lookup_kernel_map_entry(
|
||||
mapping_kernel_map, stage_name, kernel_name
|
||||
)
|
||||
if mapping_kernel_map
|
||||
else None
|
||||
),
|
||||
stage=stage,
|
||||
)
|
||||
source_map = overlap_helpers.merge_source_map_from_kernel_payload(
|
||||
source_map,
|
||||
pick_stage_value(formal_stage_payloads, stage),
|
||||
)
|
||||
stage_rows = overlap_helpers.build_action_rows(
|
||||
aggregates,
|
||||
source_map,
|
||||
formal_bundle.events,
|
||||
formal_bundle.overlap_stats["total_busy_us"],
|
||||
table_limit=max(0, args.overlap_table_limit),
|
||||
)
|
||||
for row in stage_rows:
|
||||
if row.share_pct < MIN_RENDER_SHARE_PCT:
|
||||
continue
|
||||
overlap_rows_rendered.append(
|
||||
{
|
||||
"stage": stage,
|
||||
"priority": row.priority,
|
||||
"verdict": row.verdict,
|
||||
"kernel": row.kernel,
|
||||
"python_scope": row.python_scope,
|
||||
"total_us": row.total_us,
|
||||
"share_pct": row.share_pct,
|
||||
"exclusive_ratio": row.exclusive_ratio,
|
||||
"hidden_ratio": row.hidden_ratio,
|
||||
"dependency_signal": row.dependency_signal,
|
||||
"recommendation": row.recommendation,
|
||||
}
|
||||
)
|
||||
|
||||
lines: List[str] = []
|
||||
lines.append("Triage View")
|
||||
lines.append(f"Mode: {'single-trace' if single_trace_mode else 'mapping-formal'}")
|
||||
if single_trace_mode:
|
||||
lines.append(f"Framework: {framework_display_name(formal_framework)}")
|
||||
lines.append(f"Input traces: {', '.join(str(path) for path in formal_traces)}")
|
||||
else:
|
||||
if mapping_framework == formal_framework:
|
||||
lines.append(f"Framework: {framework_display_name(formal_framework)}")
|
||||
else:
|
||||
lines.append(
|
||||
f"Mapping framework: {framework_display_name(mapping_framework)}"
|
||||
)
|
||||
lines.append(
|
||||
f"Formal framework: {framework_display_name(formal_framework)}"
|
||||
)
|
||||
lines.append(
|
||||
f"Mapping traces: {', '.join(str(path) for path in mapping_traces)}"
|
||||
)
|
||||
lines.append(f"Formal traces: {', '.join(str(path) for path in formal_traces)}")
|
||||
if formal_server_args or mapping_server_args:
|
||||
server_args = formal_server_args or mapping_server_args
|
||||
model = server_args.get("model_path") or server_args.get("model")
|
||||
if model:
|
||||
lines.append(f"Model: {model}")
|
||||
lines.append("")
|
||||
lines.append("Kernel Table")
|
||||
lines.extend(render_kernel_tables(kernel_rows_rendered))
|
||||
lines.append("")
|
||||
lines.append("Overlap Opportunity Table")
|
||||
lines.extend(render_overlap_tables(overlap_rows_rendered))
|
||||
lines.append("")
|
||||
lines.append("Fuse Opportunity Table")
|
||||
lines.extend(render_fuse_tables(fuse_rows_rendered))
|
||||
print("\n".join(lines).rstrip())
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
argv = list(argv or sys.argv[1:])
|
||||
triage_parser = build_triage_parser()
|
||||
|
||||
if not argv or argv[0] in {"-h", "--help"}:
|
||||
triage_parser.print_help()
|
||||
return 0
|
||||
|
||||
if argv[0] == "triage":
|
||||
argv = argv[1:]
|
||||
elif not argv[0].startswith("-"):
|
||||
triage_parser.error(
|
||||
"This skill exposes only the triage workflow. "
|
||||
"Use single-trace mode (--input/--url) or mapping+formal two-trace mode."
|
||||
)
|
||||
return 2
|
||||
|
||||
return run_triage(parse_triage_args(argv))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Backwards-compatibility shim for the unified LLM torch-profiler entrypoint.
|
||||
|
||||
The real implementation now lives in ``analyze_llm_torch_profile`` because this
|
||||
skill covers SGLang, vLLM, and TensorRT-LLM. Older scripts and runbooks that
|
||||
still invoke ``analyze_sglang_torch_profile.py`` keep working by forwarding to
|
||||
that module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from analyze_llm_torch_profile import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
"""Generate a TensorRT-LLM py_executor override for stable torch-profiler capture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
START_MARKER = "torch_profiler = torch.profiler.profile("
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProfileCallSpan:
|
||||
start: int
|
||||
end: int
|
||||
block: str
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Create a py_executor.py override that enables with_stack=True for "
|
||||
"TensorRT-LLM torch-profiler traces."
|
||||
)
|
||||
)
|
||||
parser.add_argument("--source", required=True, help="Original py_executor.py path.")
|
||||
parser.add_argument("--output", required=True, help="Override file path to write.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def find_profile_call_span(text: str) -> ProfileCallSpan:
|
||||
start = text.find(START_MARKER)
|
||||
if start == -1:
|
||||
raise SystemExit("Could not find torch profiler setup in source file.")
|
||||
|
||||
open_paren = text.find("(", start)
|
||||
if open_paren == -1:
|
||||
raise SystemExit("Malformed torch profiler setup in source file.")
|
||||
|
||||
depth = 0
|
||||
for index in range(open_paren, len(text)):
|
||||
char = text[index]
|
||||
if char == "(":
|
||||
depth += 1
|
||||
elif char == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return ProfileCallSpan(
|
||||
start=start,
|
||||
end=index + 1,
|
||||
block=text[start : index + 1],
|
||||
)
|
||||
raise SystemExit("Could not find the end of the torch profiler call.")
|
||||
|
||||
|
||||
def inject_with_stack(block: str) -> str:
|
||||
if "with_stack=" in block:
|
||||
return block
|
||||
|
||||
lines = block.splitlines()
|
||||
if not lines:
|
||||
raise SystemExit("Unexpected torch profiler block format.")
|
||||
|
||||
last_line = lines[-1]
|
||||
if not last_line.strip():
|
||||
raise SystemExit("Unexpected torch profiler block terminator.")
|
||||
|
||||
if last_line.strip() == ")":
|
||||
if len(lines) < 2:
|
||||
raise SystemExit("Could not find the last torch profiler argument line.")
|
||||
last_arg_index = len(lines) - 2
|
||||
last_arg_line = lines[last_arg_index]
|
||||
indent = last_arg_line[: len(last_arg_line) - len(last_arg_line.lstrip())]
|
||||
if not last_arg_line.rstrip().endswith(","):
|
||||
lines[last_arg_index] = last_arg_line.rstrip() + ","
|
||||
lines.insert(len(lines) - 1, f"{indent}with_stack=True")
|
||||
return "\n".join(lines)
|
||||
|
||||
if not last_line.rstrip().endswith(")"):
|
||||
raise SystemExit("Unexpected torch profiler block terminator.")
|
||||
|
||||
indent = last_line[: len(last_line) - len(last_line.lstrip())]
|
||||
last_arg_text = last_line.rstrip()[:-1].rstrip()
|
||||
if not last_arg_text.endswith(","):
|
||||
last_arg_text += ","
|
||||
lines[-1] = last_arg_text
|
||||
lines.append(f"{indent}with_stack=True)")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def inject_rank0_trace_guard(text: str) -> str:
|
||||
needle = (
|
||||
" enable_torch_trace = bool(torch_trace_path and profile_start_stop)\n"
|
||||
)
|
||||
replacement = (
|
||||
" # Multi-rank PyTorch backend workers race on the same chrome-trace "
|
||||
"path.\n"
|
||||
" # Keep the full torch-profiler trace on rank 0 and let the other "
|
||||
"ranks\n"
|
||||
" # continue with CUDA-profiler gating only.\n"
|
||||
" enable_torch_trace = bool(\n"
|
||||
" torch_trace_path and profile_start_stop and self.dist.rank == 0\n"
|
||||
" )\n"
|
||||
)
|
||||
if replacement in text:
|
||||
return text
|
||||
if needle not in text:
|
||||
raise SystemExit("Could not find enable_torch_trace assignment in source file.")
|
||||
return text.replace(needle, replacement, 1)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
source = Path(args.source).expanduser().resolve()
|
||||
output = Path(args.output).expanduser().resolve()
|
||||
text = source.read_text(encoding="utf-8")
|
||||
span = find_profile_call_span(text)
|
||||
patched_block = inject_with_stack(span.block)
|
||||
patched = (
|
||||
text
|
||||
if patched_block == span.block
|
||||
else (text[: span.start] + patched_block + text[span.end :])
|
||||
)
|
||||
patched = inject_rank0_trace_guard(patched)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(patched, encoding="utf-8")
|
||||
print(output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a small correctness and latency probe against an LLM server."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib import request
|
||||
|
||||
from profile_common import extract_openai_chat_text
|
||||
|
||||
DEFAULT_PROMPTS = [
|
||||
"用一句中文介绍上海。",
|
||||
"What is 2+2? Answer briefly.",
|
||||
"Write one short haiku about GPUs.",
|
||||
]
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Send a few short requests to an LLM server and record latency plus "
|
||||
"sample outputs."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--framework",
|
||||
required=True,
|
||||
choices=("sglang", "vllm", "trtllm"),
|
||||
help="Serving framework.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--url",
|
||||
required=True,
|
||||
help="Server base URL, for example http://127.0.0.1:30000.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default=None,
|
||||
help="OpenAI model id. Auto-discovered for vLLM and TensorRT-LLM when omitted.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--requests",
|
||||
type=int,
|
||||
default=6,
|
||||
help="How many probe requests to send.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-tokens",
|
||||
type=int,
|
||||
default=48,
|
||||
help="Generation length for each request.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=180.0,
|
||||
help="Per-request timeout in seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Optional prompt override. Repeat to add more prompts.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=None,
|
||||
help="Optional JSON output path.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def post_json(url: str, payload: Dict[str, Any], timeout: float) -> Dict[str, Any]:
|
||||
req = request.Request(
|
||||
url=url,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
return json.loads(raw.decode("utf-8")) if raw else {}
|
||||
|
||||
|
||||
def get_json(url: str, timeout: float) -> Dict[str, Any]:
|
||||
req = request.Request(url=url, method="GET")
|
||||
with request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
return json.loads(raw.decode("utf-8")) if raw else {}
|
||||
|
||||
|
||||
def discover_openai_model(base_url: str, timeout: float) -> str:
|
||||
payload = get_json(base_url.rstrip("/") + "/v1/models", timeout=timeout)
|
||||
data = payload.get("data")
|
||||
if not isinstance(data, list) or not data:
|
||||
raise RuntimeError(f"No models returned by {base_url.rstrip('/')}/v1/models")
|
||||
first = data[0]
|
||||
if isinstance(first, dict) and first.get("id"):
|
||||
return str(first["id"])
|
||||
raise RuntimeError(f"Malformed /v1/models payload from {base_url.rstrip('/')}")
|
||||
|
||||
|
||||
def p95(values: List[float]) -> Optional[float]:
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
index = max(0, math.ceil(len(ordered) * 0.95) - 1)
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def sglang_request(base_url: str, prompt: str, max_tokens: int, timeout: float) -> str:
|
||||
payload = {
|
||||
"text": prompt,
|
||||
"sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"max_new_tokens": max_tokens,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
body = post_json(base_url.rstrip("/") + "/generate", payload, timeout=timeout)
|
||||
return str(body.get("text", ""))
|
||||
|
||||
|
||||
def openai_request(
|
||||
base_url: str,
|
||||
model: str,
|
||||
prompt: str,
|
||||
max_tokens: int,
|
||||
timeout: float,
|
||||
) -> Dict[str, str]:
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.0,
|
||||
"max_tokens": max_tokens,
|
||||
"stream": False,
|
||||
}
|
||||
body = post_json(
|
||||
base_url.rstrip("/") + "/v1/chat/completions",
|
||||
payload,
|
||||
timeout=timeout,
|
||||
)
|
||||
text, source = extract_openai_chat_text(body)
|
||||
return {"text": text, "source": source}
|
||||
|
||||
|
||||
def run_probe(args: argparse.Namespace) -> Dict[str, Any]:
|
||||
prompts = args.prompt or list(DEFAULT_PROMPTS)
|
||||
model = args.model
|
||||
if args.framework in {"vllm", "trtllm"} and not model:
|
||||
model = discover_openai_model(args.url, timeout=args.timeout)
|
||||
|
||||
latencies: List[float] = []
|
||||
samples: List[Dict[str, Any]] = []
|
||||
errors: List[Dict[str, str]] = []
|
||||
|
||||
for request_idx in range(args.requests):
|
||||
prompt = prompts[request_idx % len(prompts)]
|
||||
start = time.time()
|
||||
try:
|
||||
if args.framework == "sglang":
|
||||
text = sglang_request(
|
||||
args.url,
|
||||
prompt,
|
||||
max_tokens=args.max_tokens,
|
||||
timeout=args.timeout,
|
||||
)
|
||||
source = "generate.text"
|
||||
else:
|
||||
assert model is not None
|
||||
result = openai_request(
|
||||
args.url,
|
||||
model,
|
||||
prompt,
|
||||
max_tokens=args.max_tokens,
|
||||
timeout=args.timeout,
|
||||
)
|
||||
text = result["text"]
|
||||
source = result["source"]
|
||||
elapsed = time.time() - start
|
||||
latencies.append(elapsed)
|
||||
samples.append(
|
||||
{
|
||||
"prompt": prompt,
|
||||
"latency_s": round(elapsed, 3),
|
||||
"content": text[:240],
|
||||
"source": source,
|
||||
"non_empty": bool(text.strip()),
|
||||
}
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - runtime probe path
|
||||
errors.append({"prompt": prompt, "error": repr(exc)})
|
||||
|
||||
return {
|
||||
"framework": args.framework,
|
||||
"url": args.url,
|
||||
"model": model,
|
||||
"requests": args.requests,
|
||||
"success": len(samples),
|
||||
"errors": len(errors),
|
||||
"all_non_empty": (
|
||||
all(sample["non_empty"] for sample in samples) if samples else False
|
||||
),
|
||||
"avg_latency_s": round(statistics.mean(latencies), 3) if latencies else None,
|
||||
"p95_latency_s": round(p95(latencies), 3) if latencies else None,
|
||||
"samples": samples[:3],
|
||||
"error_samples": errors[:3],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
summary = run_probe(args)
|
||||
rendered = json.dumps(summary, ensure_ascii=False, indent=2)
|
||||
print(rendered)
|
||||
if args.output:
|
||||
output_path = Path(args.output).expanduser().resolve()
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(rendered + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,880 @@
|
||||
"""Shared helpers for unified LLM torch-profiler skill scripts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections import Counter, defaultdict
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, Iterable, List, Optional, Sequence, Tuple
|
||||
from urllib import request
|
||||
|
||||
STAGE_ORDER = {"extend": 0, "prefill": 0, "decode": 1, "all": 2}
|
||||
FRAMEWORK_LABELS = {
|
||||
"auto": "auto",
|
||||
"sglang": "SGLang",
|
||||
"vllm": "vLLM",
|
||||
"trtllm": "TensorRT-LLM",
|
||||
}
|
||||
TRACE_FILE_PATTERNS = (
|
||||
"*.trace.json",
|
||||
"*.trace.json.gz",
|
||||
"*.pt.trace.json",
|
||||
"*.pt.trace.json.gz",
|
||||
"*.json",
|
||||
"*.json.gz",
|
||||
)
|
||||
TRACE_FILE_IGNORE_NAMES = {
|
||||
"server_args.json",
|
||||
"metadata.json",
|
||||
"config.json",
|
||||
}
|
||||
TRACE_METADATA_NAMES = {
|
||||
"process_name",
|
||||
"thread_name",
|
||||
"process_sort_index",
|
||||
"thread_sort_index",
|
||||
}
|
||||
NON_KERNEL_TRACE_CATEGORIES = ("python_function", "cpu_op", "trace")
|
||||
PYTHON_SCOPE_NAME_PREFIXES = ("python/", "nn.module:")
|
||||
|
||||
|
||||
@lru_cache(maxsize=65536)
|
||||
def _normalize_text_cached(text: str) -> str:
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return ""
|
||||
for token in (" ", "\t", "\n", "\r", "\v", "\f"):
|
||||
if token in text:
|
||||
return " ".join(text.split())
|
||||
return text
|
||||
|
||||
|
||||
def normalize_text(value: object) -> str:
|
||||
return _normalize_text_cached(value if isinstance(value, str) else str(value))
|
||||
|
||||
|
||||
def canonicalize_framework(value: object) -> str:
|
||||
lowered = normalize_text(value).lower().replace("_", "-")
|
||||
aliases = {
|
||||
"": "auto",
|
||||
"auto": "auto",
|
||||
"sglang": "sglang",
|
||||
"sgl": "sglang",
|
||||
"vllm": "vllm",
|
||||
"trt": "trtllm",
|
||||
"tllm": "trtllm",
|
||||
"trtllm": "trtllm",
|
||||
"tensorrt-llm": "trtllm",
|
||||
"tensorrtllm": "trtllm",
|
||||
}
|
||||
return aliases.get(lowered, "auto")
|
||||
|
||||
|
||||
def framework_display_name(value: object) -> str:
|
||||
return FRAMEWORK_LABELS.get(canonicalize_framework(value), str(value))
|
||||
|
||||
|
||||
@lru_cache(maxsize=65536)
|
||||
def _normalize_repo_relative_path_cached(text: str) -> str:
|
||||
text = text.replace("\\", "/")
|
||||
lowered = text.lower()
|
||||
for marker, normalized_marker in (
|
||||
("python/sglang/", "python/sglang/"),
|
||||
("sgl_kernel/", "sgl_kernel/"),
|
||||
("vllm/", "vllm/"),
|
||||
("tensorrt_llm/", "tensorrt_llm/"),
|
||||
("tensorrt-llm/", "tensorrt_llm/"),
|
||||
):
|
||||
idx = lowered.find(marker)
|
||||
if idx != -1:
|
||||
suffix = text[idx + len(marker) :].lstrip("/")
|
||||
return f"{normalized_marker}{suffix}".lstrip("/")
|
||||
idx = lowered.find("sglang/")
|
||||
if idx != -1:
|
||||
return ("python/" + text[idx:]).lstrip("/")
|
||||
return text.lstrip("/")
|
||||
|
||||
|
||||
def normalize_repo_relative_path(path: object) -> str:
|
||||
return _normalize_repo_relative_path_cached(normalize_text(path))
|
||||
|
||||
|
||||
def contains_any_keyword(text: str, keywords: Iterable[str]) -> bool:
|
||||
return any(keyword in text for keyword in keywords)
|
||||
|
||||
|
||||
def coerce_optional_int(value: object) -> Optional[int]:
|
||||
if value in (None, "", "None"):
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value) if value.is_integer() else None
|
||||
try:
|
||||
return int(str(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def extract_trace_events(trace: object) -> Sequence[dict]:
|
||||
if isinstance(trace, dict):
|
||||
events = trace.get("traceEvents", [])
|
||||
return events if isinstance(events, list) else []
|
||||
if isinstance(trace, list):
|
||||
return trace
|
||||
return []
|
||||
|
||||
|
||||
def is_trace_metadata_name(name: object) -> bool:
|
||||
return str(name) in TRACE_METADATA_NAMES
|
||||
|
||||
|
||||
def is_complete_duration_event(event: dict) -> bool:
|
||||
if event.get("ph") != "X":
|
||||
return False
|
||||
dur = event.get("dur")
|
||||
ts = event.get("ts")
|
||||
if dur is None or ts is None:
|
||||
return False
|
||||
try:
|
||||
return float(dur) > 0
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def is_annotation_event(name: object, category: object) -> bool:
|
||||
lowered_name = normalize_text(name).lower()
|
||||
lowered_category = normalize_text(category).lower()
|
||||
return "annotation" in lowered_category or lowered_name.startswith("## call ")
|
||||
|
||||
|
||||
def is_non_kernel_trace_category(category: object) -> bool:
|
||||
lowered_category = normalize_text(category).lower()
|
||||
return any(token in lowered_category for token in NON_KERNEL_TRACE_CATEGORIES)
|
||||
|
||||
|
||||
def looks_like_python_scope_name(name: object) -> bool:
|
||||
lowered_name = normalize_text(name).lower()
|
||||
return ".py(" in lowered_name or lowered_name.startswith(PYTHON_SCOPE_NAME_PREFIXES)
|
||||
|
||||
|
||||
def has_stream_marker(args: Optional[dict]) -> bool:
|
||||
trace_args = args or {}
|
||||
return "stream" in trace_args or "cuda_stream" in trace_args
|
||||
|
||||
|
||||
def load_trace_json(path: Path) -> dict:
|
||||
if path.suffix == ".gz":
|
||||
with gzip.open(path, "rt", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def load_server_args(path: Path) -> Optional[dict]:
|
||||
resolved = path.resolve()
|
||||
candidate_dirs: List[Path] = []
|
||||
if resolved.is_file():
|
||||
candidate_dirs.extend([resolved.parent, resolved.parent.parent])
|
||||
else:
|
||||
candidate_dirs.extend([resolved, resolved.parent])
|
||||
|
||||
seen: set[Path] = set()
|
||||
for candidate_dir in candidate_dirs:
|
||||
if candidate_dir in seen:
|
||||
continue
|
||||
seen.add(candidate_dir)
|
||||
candidate = candidate_dir / "server_args.json"
|
||||
if candidate.exists():
|
||||
with open(candidate, "r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
return None
|
||||
|
||||
|
||||
def try_get_json(url: str, timeout: float = 60.0) -> Optional[object]:
|
||||
try:
|
||||
with request.urlopen(url, timeout=timeout) as response:
|
||||
raw = response.read()
|
||||
except Exception:
|
||||
return None
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _flatten_chat_text_parts(value: object) -> List[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
return [text] if text else []
|
||||
if isinstance(value, list):
|
||||
parts: List[str] = []
|
||||
for item in value:
|
||||
parts.extend(_flatten_chat_text_parts(item))
|
||||
return parts
|
||||
if isinstance(value, dict):
|
||||
parts: List[str] = []
|
||||
text_keys = (
|
||||
"text",
|
||||
"content",
|
||||
"reasoning_content",
|
||||
"reasoning",
|
||||
"output_text",
|
||||
)
|
||||
if any(key in value for key in text_keys):
|
||||
for key in text_keys:
|
||||
parts.extend(_flatten_chat_text_parts(value.get(key)))
|
||||
if parts:
|
||||
return parts
|
||||
item_type = normalize_text(value.get("type")).lower()
|
||||
if item_type in {"text", "output_text", "input_text"}:
|
||||
for key in ("text", "content", "value"):
|
||||
parts.extend(_flatten_chat_text_parts(value.get(key)))
|
||||
elif item_type in {"reasoning", "thinking"}:
|
||||
for key in ("text", "content", "reasoning_content", "reasoning"):
|
||||
parts.extend(_flatten_chat_text_parts(value.get(key)))
|
||||
return parts
|
||||
return []
|
||||
|
||||
|
||||
def flatten_chat_text(value: object) -> str:
|
||||
return "\n".join(_flatten_chat_text_parts(value)).strip()
|
||||
|
||||
|
||||
def extract_openai_chat_text(body: object) -> Tuple[str, str]:
|
||||
if not isinstance(body, dict):
|
||||
return "", "invalid_body"
|
||||
|
||||
choices = body.get("choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
fallback = flatten_chat_text(body.get("output_text"))
|
||||
if fallback:
|
||||
return fallback, "body.output_text"
|
||||
return "", "missing_choices"
|
||||
|
||||
first_choice = choices[0]
|
||||
if not isinstance(first_choice, dict):
|
||||
return "", "invalid_choice"
|
||||
|
||||
message = first_choice.get("message")
|
||||
if isinstance(message, dict):
|
||||
for key in ("content", "reasoning_content", "reasoning"):
|
||||
text = flatten_chat_text(message.get(key))
|
||||
if text:
|
||||
return text, f"message.{key}"
|
||||
|
||||
for key in ("text", "content", "reasoning_content", "reasoning"):
|
||||
text = flatten_chat_text(first_choice.get(key))
|
||||
if text:
|
||||
return text, f"choice.{key}"
|
||||
|
||||
delta = first_choice.get("delta")
|
||||
if isinstance(delta, dict):
|
||||
for key in ("content", "reasoning_content", "reasoning"):
|
||||
text = flatten_chat_text(delta.get(key))
|
||||
if text:
|
||||
return text, f"delta.{key}"
|
||||
|
||||
fallback = flatten_chat_text(body.get("output_text"))
|
||||
if fallback:
|
||||
return fallback, "body.output_text"
|
||||
return "", "empty"
|
||||
|
||||
|
||||
def detect_framework_from_text(text: object) -> Optional[str]:
|
||||
lowered = normalize_text(text).lower()
|
||||
if not lowered:
|
||||
return None
|
||||
if any(
|
||||
token in lowered
|
||||
for token in (
|
||||
"tensorrt_llm",
|
||||
"tensorrt-llm",
|
||||
"trtllm",
|
||||
"pyexecutor",
|
||||
)
|
||||
):
|
||||
return "trtllm"
|
||||
if "vllm" in lowered:
|
||||
return "vllm"
|
||||
if any(token in lowered for token in ("python/sglang/", "sgl_kernel/", "sglang/")):
|
||||
return "sglang"
|
||||
return None
|
||||
|
||||
|
||||
def detect_framework_from_server_args(server_args: Optional[dict]) -> Optional[str]:
|
||||
if not isinstance(server_args, dict) or not server_args:
|
||||
return None
|
||||
lowered_keys = {normalize_text(key).lower() for key in server_args}
|
||||
if lowered_keys & {
|
||||
"attention_backend",
|
||||
"sampling_backend",
|
||||
"disable_cuda_graph",
|
||||
"disable_piecewise_cuda_graph",
|
||||
"chunked_prefill_size",
|
||||
"schedule_policy",
|
||||
}:
|
||||
return "sglang"
|
||||
return detect_framework_from_text(json.dumps(server_args, sort_keys=True))
|
||||
|
||||
|
||||
def detect_framework_from_trace(trace: object) -> Optional[str]:
|
||||
text_samples: List[str] = []
|
||||
for event in extract_trace_events(trace)[:256]:
|
||||
text_samples.extend(
|
||||
[
|
||||
str(event.get("name", "")),
|
||||
str(event.get("cat", "")),
|
||||
str(event.get("pid", "")),
|
||||
]
|
||||
)
|
||||
trace_args = event.get("args")
|
||||
if isinstance(trace_args, dict):
|
||||
for key, value in list(trace_args.items())[:8]:
|
||||
text_samples.append(str(key))
|
||||
if isinstance(value, str):
|
||||
text_samples.append(value)
|
||||
return detect_framework_from_text(" ".join(text_samples))
|
||||
|
||||
|
||||
def detect_framework_from_path(path: Path) -> Optional[str]:
|
||||
hint = detect_framework_from_text(str(path))
|
||||
if hint:
|
||||
return hint
|
||||
server_args = load_server_args(path)
|
||||
hint = detect_framework_from_server_args(server_args)
|
||||
if hint:
|
||||
return hint
|
||||
if path.is_file():
|
||||
try:
|
||||
return detect_framework_from_trace(load_trace_json(path))
|
||||
except Exception:
|
||||
return None
|
||||
trace_files = discover_trace_files(path, recursive=True, limit=3)
|
||||
for trace_file in trace_files:
|
||||
try:
|
||||
hint = detect_framework_from_trace(load_trace_json(trace_file))
|
||||
except Exception:
|
||||
hint = None
|
||||
if hint:
|
||||
return hint
|
||||
return None
|
||||
|
||||
|
||||
def detect_framework_from_url(
|
||||
url: str, output_dir: Optional[str] = None
|
||||
) -> Optional[str]:
|
||||
hint = detect_framework_from_text(output_dir or "")
|
||||
if hint:
|
||||
return hint
|
||||
server_info = try_get_json(url.rstrip("/") + "/server_info")
|
||||
if isinstance(server_info, dict) and (
|
||||
"internal_states" in server_info
|
||||
or "tokenizer_path" in server_info
|
||||
or "prefill" in server_info
|
||||
or "decode" in server_info
|
||||
):
|
||||
return "sglang"
|
||||
models = try_get_json(url.rstrip("/") + "/v1/models")
|
||||
if isinstance(models, dict) and isinstance(models.get("data"), list):
|
||||
return "vllm"
|
||||
return None
|
||||
|
||||
|
||||
def resolve_framework(
|
||||
requested: object,
|
||||
*,
|
||||
input_path: Optional[Path] = None,
|
||||
url: Optional[str] = None,
|
||||
server_args: Optional[dict] = None,
|
||||
) -> str:
|
||||
explicit = canonicalize_framework(requested)
|
||||
if explicit != "auto":
|
||||
return explicit
|
||||
for hint in (
|
||||
detect_framework_from_server_args(server_args),
|
||||
detect_framework_from_path(input_path) if input_path else None,
|
||||
(
|
||||
detect_framework_from_url(url, str(input_path) if input_path else None)
|
||||
if url
|
||||
else None
|
||||
),
|
||||
):
|
||||
if hint:
|
||||
return hint
|
||||
return "sglang"
|
||||
|
||||
|
||||
def parse_stage(path: Path) -> str:
|
||||
name = path.name.lower()
|
||||
if "-extend" in name or "-prefill" in name:
|
||||
return "extend"
|
||||
if "-decode" in name:
|
||||
return "decode"
|
||||
return "all"
|
||||
|
||||
|
||||
def parse_tp_rank(path: Path) -> Optional[int]:
|
||||
for pattern in (
|
||||
r"(?:^|[_-])tp(\d+)(?:[_.-]|$)",
|
||||
r"TP-(\d+)",
|
||||
r"(?:^|[_-])rank(\d+)(?:[_.-]|$)",
|
||||
r"(?:^|[_-])worker(\d+)(?:[_.-]|$)",
|
||||
):
|
||||
match = re.search(pattern, path.name, re.IGNORECASE)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def file_looks_like_trace(path: Path) -> bool:
|
||||
name = path.name.lower()
|
||||
if name in TRACE_FILE_IGNORE_NAMES:
|
||||
return False
|
||||
if path.is_dir():
|
||||
return False
|
||||
if any(name.endswith(suffix) for suffix in (".trace.json", ".trace.json.gz")):
|
||||
return True
|
||||
if ".pt.trace.json" in name:
|
||||
return True
|
||||
if not any(name.endswith(suffix) for suffix in (".json", ".json.gz")):
|
||||
return False
|
||||
try:
|
||||
trace = load_trace_json(path)
|
||||
except Exception:
|
||||
return False
|
||||
if isinstance(trace, dict):
|
||||
return isinstance(trace.get("traceEvents"), list)
|
||||
if isinstance(trace, list):
|
||||
return bool(trace) and all(isinstance(item, dict) for item in trace[:8])
|
||||
return False
|
||||
|
||||
|
||||
def discover_trace_files(
|
||||
path: Path,
|
||||
*,
|
||||
recursive: bool,
|
||||
limit: Optional[int] = None,
|
||||
) -> List[Path]:
|
||||
if path.is_file():
|
||||
return [path] if file_looks_like_trace(path) else []
|
||||
|
||||
candidates: List[Path] = []
|
||||
seen: set[Path] = set()
|
||||
for pattern in TRACE_FILE_PATTERNS:
|
||||
iterator = path.rglob(pattern) if recursive else path.glob(pattern)
|
||||
for candidate in iterator:
|
||||
resolved = candidate.resolve()
|
||||
if resolved in seen:
|
||||
continue
|
||||
seen.add(resolved)
|
||||
candidates.append(resolved)
|
||||
candidates = [
|
||||
candidate
|
||||
for candidate in candidates
|
||||
if candidate.exists() and file_looks_like_trace(candidate)
|
||||
]
|
||||
candidates.sort(key=lambda item: item.stat().st_mtime)
|
||||
if limit is not None and limit >= 0:
|
||||
return candidates[-limit:] if limit else []
|
||||
return candidates
|
||||
|
||||
|
||||
def newest_trace_dir(path: Path) -> Path:
|
||||
if path.is_file():
|
||||
return path.parent
|
||||
direct = discover_trace_files(path, recursive=False)
|
||||
if direct:
|
||||
return path
|
||||
traces = discover_trace_files(path, recursive=True)
|
||||
trace_dirs = list({trace.parent for trace in traces})
|
||||
if not trace_dirs:
|
||||
raise FileNotFoundError(f"No trace files found under {path}")
|
||||
trace_dirs.sort(
|
||||
key=lambda item: max(
|
||||
trace.stat().st_mtime for trace in traces if trace.parent == item
|
||||
)
|
||||
)
|
||||
return trace_dirs[-1]
|
||||
|
||||
|
||||
def discover_trace_targets(
|
||||
path: Path, all_traces: bool
|
||||
) -> Tuple[List[Path], Optional[dict]]:
|
||||
if path.is_file():
|
||||
return [path], load_server_args(path)
|
||||
|
||||
trace_dir = newest_trace_dir(path)
|
||||
traces = discover_trace_files(trace_dir, recursive=False)
|
||||
if not traces:
|
||||
raise FileNotFoundError(f"No trace files found under {trace_dir}")
|
||||
|
||||
non_merged = [trace for trace in traces if not trace.name.startswith("merged-")]
|
||||
selected = non_merged or traces
|
||||
if not all_traces:
|
||||
ranks = sorted(
|
||||
{
|
||||
rank
|
||||
for rank in (parse_tp_rank(trace) for trace in selected)
|
||||
if rank is not None
|
||||
}
|
||||
)
|
||||
if ranks:
|
||||
rank = 0 if 0 in ranks else ranks[0]
|
||||
selected = [trace for trace in selected if parse_tp_rank(trace) == rank]
|
||||
grouped: Dict[str, List[Path]] = defaultdict(list)
|
||||
for trace in selected:
|
||||
grouped[parse_stage(trace)].append(trace)
|
||||
selected = [
|
||||
sorted(group, key=lambda item: item.stat().st_mtime)[-1]
|
||||
for group in grouped.values()
|
||||
]
|
||||
|
||||
selected.sort(key=lambda item: (STAGE_ORDER.get(parse_stage(item), 99), item.name))
|
||||
return selected, load_server_args(trace_dir)
|
||||
|
||||
|
||||
def post_json(
|
||||
url: str, payload: Optional[dict] = None, timeout: float = 60.0
|
||||
) -> Optional[dict]:
|
||||
req = request.Request(
|
||||
url=url,
|
||||
data=(None if payload is None else json.dumps(payload).encode("utf-8")),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with request.urlopen(req, timeout=timeout) as response:
|
||||
raw = response.read()
|
||||
return json.loads(raw.decode("utf-8")) if raw else None
|
||||
|
||||
|
||||
def send_probe_request(
|
||||
url: str,
|
||||
prompt: str,
|
||||
max_new_tokens: int,
|
||||
sampling_seed: int,
|
||||
framework: str,
|
||||
model: Optional[str] = None,
|
||||
) -> None:
|
||||
framework = canonicalize_framework(framework)
|
||||
if framework == "sglang":
|
||||
payload = {
|
||||
"text": prompt,
|
||||
"sampling_params": {
|
||||
"sampling_seed": sampling_seed,
|
||||
"temperature": 0.0,
|
||||
"max_new_tokens": max_new_tokens,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
post_json(url.rstrip("/") + "/generate", payload, timeout=300.0)
|
||||
return
|
||||
|
||||
resolved_model = model or discover_openai_model(url)
|
||||
chat_payload = {
|
||||
"model": resolved_model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0.0,
|
||||
"max_tokens": max_new_tokens,
|
||||
"stream": False,
|
||||
}
|
||||
try:
|
||||
post_json(url.rstrip("/") + "/v1/chat/completions", chat_payload, timeout=300.0)
|
||||
return
|
||||
except Exception:
|
||||
completion_payload = {
|
||||
"model": resolved_model,
|
||||
"prompt": prompt,
|
||||
"temperature": 0.0,
|
||||
"max_tokens": max_new_tokens,
|
||||
"stream": False,
|
||||
}
|
||||
post_json(
|
||||
url.rstrip("/") + "/v1/completions",
|
||||
completion_payload,
|
||||
timeout=300.0,
|
||||
)
|
||||
|
||||
|
||||
def discover_openai_model(url: str) -> str:
|
||||
payload = try_get_json(url.rstrip("/") + "/v1/models", timeout=60.0)
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError(f"Could not read {url.rstrip('/')}/v1/models")
|
||||
data = payload.get("data")
|
||||
if not isinstance(data, list) or not data:
|
||||
raise RuntimeError(f"No models returned by {url.rstrip('/')}/v1/models")
|
||||
first = data[0]
|
||||
if isinstance(first, dict) and first.get("id"):
|
||||
return str(first["id"])
|
||||
raise RuntimeError(f"Malformed /v1/models payload from {url.rstrip('/')}")
|
||||
|
||||
|
||||
def ensure_remote_profiler_output_path(
|
||||
output_dir: Optional[str], framework: str
|
||||
) -> Path:
|
||||
if not output_dir:
|
||||
raise ValueError(
|
||||
f"{framework_display_name(framework)} live capture requires --output-dir "
|
||||
"to point at the server-side torch profiler trace path that is visible "
|
||||
"from this machine."
|
||||
)
|
||||
output_path = Path(output_dir).expanduser().resolve()
|
||||
if output_path.suffix in {".json", ".gz"}:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
else:
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
return output_path
|
||||
|
||||
|
||||
def wait_for_profiler_artifact(path: Path, timeout_s: float = 60.0) -> Path:
|
||||
deadline = time.time() + timeout_s
|
||||
while time.time() < deadline:
|
||||
if path.is_file() and file_looks_like_trace(path):
|
||||
return path
|
||||
if path.exists():
|
||||
trace_files = discover_trace_files(path, recursive=True)
|
||||
if trace_files:
|
||||
return newest_trace_dir(path)
|
||||
if path.is_dir():
|
||||
child_dirs = [item for item in path.iterdir() if item.is_dir()]
|
||||
if child_dirs:
|
||||
child_dirs.sort(key=lambda item: item.stat().st_mtime)
|
||||
newest_child = child_dirs[-1]
|
||||
child_traces = discover_trace_files(newest_child, recursive=True)
|
||||
if child_traces:
|
||||
return newest_child
|
||||
time.sleep(0.5)
|
||||
return path
|
||||
|
||||
|
||||
def start_remote_profiler(url: str, framework: str) -> None:
|
||||
try:
|
||||
post_json(url.rstrip("/") + "/start_profile", timeout=60.0)
|
||||
except Exception as exc:
|
||||
if framework == "vllm":
|
||||
raise RuntimeError(
|
||||
"vLLM live torch profiling requires the server to be launched with "
|
||||
'--profiler-config \'{"profiler":"torch","torch_profiler_dir":"..."}\' '
|
||||
"and to expose POST /start_profile."
|
||||
) from exc
|
||||
if framework == "trtllm":
|
||||
raise RuntimeError(
|
||||
"TensorRT-LLM live torch profiling requires "
|
||||
"a server build that exposes POST /start_profile plus the env vars "
|
||||
"TLLM_PROFILE_START_STOP=1 and TLLM_TORCH_PROFILE_TRACE=/shared/path."
|
||||
) from exc
|
||||
raise
|
||||
|
||||
|
||||
def stop_remote_profiler(url: str, framework: str) -> None:
|
||||
try:
|
||||
post_json(url.rstrip("/") + "/stop_profile", timeout=300.0)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"Failed to stop {framework_display_name(framework)} profiler via "
|
||||
f"{url.rstrip('/')}/stop_profile"
|
||||
) from exc
|
||||
|
||||
|
||||
def run_remote_profiler(
|
||||
url: str,
|
||||
output_dir: Optional[str],
|
||||
framework: str,
|
||||
probe_requests: int,
|
||||
probe_prompt: str,
|
||||
probe_max_new_tokens: Optional[int],
|
||||
probe_delay: float,
|
||||
num_steps: int,
|
||||
) -> Path:
|
||||
framework = canonicalize_framework(framework)
|
||||
output_path = ensure_remote_profiler_output_path(output_dir, framework)
|
||||
start_remote_profiler(url, framework)
|
||||
stop_error: Optional[BaseException] = None
|
||||
try:
|
||||
if probe_requests > 0:
|
||||
# Some profiler endpoints need a brief setup window after
|
||||
# POST /start_profile. A very short delay can send probes too early
|
||||
# and miss the profiling window entirely.
|
||||
time.sleep(max(5.0, probe_delay))
|
||||
effective_max_new_tokens = probe_max_new_tokens or max(64, num_steps * 8)
|
||||
model = (
|
||||
discover_openai_model(url) if framework in {"vllm", "trtllm"} else None
|
||||
)
|
||||
for request_idx in range(probe_requests):
|
||||
send_probe_request(
|
||||
url=url,
|
||||
prompt=probe_prompt,
|
||||
max_new_tokens=effective_max_new_tokens,
|
||||
sampling_seed=request_idx,
|
||||
framework=framework,
|
||||
model=model,
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
stop_remote_profiler(url, framework)
|
||||
except BaseException as exc: # pragma: no cover - preserve original failure
|
||||
stop_error = exc
|
||||
if stop_error is not None:
|
||||
raise stop_error
|
||||
return wait_for_profiler_artifact(output_path)
|
||||
|
||||
|
||||
def run_sglang_profiler(
|
||||
url: str,
|
||||
output_dir: Optional[str],
|
||||
num_steps: int,
|
||||
profile_by_stage: bool,
|
||||
merge_profiles: bool,
|
||||
profile_prefix: Optional[str],
|
||||
probe_requests: int,
|
||||
probe_prompt: str,
|
||||
probe_max_new_tokens: Optional[int],
|
||||
probe_delay: float,
|
||||
start_step: Optional[int] = None,
|
||||
) -> Path:
|
||||
if output_dir is None:
|
||||
output_dir = tempfile.mkdtemp(prefix="sglang-torch-profile-")
|
||||
output_root = Path(output_dir).resolve()
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
output_path = output_root / str(time.time())
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
server_args = try_get_json(url.rstrip("/") + "/server_info", timeout=60.0)
|
||||
if server_args is not None:
|
||||
with open(output_path / "server_args.json", "w", encoding="utf-8") as handle:
|
||||
json.dump(server_args, handle)
|
||||
|
||||
payload = {
|
||||
"output_dir": str(output_path),
|
||||
"num_steps": str(num_steps),
|
||||
"activities": ["CPU", "GPU"],
|
||||
"profile_by_stage": profile_by_stage,
|
||||
"merge_profiles": merge_profiles,
|
||||
"profile_prefix": profile_prefix,
|
||||
}
|
||||
if start_step is not None:
|
||||
payload["start_step"] = str(start_step)
|
||||
|
||||
req = request.Request(
|
||||
url.rstrip("/") + "/start_profile",
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with request.urlopen(req, timeout=300.0):
|
||||
pass
|
||||
|
||||
if probe_requests > 0:
|
||||
time.sleep(max(0.0, probe_delay))
|
||||
effective_max_new_tokens = probe_max_new_tokens or max(64, num_steps * 8)
|
||||
for request_idx in range(probe_requests):
|
||||
send_probe_request(
|
||||
url=url,
|
||||
prompt=probe_prompt,
|
||||
max_new_tokens=effective_max_new_tokens,
|
||||
sampling_seed=request_idx,
|
||||
framework="sglang",
|
||||
)
|
||||
|
||||
return wait_for_profiler_artifact(output_path, timeout_s=180.0)
|
||||
|
||||
|
||||
def run_profiler(
|
||||
url: str,
|
||||
output_dir: Optional[str],
|
||||
num_steps: int,
|
||||
profile_by_stage: bool,
|
||||
merge_profiles: bool,
|
||||
profile_prefix: Optional[str],
|
||||
probe_requests: int,
|
||||
probe_prompt: str,
|
||||
probe_max_new_tokens: Optional[int],
|
||||
probe_delay: float,
|
||||
start_step: Optional[int] = None,
|
||||
framework: str = "auto",
|
||||
framework_hint_path: Optional[str] = None,
|
||||
) -> Path:
|
||||
resolved_framework = resolve_framework(
|
||||
framework,
|
||||
url=url,
|
||||
input_path=(
|
||||
Path(framework_hint_path).expanduser().resolve()
|
||||
if framework_hint_path
|
||||
else None
|
||||
),
|
||||
)
|
||||
if resolved_framework == "sglang":
|
||||
return run_sglang_profiler(
|
||||
url=url,
|
||||
output_dir=output_dir,
|
||||
num_steps=num_steps,
|
||||
profile_by_stage=profile_by_stage,
|
||||
merge_profiles=merge_profiles,
|
||||
profile_prefix=profile_prefix,
|
||||
probe_requests=probe_requests,
|
||||
probe_prompt=probe_prompt,
|
||||
probe_max_new_tokens=probe_max_new_tokens,
|
||||
probe_delay=probe_delay,
|
||||
start_step=start_step,
|
||||
)
|
||||
if start_step is not None:
|
||||
raise ValueError("--start-step is only supported for SGLang live capture.")
|
||||
if profile_by_stage:
|
||||
raise ValueError(
|
||||
"--profile-by-stage is only supported for SGLang live capture. "
|
||||
"Disable it when profiling vLLM or TensorRT-LLM."
|
||||
)
|
||||
if merge_profiles:
|
||||
raise ValueError(
|
||||
"--merge-profiles is only supported for SGLang live capture. "
|
||||
"Disable it when profiling vLLM or TensorRT-LLM."
|
||||
)
|
||||
if profile_prefix:
|
||||
print(
|
||||
f"Note: {framework_display_name(resolved_framework)} ignores "
|
||||
"--profile-prefix on the HTTP profiler control path.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return run_remote_profiler(
|
||||
url=url,
|
||||
output_dir=output_dir,
|
||||
framework=resolved_framework,
|
||||
probe_requests=probe_requests,
|
||||
probe_prompt=probe_prompt,
|
||||
probe_max_new_tokens=probe_max_new_tokens,
|
||||
probe_delay=probe_delay,
|
||||
num_steps=num_steps,
|
||||
)
|
||||
|
||||
|
||||
def select_heaviest_pid(
|
||||
events: Sequence[dict],
|
||||
event_filter: Callable[[dict], bool],
|
||||
pid_substring: Optional[str] = None,
|
||||
preferred_substrings: Iterable[str] = (),
|
||||
) -> Optional[str]:
|
||||
durations: Counter = Counter()
|
||||
for event in events:
|
||||
if not event_filter(event):
|
||||
continue
|
||||
pid = str(event.get("pid"))
|
||||
if pid_substring and pid_substring not in pid:
|
||||
continue
|
||||
durations[pid] += float(event["dur"])
|
||||
if not durations:
|
||||
return None
|
||||
|
||||
for substring in preferred_substrings:
|
||||
preferred = [pid for pid in durations if substring in pid]
|
||||
if preferred:
|
||||
return max(preferred, key=lambda pid: durations[pid])
|
||||
return max(durations, key=lambda pid: durations[pid])
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Bundle one or more triage text reports into a single markdown document."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
FRAMEWORK_LABELS = {
|
||||
"sglang": "SGLang",
|
||||
"vllm": "vLLM",
|
||||
"trtllm": "TensorRT-LLM",
|
||||
}
|
||||
|
||||
FRAMEWORK_ORDER = {"sglang": 0, "vllm": 1, "trtllm": 2}
|
||||
|
||||
|
||||
def parse_args(argv: Optional[Sequence[str]] = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Render multiple profiler triage text outputs into one markdown file. "
|
||||
"Input files are expected to be the existing analysis_*.txt outputs "
|
||||
"already emitted by analyze_llm_torch_profile.py."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--analysis-root",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Root directory to scan recursively for analysis_*.txt files. "
|
||||
"Parent directory names are used as model section ids."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--analysis-file",
|
||||
action="append",
|
||||
default=[],
|
||||
help=(
|
||||
"Explicit analysis file entry. Use either PATH or LABEL=PATH. "
|
||||
"When LABEL is omitted, the parent directory name is used."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--title",
|
||||
type=str,
|
||||
default="Unified LLM Torch Profiler Triage Bundle",
|
||||
help="Top-level markdown title.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Write the bundled markdown to this file. Prints to stdout when omitted.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-toc",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="Include a simple table of contents.",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
if not args.analysis_root and not args.analysis_file:
|
||||
parser.error("Provide at least one of --analysis-root or --analysis-file.")
|
||||
return args
|
||||
|
||||
|
||||
def framework_key_from_path(path: Path) -> str:
|
||||
lowered = path.name.lower()
|
||||
if "sglang" in lowered:
|
||||
return "sglang"
|
||||
if "vllm" in lowered:
|
||||
return "vllm"
|
||||
if "trtllm" in lowered or "tensorrt" in lowered:
|
||||
return "trtllm"
|
||||
return "other"
|
||||
|
||||
|
||||
def framework_label(framework_key: str) -> str:
|
||||
return FRAMEWORK_LABELS.get(framework_key, framework_key)
|
||||
|
||||
|
||||
def discover_analysis_files(root: Path) -> List[Tuple[str, Path]]:
|
||||
entries: List[Tuple[str, Path]] = []
|
||||
for path in sorted(root.rglob("analysis*.txt")):
|
||||
entries.append((path.parent.name, path))
|
||||
return entries
|
||||
|
||||
|
||||
def parse_explicit_entry(raw: str) -> Tuple[str, Path]:
|
||||
if "=" in raw:
|
||||
label, path_text = raw.split("=", 1)
|
||||
path = Path(path_text).expanduser().resolve()
|
||||
return label.strip(), path
|
||||
path = Path(raw).expanduser().resolve()
|
||||
return path.parent.name, path
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
chars = []
|
||||
last_dash = False
|
||||
for char in text.lower():
|
||||
if char.isalnum():
|
||||
chars.append(char)
|
||||
last_dash = False
|
||||
elif not last_dash:
|
||||
chars.append("-")
|
||||
last_dash = True
|
||||
return "".join(chars).strip("-")
|
||||
|
||||
|
||||
def extract_model_name(report_text: str) -> Optional[str]:
|
||||
for line in report_text.splitlines():
|
||||
if line.startswith("Model: "):
|
||||
return line.split("Model: ", 1)[1].strip()
|
||||
return None
|
||||
|
||||
|
||||
def choose_model_display_name(
|
||||
current: Optional[str],
|
||||
candidate: Optional[str],
|
||||
*,
|
||||
label: str,
|
||||
) -> str:
|
||||
if candidate and candidate != label:
|
||||
if not current or current == label:
|
||||
return candidate
|
||||
if len(candidate) > len(current):
|
||||
return candidate
|
||||
return current
|
||||
if current:
|
||||
return current
|
||||
return label
|
||||
|
||||
|
||||
def normalize_report_text(report_text: str) -> str:
|
||||
text = report_text.replace("\r\n", "\n").strip()
|
||||
if not text:
|
||||
return "_Empty analysis output._"
|
||||
heading_map = {
|
||||
"Triage View": "#### Triage View",
|
||||
"Kernel Table": "#### Kernel Table",
|
||||
"Overlap Opportunity Table": "#### Overlap Opportunity Table",
|
||||
"Fuse Opportunity Table": "#### Fuse Opportunity Table",
|
||||
}
|
||||
normalized_lines = []
|
||||
for line in text.splitlines():
|
||||
normalized_lines.append(heading_map.get(line, line))
|
||||
return "\n".join(normalized_lines)
|
||||
|
||||
|
||||
def build_bundle_markdown(
|
||||
*,
|
||||
title: str,
|
||||
labeled_paths: Sequence[Tuple[str, Path]],
|
||||
include_toc: bool,
|
||||
) -> str:
|
||||
grouped: Dict[str, List[Tuple[str, Path, str]]] = defaultdict(list)
|
||||
model_display: Dict[str, str] = {}
|
||||
|
||||
for label, path in labeled_paths:
|
||||
raw_text = path.read_text(encoding="utf-8")
|
||||
report_text = normalize_report_text(raw_text)
|
||||
model_name = extract_model_name(report_text)
|
||||
grouped[label].append((framework_key_from_path(path), path, report_text))
|
||||
model_display[label] = choose_model_display_name(
|
||||
model_display.get(label),
|
||||
model_name,
|
||||
label=label,
|
||||
)
|
||||
|
||||
ordered_labels = sorted(
|
||||
grouped,
|
||||
key=lambda item: (model_display[item].lower(), item.lower()),
|
||||
)
|
||||
|
||||
lines: List[str] = [f"# {title}", ""]
|
||||
lines.append(
|
||||
f"_Generated on {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}_"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
if include_toc:
|
||||
lines.append("## Contents")
|
||||
lines.append("")
|
||||
for label in ordered_labels:
|
||||
lines.append(
|
||||
f"- [{model_display[label]}](#{slugify(model_display[label])})"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
for label in ordered_labels:
|
||||
display_name = model_display[label]
|
||||
lines.append(f"## {display_name}")
|
||||
lines.append("")
|
||||
lines.append(f"Model id: `{label}`")
|
||||
lines.append("")
|
||||
|
||||
records = sorted(
|
||||
grouped[label],
|
||||
key=lambda item: (
|
||||
FRAMEWORK_ORDER.get(item[0], 99),
|
||||
item[1].name.lower(),
|
||||
),
|
||||
)
|
||||
|
||||
for framework_key, path, report_text in records:
|
||||
lines.append(f"### {framework_label(framework_key)}")
|
||||
lines.append("")
|
||||
lines.append(f"Source: `{path}`")
|
||||
lines.append("")
|
||||
lines.append(report_text)
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
args = parse_args(argv)
|
||||
|
||||
labeled_paths: List[Tuple[str, Path]] = []
|
||||
if args.analysis_root:
|
||||
labeled_paths.extend(
|
||||
discover_analysis_files(Path(args.analysis_root).expanduser().resolve())
|
||||
)
|
||||
for raw_entry in args.analysis_file:
|
||||
labeled_paths.append(parse_explicit_entry(raw_entry))
|
||||
|
||||
existing = []
|
||||
missing = []
|
||||
for label, path in labeled_paths:
|
||||
if path.is_file():
|
||||
existing.append((label, path))
|
||||
else:
|
||||
missing.append(str(path))
|
||||
if missing:
|
||||
raise SystemExit("Missing analysis files:\n" + "\n".join(missing))
|
||||
if not existing:
|
||||
raise SystemExit("No analysis files found.")
|
||||
|
||||
markdown = build_bundle_markdown(
|
||||
title=args.title,
|
||||
labeled_paths=existing,
|
||||
include_toc=args.include_toc,
|
||||
)
|
||||
|
||||
if args.output:
|
||||
output_path = Path(args.output).expanduser().resolve()
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(markdown, encoding="utf-8")
|
||||
else:
|
||||
print(markdown, end="")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user