minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Tracing] update script for converting otel tracing data to perfetto format (#19396)

Browse Source
This commit is contained in:
Feng Su
2026-02-26 14:03:22 +08:00
committed by GitHub
parent de3d1e7669
commit d2b3c7fb14
1 changed files with 188 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/convert_otel_2_perfetto.py
+188 -100
View File
@@ -136,9 +136,23 @@ def load_otel_data(path: str | Path):
def extract_all_otel_spans(otel_data): def extract_all_otel_spans(otel_data):
otel_spans = [] engine_otel_spans = []
smg_otel_spans = []
for line_data in otel_data: for line_data in otel_data:
for resource_spans in line_data["resourceSpans"]: for resource_spans in line_data["resourceSpans"]:
# filter: only keep spans which service.name is 'sglang' or 'smg'
service_name = ""
for attr in resource_spans["resource"]["attributes"]:
if attr["key"] == "service.name":
service_name = attr["value"]["stringValue"]
if service_name == "sglang":
spans_ref = engine_otel_spans
elif service_name == "smg":
spans_ref = smg_otel_spans
else:
continue
for scope_spans in resource_spans["scopeSpans"]: for scope_spans in resource_spans["scopeSpans"]:
for span in scope_spans["spans"]: for span in scope_spans["spans"]:
if "attributes" in span: if "attributes" in span:
@@ -151,8 +165,8 @@ def extract_all_otel_spans(otel_data):
span["attributes"] = attributes_dict span["attributes"] = attributes_dict
else: else:
span["attributes"] = {} span["attributes"] = {}
otel_spans.append(span) spans_ref.append(span)
return otel_spans return engine_otel_spans, smg_otel_spans
def build_otel_span_tree(otel_spans): def build_otel_span_tree(otel_spans):
@@ -160,15 +174,12 @@ def build_otel_span_tree(otel_spans):
for span in otel_spans: for span in otel_spans:
span["child"] = [] span["child"] = []
bootstrap_room_spans = [] root_spans = []
for span in otel_spans: for span in otel_spans:
span_id = span["spanId"]
parent_span_id = span.get("parentSpanId", "") parent_span_id = span.get("parentSpanId", "")
if parent_span_id == "": if span.get("attributes", {}).get("module") == "sglang::request":
# check if root span is a request span root_spans.append(span)
attrs = span.get("attributes", {})
bootstrap_room_spans.append(span)
elif parent_span_id in span_id_map: elif parent_span_id in span_id_map:
parent_span = span_id_map[parent_span_id] parent_span = span_id_map[parent_span_id]
parent_span["child"].append(span) parent_span["child"].append(span)
@@ -181,65 +192,88 @@ def build_otel_span_tree(otel_spans):
link_spans.append(link_span) link_spans.append(link_span)
span["links"] = link_spans span["links"] = link_spans
return bootstrap_room_spans return root_spans
def generate_perfetto_span(otel_bootstrap_room_spans, thread_meta_data): def __convert_to_perfetto_span(span, rid, bootstrap_room, pid, host_id):
for bootstrap_room_span in otel_bootstrap_room_spans: if bootstrap_room:
bootstrap_room = bootstrap_room_span["attributes"]["bootstrap_room"] span["attributes"]["bootstrap_room"] = bootstrap_room
bootstrap_room_span["spans"] = [] if rid:
span["attributes"]["rid"] = rid
if host_id:
span["host_id"] = host_id
span["pid"] = pid
for node_req_span in bootstrap_room_span["child"]: span["startTimeUnixNano"] = int(span["startTimeUnixNano"])
rid = node_req_span["attributes"]["rid"] span["endTimeUnixNano"] = int(span["endTimeUnixNano"]) - 1000
ts = span["startTimeUnixNano"]
dur = span["endTimeUnixNano"] - ts
for thread_span in node_req_span["child"]: perfetto_span = {
pid = int(thread_span["attributes"]["pid"]) "ph": "X",
thread_name = f'{thread_span["attributes"]["host_id"][:8]}:{thread_span["attributes"]["thread_label"]}' "name": span.get("name", "unknown"),
if "tp_rank" in thread_span["attributes"]: "cat": "sglang",
thread_name += f"-TP{thread_span['attributes']['tp_rank']}" "ts": (ts - baseline) / 1000.0,
"dur": dur / 1000.0,
"pid": pid,
"tid": 0,
"args": span["attributes"],
}
if pid not in thread_meta_data: span["perfetto_span"] = perfetto_span
thread_meta_data[pid] = new_metadata_level1(thread_name, pid)
for span in thread_span["child"]: for child_span in span["child"]:
span["attributes"]["bootstrap_room"] = bootstrap_room __convert_to_perfetto_span(child_span, rid, bootstrap_room, pid, host_id)
span["attributes"]["rid"] = rid
span["host_id"] = thread_span["attributes"]["host_id"]
span["pid"] = pid
span["startTimeUnixNano"] = int(span["startTimeUnixNano"])
span["endTimeUnixNano"] = int(span["endTimeUnixNano"])
ts = span["startTimeUnixNano"]
dur = span["endTimeUnixNano"] - ts
perfetto_span = {
"ph": "X",
"name": span.get("name", "unknown"),
"cat": "sglang",
"ts": (ts - baseline) / 1000.0,
"dur": (dur - 1000) / 1000.0,
"pid": pid,
"tid": 0,
"args": span["attributes"],
}
span["perfetto_span"] = perfetto_span
bootstrap_room_span["spans"].append(span)
def generate_perfetto_span_layout(otel_bootstrap_room_spans, slot_meta_data): def generate_perfetto_span(engine_root_spans, smg_otel_spans, thread_meta_data):
for bootstrap_room_span in otel_bootstrap_room_spans: for root_span in engine_root_spans:
bootstrap_room_span["spans"] = sorted( root_span["spans"] = []
bootstrap_room_span["spans"], key=lambda x: int(x["startTimeUnixNano"])
rid = root_span["attributes"]["rid"]
bootstrap_room = root_span["attributes"].get("bootstrap_room", "")
for thread_span in root_span["child"]:
pid = int(thread_span["attributes"]["pid"])
host_id = thread_span["attributes"]["host_id"]
thread_name = f'{thread_span["attributes"]["host_id"][:8]}:{thread_span["attributes"]["thread_label"]}'
if "tp_rank" in thread_span["attributes"]:
thread_name += f"-TP{thread_span['attributes']['tp_rank']}"
if pid not in thread_meta_data:
thread_meta_data[pid] = new_metadata_level1(thread_name, pid)
for span in thread_span["child"]:
__convert_to_perfetto_span(span, rid, bootstrap_room, pid, host_id)
root_span["spans"].append(span)
smg_pid = "smg"
thread_meta_data[smg_pid] = new_metadata_level1("smg", smg_pid)
for span in smg_otel_spans:
span["pid"] = smg_pid
__convert_to_perfetto_span(span, None, None, smg_pid, None)
def __set_span_tid(span, line):
span["perfetto_span"]["tid"] = line
for child_span in span["child"]:
__set_span_tid(child_span, line)
def generate_perfetto_span_layout(engine_root_spans, smg_otel_spans, slot_meta_data):
for root_span in engine_root_spans:
root_span["spans"] = sorted(
root_span["spans"], key=lambda x: int(x["startTimeUnixNano"])
) )
otel_bootstrap_room_spans = sorted( engine_root_spans = sorted(
otel_bootstrap_room_spans, key=lambda x: int(x["spans"][0]["startTimeUnixNano"]) engine_root_spans, key=lambda x: int(x["spans"][0]["startTimeUnixNano"])
) )
graph = {} graph = {}
for bootstrap_room_span in otel_bootstrap_room_spans: for root_span in engine_root_spans:
req_thread_status = {} req_thread_status = {}
for span in bootstrap_room_span["spans"]: for span in root_span["spans"]:
line = __find_line( line = __find_line(
graph, graph,
req_thread_status, req_thread_status,
@@ -251,52 +285,90 @@ def generate_perfetto_span_layout(otel_bootstrap_room_spans, slot_meta_data):
graph[span["perfetto_span"]["pid"]][line].insert_span( graph[span["perfetto_span"]["pid"]][line].insert_span(
span["startTimeUnixNano"], span["endTimeUnixNano"] span["startTimeUnixNano"], span["endTimeUnixNano"]
) )
span["perfetto_span"]["tid"] = line __set_span_tid(span, line)
smg_otel_spans = sorted(smg_otel_spans, key=lambda x: int(x["startTimeUnixNano"]))
req_thread_status = {}
for span in smg_otel_spans:
line = __find_line(
graph,
req_thread_status,
slot_meta_data,
span["perfetto_span"]["pid"],
span["startTimeUnixNano"],
span["endTimeUnixNano"],
)
graph[span["perfetto_span"]["pid"]][line].insert_span(
span["startTimeUnixNano"], span["endTimeUnixNano"]
)
span["perfetto_span"]["tid"] = line
def generate_perfetto_events(otel_bootstrap_room_spans): def __convert_to_perfetto_events(span):
for bootstrap_room_span in otel_bootstrap_room_spans: span["perfetto_events"] = []
for span in bootstrap_room_span["spans"]: if "events" in span:
span["perfetto_events"] = [] for event in span["events"]:
if "events" in span: attributes_dict = {
for event in span["events"]: attr.get("key"): next(iter(attr.get("value", {}).values()), None)
attributes_dict = { for attr in event["attributes"]
attr.get("key"): next( }
iter(attr.get("value", {}).values()), None perfetto_event = {
) "ph": "i",
for attr in event["attributes"] "cat": "sglang",
} "ts": (int(event["timeUnixNano"]) - baseline) / 1000.0,
perfetto_event = { "pid": span["perfetto_span"]["pid"],
"ph": "i", "tid": span["perfetto_span"]["tid"],
"cat": "sglang", "name": event.get("name", "unknown"),
"ts": (int(event["timeUnixNano"]) - baseline) / 1000.0, "args": attributes_dict,
"pid": span["perfetto_span"]["pid"], }
"tid": span["perfetto_span"]["tid"],
"name": event.get("name", "unknown"),
"args": attributes_dict,
}
span["perfetto_events"].append(perfetto_event) span["perfetto_events"].append(perfetto_event)
for child_span in span["child"]:
__convert_to_perfetto_events(child_span)
def generate_perfetto_links(otel_bootstrap_room_spans): def generate_perfetto_events(engine_root_spans, smg_otel_spans):
for bootstrap_room_span in otel_bootstrap_room_spans: spans = [span for root_span in engine_root_spans for span in root_span["spans"]]
for span in bootstrap_room_span["spans"]:
for span in spans:
__convert_to_perfetto_events(span)
for span in smg_otel_spans:
__convert_to_perfetto_events(span)
def generate_perfetto_links(engine_root_spans, smg_otel_spans):
# build link between engine span and smg span
span_id_map = {span["spanId"]: span for span in smg_otel_spans}
for root_span in engine_root_spans:
if "parentSpanId" in root_span and root_span["parentSpanId"] in span_id_map:
parent_span = span_id_map[root_span["parentSpanId"]]
root_span["spans"][0]["links"] = [parent_span]
for span in root_span["spans"]:
span["perfetto_links"] = [] span["perfetto_links"] = []
if "links" in span: if "links" in span:
for link_span in span["links"]: for link_span in span["links"]:
if "correlation" in link_span["perfetto_span"]["args"]: try:
id = link_span["perfetto_span"]["args"]["correlation"] link_perfetto_span = link_span["perfetto_span"]
except (KeyError, AttributeError):
continue
if "correlation" in link_perfetto_span["args"]:
id = link_perfetto_span["args"]["correlation"]
else: else:
id = next(relation_id_gen) id = next(relation_id_gen)
link_span["perfetto_span"]["args"]["correlation"] = id link_perfetto_span["args"]["correlation"] = id
perfetto_start_node = { perfetto_start_node = {
"ph": "s", "ph": "s",
"id": id, "id": id,
"pid": link_span["perfetto_span"]["pid"], "pid": link_perfetto_span["pid"],
"tid": link_span["perfetto_span"]["tid"], "tid": link_perfetto_span["tid"],
"ts": link_span["perfetto_span"]["ts"], "ts": link_perfetto_span["ts"],
"cat": "ac2g", "cat": "ac2g",
"name": "ac2g", "name": "ac2g",
} }
@@ -316,17 +388,33 @@ def generate_perfetto_links(otel_bootstrap_room_spans):
span["perfetto_links"].append(perfetto_end_node) span["perfetto_links"].append(perfetto_end_node)
def __gather_one_span(span):
elems = []
elems.append(span["perfetto_span"])
if "perfetto_events" in span:
elems.extend(span["perfetto_events"])
if "perfetto_links" in span:
elems.extend(span["perfetto_links"])
for child_span in span["child"]:
elems.extend(__gather_one_span(child_span))
return elems
def gather_all_perfetto_elems( def gather_all_perfetto_elems(
otel_bootstrap_room_spans, thread_meta_data, slot_meta_data engine_root_spans, smg_otel_spans, thread_meta_data, slot_meta_data
): ):
elems = [] elems = []
elems.extend(thread_meta_data.values()) elems.extend(thread_meta_data.values())
elems.extend(slot_meta_data) elems.extend(slot_meta_data)
for bootstrap_room_span in otel_bootstrap_room_spans: for root_span in engine_root_spans:
for span in bootstrap_room_span["spans"]: for span in root_span["spans"]:
elems.append(span["perfetto_span"]) elems.extend(__gather_one_span(span))
elems.extend(span["perfetto_events"])
elems.extend(span["perfetto_links"]) for span in smg_otel_spans:
elems.append(span["perfetto_span"])
elems.extend(span["perfetto_events"])
return elems return elems
@@ -352,16 +440,16 @@ def write_json(perfetto_elems):
def main(): def main():
start_time = time.time() start_time = time.time()
otel_data = load_otel_data(args.input_file) otel_data = load_otel_data(args.input_file)
otel_spans = extract_all_otel_spans(otel_data) engine_otel_spans, smg_otel_spans = extract_all_otel_spans(otel_data)
otel_bootstrap_room_spans = build_otel_span_tree(otel_spans) engine_root_spans = build_otel_span_tree(engine_otel_spans)
thread_meta_data = {} thread_meta_data = {}
generate_perfetto_span(otel_bootstrap_room_spans, thread_meta_data) generate_perfetto_span(engine_root_spans, smg_otel_spans, thread_meta_data)
slot_meta_data = [] slot_meta_data = []
generate_perfetto_span_layout(otel_bootstrap_room_spans, slot_meta_data) generate_perfetto_span_layout(engine_root_spans, smg_otel_spans, slot_meta_data)
generate_perfetto_events(otel_bootstrap_room_spans) generate_perfetto_events(engine_root_spans, smg_otel_spans)
generate_perfetto_links(otel_bootstrap_room_spans) generate_perfetto_links(engine_root_spans, smg_otel_spans)
perfetto_elems = gather_all_perfetto_elems( perfetto_elems = gather_all_perfetto_elems(
otel_bootstrap_room_spans, thread_meta_data, slot_meta_data engine_root_spans, smg_otel_spans, thread_meta_data, slot_meta_data
) )
write_json(perfetto_elems) write_json(perfetto_elems)
end_time = time.time() end_time = time.time()