From d230f2c35b82aa3c0b2eb11da97538ba455af745 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 16 Oct 2026 16:51:20 -0700 Subject: [PATCH] [Feat] profiler: Add the forward pass id to detailed step span labels (#44775) Co-authored-by: metamergebot Co-authored-by: charlotte12l --- .../sglang/srt/model_executor/model_runner.py | 6 +++- python/sglang/srt/utils/profile_utils.py | 5 ++++ .../managers/test_detailed_annotations.py | 30 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index e1c12af64e..6818ad7778 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1811,7 +1811,11 @@ class ModelRunner: # Step span step_span_ctx = profile_range( - build_step_span_name(forward_batch, is_draft_worker=self.is_draft_worker) + build_step_span_name( + forward_batch, + is_draft_worker=self.is_draft_worker, + forward_id=self.forward_pass_id, + ) ) canary_ctx = ( diff --git a/python/sglang/srt/utils/profile_utils.py b/python/sglang/srt/utils/profile_utils.py index 1e116da443..42866c657a 100644 --- a/python/sglang/srt/utils/profile_utils.py +++ b/python/sglang/srt/utils/profile_utils.py @@ -464,6 +464,7 @@ def build_step_span_name( detailed_annotations: bool | None = None, *, is_draft_worker: bool = False, + forward_id: int | None = None, ) -> str: """Build the profile-trace span name for one forward step. @@ -475,6 +476,8 @@ def build_step_span_name( The target-verify step is labeled ``VERIFY``; every step a draft model runner emits is labeled ``DRAFT`` (some draft paths borrow the TARGET_VERIFY mode, so the mode name alone cannot tell the two models apart). + When detailed annotations are enabled, ``forward_id`` binds the span to + the expert-distribution recorder's forward ID for this model runner. """ if detailed_annotations is None: detailed_annotations = detailed_annotations_enabled() @@ -497,4 +500,6 @@ def build_step_span_name( suffix = build_detailed_annotation_suffix(forward_batch) if suffix: base = f"{base} {suffix}" + if forward_id is not None: + base = f"{base} forward_id={forward_id}" return f"{base}]" diff --git a/test/registered/unit/managers/test_detailed_annotations.py b/test/registered/unit/managers/test_detailed_annotations.py index 13dfc47843..4208840ecc 100644 --- a/test/registered/unit/managers/test_detailed_annotations.py +++ b/test/registered/unit/managers/test_detailed_annotations.py @@ -195,6 +195,36 @@ class TestStepSpanDetailedAnnotations(CustomTestCase): fb = _fb(ForwardMode.DECODE, batch_size=2, seq_lens_cpu=None) self.assertEqual(self._name(fb), "step[DECODE bs=2]") + def test_forward_id_disambiguates_identical_shapes_and_preserves_model(self): + # Consecutive forwards can have identical shapes and prefix sums. The + # recorder's runner-local ID binds them without matching by shape/order. + fb = _fb( + ForwardMode.TARGET_VERIFY, + batch_size=2, + seq_lens_cpu=[10, 20], + num_tokens_per_req=3, + ) + for is_draft, stage in [(False, "VERIFY"), (True, "DRAFT")]: + for forward_id in (41, 42): + self.assertEqual( + build_step_span_name( + fb, + detailed_annotations=True, + is_draft_worker=is_draft, + forward_id=forward_id, + ), + f"step[{stage} bs=2 g_sq=6 g_sqsq=18 g_sqsk=90 g_sk=30 forward_id={forward_id}]", + ) + self.assertEqual( + build_step_span_name(fb, detailed_annotations=False, forward_id=42), + "step[VERIFY bs=2]", + ) + fb.seq_lens_cpu = None + self.assertEqual( + build_step_span_name(fb, detailed_annotations=True, forward_id=42), + "step[VERIFY bs=2 forward_id=42]", + ) + class TestStepSpanGating(CustomTestCase): def test_disabled_flag_emits_base_label(self):